diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4cac639 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,36 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + # Pinned to the version this repo's scripts/build-wasm.sh was built + # and verified against -- see that file's header comment. + - uses: mymindstorm/setup-emsdk@v14 + with: + version: 6.0.5 + + - name: Install dependencies + run: npm install + + - name: Build native addon + run: npm run build + + - name: Build WASM backend + run: npm run build:wasm + + - name: Test + run: npm test diff --git a/appveyor.yml b/appveyor.yml index 2591b60..00fd075 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,17 +1,15 @@ -os: Visual Studio 2015 +os: Visual Studio 2022 version: "{build}" build: off platform: x64 environment: matrix: - - nodejs_version: "8" + - nodejs_version: "20" nodejs_arch: "x64" - - nodejs_version: "10" - nodejs_arch: "x64" - - nodejs_version: "12" + - nodejs_version: "22" nodejs_arch: "x64" install: - ps: Install-Product node $env:nodejs_version $env:nodejs_arch - - npm install --msvs_version=2015 + - npm install test_script: - npm test diff --git a/bin/build-native.js b/bin/build-native.js new file mode 100644 index 0000000..2f64803 --- /dev/null +++ b/bin/build-native.js @@ -0,0 +1,23 @@ +#!/usr/bin/env node +// Runs as the package's "install" script. A failed native build here must +// NOT fail the overall `npm install` -- lib/backend.js falls back to the +// WASM backend at runtime when the native addon isn't present, but that +// fallback is unreachable if npm never finishes installing in the first +// place. So: try the native build, but always exit 0. + +const { spawnSync } = require('child_process'); + +const result = spawnSync('node-gyp', ['rebuild'], { + stdio: 'inherit', + shell: true +}); + +if (result.status !== 0) { + console.warn( + '\n[cld] Native build failed (no working C++ toolchain for this ' + + 'platform?) -- this is not fatal. cld will use its WASM fallback ' + + 'backend at runtime instead of the native addon.\n' + ); +} + +process.exit(0); diff --git a/bin/postinstall.js b/bin/postinstall.js index ff51ddb..6d45f83 100755 --- a/bin/postinstall.js +++ b/bin/postinstall.js @@ -8,7 +8,7 @@ deleteBuildFiles(function(err) { function deleteBuildFiles(cb) { var pattern = path.resolve(__dirname, '..', 'build', '**', '*'); - glob(pattern, {nodir:true}, function(err, files) { + glob.globSync(pattern, {nodir:true}, function(err, files) { if (err) { return cb(err); } diff --git a/binding.gyp b/binding.gyp index 12cc5fa..045cbfb 100644 --- a/binding.gyp +++ b/binding.gyp @@ -9,7 +9,7 @@ "deps/cld/public", " string }): void; + export declare function detect(text: string, options: Options, callback: (err: string, result: DetectLanguage) => void): void; export declare function detect(text: string, callback: (err: string, result: DetectLanguage) => void): void; export declare function detect(text: string, options: Options): Promise; diff --git a/index.js b/index.js index 72109a8..a891f97 100644 --- a/index.js +++ b/index.js @@ -1,92 +1,10 @@ -const _ = require('underscore'); -const cld2 = require('./build/Release/cld'); +const meta = require('./lib/metadata.json'); +const backend = require('./lib/backend'); +const { createDetect } = require('./lib/detect-shape'); module.exports = { - LANGUAGES : cld2.LANGUAGES, - DETECTED_LANGUAGES : cld2.DETECTED_LANGUAGES, - ENCODINGS : cld2.ENCODINGS, - - async detect(text, options) { - let cb = arguments[2]; - if (typeof cb !== 'function' && typeof options === 'function') { - cb = options; - options = {}; - } - - try { - if (arguments.length < 1) { - throw new Error('Not enough arguments provided'); - } - - if (!_.isString(text) || text.length < 1) { - throw new Error('Empty or invalid text'); - } - - const defaults = { - isHTML : false, - languageHint : '', - encodingHint : '', - tldHint : '', - httpHint : '', - bestEffort : false - }; - options = _.defaults({}, options, defaults); - - if (!_.isBoolean(options.isHTML)) { - throw new Error('Invalid isHTML value'); - } - if (!_.isBoolean(options.bestEffort)) { - throw new Error('Invalid bestEffort value'); - } - if (!_.isString(options.languageHint)) { - throw new Error('Invalid languageHint'); - } - if (!_.isString(options.encodingHint)) { - throw new Error('Invalid encodingHint'); - } - if (!_.isString(options.tldHint)) { - throw new Error('Invalid tldHint'); - } - if (!_.isString(options.httpHint)) { - throw new Error('Invalid httpHint'); - } - if (options.encodingHint.length > 0 && - !~cld2.ENCODINGS.indexOf(options.encodingHint)) { - - throw new Error('Invalid encodingHint, see ENCODINGS'); - } - if (options.languageHint.length > 0 && - !~_.keys(cld2.LANGUAGES).indexOf(options.languageHint) && - !~_.values(cld2.LANGUAGES).indexOf(options.languageHint)) { - - throw new Error('Invalid languageHint, see LANGUAGES'); - } - - const result = await cld2.detectAsync( - text, - !options.isHTML, - options.languageHint, - options.encodingHint, - options.tldHint, - options.httpHint, - options.bestEffort - ); - - if (result.languages.length < 1) { - throw new Error('Failed to identify language'); - } - - if (cb) { - return cb(null, result); - } else { - return result; - } - } catch (error) { - if (cb) { - cb(error); - } else { - throw error; - } - } - } + LANGUAGES : meta.LANGUAGES, + DETECTED_LANGUAGES : meta.DETECTED_LANGUAGES, + ENCODINGS : meta.ENCODINGS, + detect : createDetect(backend.loadBackend, meta) }; diff --git a/lib/backend.js b/lib/backend.js new file mode 100644 index 0000000..08b84e3 --- /dev/null +++ b/lib/backend.js @@ -0,0 +1,30 @@ +// Resolves which compiled backend (native N-API addon, or the WASM +// fallback) actually runs detectAsync(). Both backends expose the same +// {detectAsync(text, isPlainText, languageHint, encodingHint, tldHint, +// httpHint, bestEffort)} shape, so index.js needs no branching beyond +// awaiting loadBackend() once. + +const { wrapWasmModule } = require('./wasm-wrap'); + +let backendPromise = null; + +function loadBackend() { + if (!backendPromise) { + backendPromise = (async () => { + try { + return require('../build/Release/cld'); + } catch (nativeErr) { + return loadWasmBackend(); + } + })(); + } + return backendPromise; +} + +async function loadWasmBackend() { + const createCldModule = require('../wasm/dist/cld.node.js'); + const mod = await createCldModule(); + return wrapWasmModule(mod); +} + +module.exports = { loadBackend, loadWasmBackend }; diff --git a/lib/detect-shape.js b/lib/detect-shape.js new file mode 100644 index 0000000..5f5567e --- /dev/null +++ b/lib/detect-shape.js @@ -0,0 +1,95 @@ +// Shared detect() implementation used by both index.js (Node, native-first +// with WASM fallback) and wasm/browser-entry.mjs (bundler/browser, WASM +// only) -- the validation/defaults/error-mapping logic is identical for +// both, only how the backend is loaded differs. + +const _ = require('underscore'); + +function createDetect(loadBackend, meta) { + return async function detect(text, options) { + let cb = arguments[2]; + if (typeof cb !== 'function' && typeof options === 'function') { + cb = options; + options = {}; + } + + try { + if (arguments.length < 1) { + throw new Error('Not enough arguments provided'); + } + + if (!_.isString(text) || text.length < 1) { + throw new Error('Empty or invalid text'); + } + + const defaults = { + isHTML : false, + languageHint : '', + encodingHint : '', + tldHint : '', + httpHint : '', + bestEffort : false + }; + options = _.defaults({}, options, defaults); + + if (!_.isBoolean(options.isHTML)) { + throw new Error('Invalid isHTML value'); + } + if (!_.isBoolean(options.bestEffort)) { + throw new Error('Invalid bestEffort value'); + } + if (!_.isString(options.languageHint)) { + throw new Error('Invalid languageHint'); + } + if (!_.isString(options.encodingHint)) { + throw new Error('Invalid encodingHint'); + } + if (!_.isString(options.tldHint)) { + throw new Error('Invalid tldHint'); + } + if (!_.isString(options.httpHint)) { + throw new Error('Invalid httpHint'); + } + if (options.encodingHint.length > 0 && + !~meta.ENCODINGS.indexOf(options.encodingHint)) { + + throw new Error('Invalid encodingHint, see ENCODINGS'); + } + if (options.languageHint.length > 0 && + !~_.keys(meta.LANGUAGES).indexOf(options.languageHint) && + !~_.values(meta.LANGUAGES).indexOf(options.languageHint)) { + + throw new Error('Invalid languageHint, see LANGUAGES'); + } + + const cld2 = await loadBackend(); + const result = await cld2.detectAsync( + text, + !options.isHTML, + options.languageHint, + options.encodingHint, + options.tldHint, + options.httpHint, + options.bestEffort + ); + + if (result.languages.length < 1) { + throw new Error('Failed to identify language'); + } + + if (cb) { + return cb(null, result); + } else { + return result; + } + } catch (error) { + if (cb) { + cb(error); + } else { + throw error; + } + } + }; +} + +module.exports = { createDetect }; diff --git a/lib/metadata.json b/lib/metadata.json new file mode 100644 index 0000000..c0ed773 --- /dev/null +++ b/lib/metadata.json @@ -0,0 +1,529 @@ +{ + "LANGUAGES": { + "ABKHAZIAN": "ab", + "AFAR": "aa", + "AFRIKAANS": "af", + "AKAN": "ak", + "ALBANIAN": "sq", + "AMHARIC": "am", + "ARABIC": "ar", + "ARMENIAN": "hy", + "ASSAMESE": "as", + "AYMARA": "ay", + "AZERBAIJANI": "az", + "BASHKIR": "ba", + "BASQUE": "eu", + "BELARUSIAN": "be", + "BENGALI": "bn", + "BIHARI": "bh", + "BISLAMA": "bi", + "BOSNIAN": "bs", + "BRETON": "br", + "BULGARIAN": "bg", + "BURMESE": "my", + "CATALAN": "ca", + "CEBUANO": "ceb", + "CHEROKEE": "chr", + "NYANJA": "ny", + "CORSICAN": "co", + "CROATIAN": "hr", + "CZECH": "cs", + "Chinese": "zh", + "ChineseT": "zh-Hant", + "DANISH": "da", + "DHIVEHI": "dv", + "DUTCH": "nl", + "DZONGKHA": "dz", + "ENGLISH": "en", + "ESPERANTO": "eo", + "ESTONIAN": "et", + "EWE": "ee", + "FAROESE": "fo", + "FIJIAN": "fj", + "FINNISH": "fi", + "FRENCH": "fr", + "FRISIAN": "fy", + "GA": "gaa", + "GALICIAN": "gl", + "GANDA": "lg", + "GEORGIAN": "ka", + "GERMAN": "de", + "GREEK": "el", + "GREENLANDIC": "kl", + "GUARANI": "gn", + "GUJARATI": "gu", + "HAITIAN_CREOLE": "ht", + "HAUSA": "ha", + "HAWAIIAN": "haw", + "HEBREW": "iw", + "HINDI": "hi", + "HMONG": "hmn", + "HUNGARIAN": "hu", + "ICELANDIC": "is", + "IGBO": "ig", + "INDONESIAN": "id", + "INTERLINGUA": "ia", + "INTERLINGUE": "ie", + "INUKTITUT": "iu", + "INUPIAK": "ik", + "IRISH": "ga", + "ITALIAN": "it", + "Ignore": "xxx", + "JAVANESE": "jw", + "Japanese": "ja", + "KANNADA": "kn", + "KASHMIRI": "ks", + "KAZAKH": "kk", + "KHASI": "kha", + "KHMER": "km", + "KINYARWANDA": "rw", + "KRIO": "kri", + "KURDISH": "ku", + "KYRGYZ": "ky", + "Korean": "ko", + "LAOTHIAN": "lo", + "LATIN": "la", + "LATVIAN": "lv", + "LIMBU": "lif", + "LINGALA": "ln", + "LITHUANIAN": "lt", + "LOZI": "loz", + "LUBA_LULUA": "lua", + "LUO_KENYA_AND_TANZANIA": "luo", + "LUXEMBOURGISH": "lb", + "MACEDONIAN": "mk", + "MALAGASY": "mg", + "MALAY": "ms", + "MALAYALAM": "ml", + "MALTESE": "mt", + "MANX": "gv", + "MAORI": "mi", + "MARATHI": "mr", + "MAURITIAN_CREOLE": "mfe", + "ROMANIAN": "ro", + "MONGOLIAN": "mn", + "MONTENEGRIN": "sr-ME", + "NAURU": "na", + "NDEBELE": "nr", + "NEPALI": "ne", + "NEWARI": "new", + "NORWEGIAN": "no", + "NORWEGIAN_N": "nn", + "OCCITAN": "oc", + "ORIYA": "or", + "OROMO": "om", + "OSSETIAN": "os", + "PAMPANGA": "pam", + "PASHTO": "ps", + "PEDI": "nso", + "PERSIAN": "fa", + "POLISH": "pl", + "PORTUGUESE": "pt", + "PUNJABI": "pa", + "QUECHUA": "qu", + "RAJASTHANI": "raj", + "RHAETO_ROMANCE": "rm", + "RUNDI": "rn", + "RUSSIAN": "ru", + "SAMOAN": "sm", + "SANGO": "sg", + "SANSKRIT": "sa", + "SCOTS": "sco", + "SCOTS_GAELIC": "gd", + "SERBIAN": "sr", + "SESELWA": "crs", + "SESOTHO": "st", + "SHONA": "sn", + "SINDHI": "sd", + "SINHALESE": "si", + "SISWANT": "ss", + "SLOVAK": "sk", + "SLOVENIAN": "sl", + "SOMALI": "so", + "SPANISH": "es", + "SUNDANESE": "su", + "SWAHILI": "sw", + "SWEDISH": "sv", + "SYRIAC": "syr", + "TAGALOG": "tl", + "TAJIK": "tg", + "TAMIL": "ta", + "TATAR": "tt", + "TELUGU": "te", + "THAI": "th", + "TIBETAN": "bo", + "TIGRINYA": "ti", + "TONGA": "to", + "TSONGA": "ts", + "TSWANA": "tn", + "TUMBUKA": "tum", + "TURKISH": "tr", + "TURKMEN": "tk", + "TWI": "tw", + "UIGHUR": "ug", + "UKRAINIAN": "uk", + "URDU": "ur", + "UZBEK": "uz", + "VENDA": "ve", + "VIETNAMESE": "vi", + "VOLAPUK": "vo", + "WARAY_PHILIPPINES": "war", + "WELSH": "cy", + "WOLOF": "wo", + "XHOSA": "xh", + "X_Arabic": "xx-Arab", + "X_Armenian": "xx-Armn", + "X_Avestan": "xx-Avst", + "X_BORK_BORK_BORK": "zzb", + "X_Balinese": "xx-Bali", + "X_Bamum": "xx-Bamu", + "X_Batak": "xx-Batk", + "X_Bengali": "xx-Beng", + "X_Bopomofo": "xx-Bopo", + "X_Brahmi": "xx-Brah", + "X_Braille": "xx-Brai", + "X_Buginese": "xx-Bugi", + "X_Buhid": "xx-Buhd", + "X_Canadian_Aboriginal": "xx-Cans", + "X_Carian": "xx-Cari", + "X_Chakma": "xx-Cakm", + "X_Cham": "xx-Cham", + "X_Cherokee": "xx-Cher", + "X_Common": "xx-Zyyy", + "X_Coptic": "xx-Copt", + "X_Cuneiform": "xx-Xsux", + "X_Cypriot": "xx-Cprt", + "X_Cyrillic": "xx-Cyrl", + "X_Deseret": "xx-Dsrt", + "X_Devanagari": "xx-Deva", + "X_ELMER_FUDD": "zze", + "X_Egyptian_Hieroglyphs": "xx-Egyp", + "X_Ethiopic": "xx-Ethi", + "X_Georgian": "xx-Geor", + "X_Glagolitic": "xx-Glag", + "X_Gothic": "xx-Goth", + "X_Greek": "xx-Grek", + "X_Gujarati": "xx-Gujr", + "X_Gurmukhi": "xx-Guru", + "X_HACKER": "zzh", + "X_Han": "xx-Hani", + "X_Hangul": "xx-Hang", + "X_Hanunoo": "xx-Hano", + "X_Hebrew": "xx-Hebr", + "X_Hiragana": "xx-Hira", + "X_Imperial_Aramaic": "xx-Armi", + "X_Inherited": "xx-Qaai", + "X_Inscriptional_Pahlavi": "xx-Phli", + "X_Inscriptional_Parthian": "xx-Prti", + "X_Javanese": "xx-Java", + "X_KLINGON": "tlh", + "X_Kaithi": "xx-Kthi", + "X_Kannada": "xx-Knda", + "X_Katakana": "xx-Kana", + "X_Kayah_Li": "xx-Kali", + "X_Kharoshthi": "xx-Khar", + "X_Khmer": "xx-Khmr", + "X_Lao": "xx-Laoo", + "X_Latin": "xx-Latn", + "X_Lepcha": "xx-Lepc", + "X_Limbu": "xx-Limb", + "X_Linear_B": "xx-Linb", + "X_Lisu": "xx-Lisu", + "X_Lycian": "xx-Lyci", + "X_Lydian": "xx-Lydi", + "X_Malayalam": "xx-Mlym", + "X_Mandaic": "xx-Mand", + "X_Meetei_Mayek": "xx-Mtei", + "X_Meroitic_Cursive": "xx-Merc", + "X_Meroitic_Hieroglyphs": "xx-Mero", + "X_Miao": "xx-Plrd", + "X_Mongolian": "xx-Mong", + "X_Myanmar": "xx-Mymr", + "X_New_Tai_Lue": "xx-Talu", + "X_Nko": "xx-Nkoo", + "X_Ogham": "xx-Ogam", + "X_Ol_Chiki": "xx-Olck", + "X_Old_Italic": "xx-Ital", + "X_Old_Persian": "xx-Xpeo", + "X_Old_South_Arabian": "xx-Sarb", + "X_Old_Turkic": "xx-Orkh", + "X_Oriya": "xx-Orya", + "X_Osmanya": "xx-Osma", + "X_PIG_LATIN": "zzp", + "X_Phags_Pa": "xx-Phag", + "X_Phoenician": "xx-Phnx", + "X_Rejang": "xx-Rjng", + "X_Runic": "xx-Runr", + "X_Samaritan": "xx-Samr", + "X_Saurashtra": "xx-Saur", + "X_Sharada": "xx-Shrd", + "X_Shavian": "xx-Shaw", + "X_Sinhala": "xx-Sinh", + "X_Sora_Sompeng": "xx-Sora", + "X_Sundanese": "xx-Sund", + "X_Syloti_Nagri": "xx-Sylo", + "X_Syriac": "xx-Syrc", + "X_Tagalog": "xx-Tglg", + "X_Tagbanwa": "xx-Tagb", + "X_Tai_Le": "xx-Tale", + "X_Tai_Tham": "xx-Lana", + "X_Tai_Viet": "xx-Tavt", + "X_Takri": "xx-Takr", + "X_Tamil": "xx-Taml", + "X_Telugu": "xx-Telu", + "X_Thaana": "xx-Thaa", + "X_Thai": "xx-Thai", + "X_Tibetan": "xx-Tibt", + "X_Tifinagh": "xx-Tfng", + "X_Ugaritic": "xx-Ugar", + "X_Vai": "xx-Vaii", + "X_Yi": "xx-Yiii", + "YIDDISH": "yi", + "YORUBA": "yo", + "ZHUANG": "za", + "ZULU": "zu" + }, + "DETECTED_LANGUAGES": [ + "ABKHAZIAN", + "AFAR", + "AFRIKAANS", + "AKAN", + "ALBANIAN", + "AMHARIC", + "ARABIC", + "ARMENIAN", + "ASSAMESE", + "AYMARA", + "AZERBAIJANI", + "BASHKIR", + "BASQUE", + "BELARUSIAN", + "BENGALI", + "BIHARI", + "BISLAMA", + "BOSNIAN", + "BRETON", + "BULGARIAN", + "BURMESE", + "CATALAN", + "CEBUANO", + "CHEROKEE", + "CORSICAN", + "CROATIAN", + "CZECH", + "Chinese", + "ChineseT", + "DANISH", + "DHIVEHI", + "DUTCH", + "DZONGKHA", + "ENGLISH", + "ESPERANTO", + "ESTONIAN", + "FAROESE", + "FIJIAN", + "FINNISH", + "FRENCH", + "FRISIAN", + "GALICIAN", + "GANDA", + "GEORGIAN", + "GERMAN", + "GREEK", + "GREENLANDIC", + "GUARANI", + "GUJARATI", + "HAITIAN_CREOLE", + "HAUSA", + "HAWAIIAN", + "HEBREW", + "HINDI", + "HMONG", + "HUNGARIAN", + "ICELANDIC", + "IGBO", + "INDONESIAN", + "INTERLINGUA", + "INTERLINGUE", + "INUKTITUT", + "INUPIAK", + "IRISH", + "ITALIAN", + "JAVANESE", + "Japanese", + "KANNADA", + "KASHMIRI", + "KAZAKH", + "KHASI", + "KHMER", + "KINYARWANDA", + "KURDISH", + "KYRGYZ", + "Korean", + "LAOTHIAN", + "LATIN", + "LATVIAN", + "LIMBU", + "LINGALA", + "LITHUANIAN", + "LUXEMBOURGISH", + "MACEDONIAN", + "MALAGASY", + "MALAY", + "MALAYALAM", + "MALTESE", + "MANX", + "MAORI", + "MARATHI", + "MAURITIAN_CREOLE", + "MONGOLIAN", + "NAURU", + "NDEBELE", + "NEPALI", + "NORWEGIAN", + "NORWEGIAN_N", + "NYANJA", + "OCCITAN", + "ORIYA", + "OROMO", + "PASHTO", + "PEDI", + "PERSIAN", + "POLISH", + "PORTUGUESE", + "PUNJABI", + "QUECHUA", + "RHAETO_ROMANCE", + "ROMANIAN", + "RUNDI", + "RUSSIAN", + "SAMOAN", + "SANGO", + "SANSKRIT", + "SCOTS", + "SCOTS_GAELIC", + "SERBIAN", + "SESELWA", + "SESOTHO", + "SHONA", + "SINDHI", + "SINHALESE", + "SISWANT", + "SLOVAK", + "SLOVENIAN", + "SOMALI", + "SPANISH", + "SUNDANESE", + "SWAHILI", + "SWEDISH", + "SYRIAC", + "TAGALOG", + "TAJIK", + "TAMIL", + "TATAR", + "TELUGU", + "THAI", + "TIBETAN", + "TIGRINYA", + "TONGA", + "TSONGA", + "TSWANA", + "TURKISH", + "TURKMEN", + "UIGHUR", + "UKRAINIAN", + "URDU", + "UZBEK", + "VENDA", + "VIETNAMESE", + "VOLAPUK", + "WARAY_PHILIPPINES", + "WELSH", + "WOLOF", + "XHOSA", + "X_Buginese", + "X_Gothic", + "X_KLINGON", + "X_PIG_LATIN", + "YIDDISH", + "YORUBA", + "ZHUANG", + "ZULU" + ], + "ENCODINGS": [ + "ISO_8859_1", + "ISO_8859_2", + "ISO_8859_3", + "ISO_8859_4", + "ISO_8859_5", + "ISO_8859_6", + "ISO_8859_7", + "ISO_8859_8", + "ISO_8859_9", + "ISO_8859_10", + "JAPANESE_EUC_JP", + "JAPANESE_SHIFT_JIS", + "JAPANESE_JIS", + "CHINESE_BIG5", + "CHINESE_GB", + "CHINESE_EUC_CN", + "KOREAN_EUC_KR", + "UNICODE_UNUSED", + "CHINESE_EUC_DEC", + "CHINESE_CNS", + "CHINESE_BIG5_CP950", + "JAPANESE_CP932", + "UTF8", + "ASCII_7BIT", + "RUSSIAN_KOI8_R", + "RUSSIAN_CP1251", + "MSFT_CP1252", + "RUSSIAN_KOI8_RU", + "MSFT_CP1250", + "ISO_8859_15", + "MSFT_CP1254", + "MSFT_CP1257", + "ISO_8859_11", + "MSFT_CP874", + "MSFT_CP1256", + "MSFT_CP1255", + "ISO_8859_8_I", + "HEBREW_VISUAL", + "CZECH_CP852", + "CZECH_CSN_369103", + "MSFT_CP1253", + "RUSSIAN_CP866", + "ISO_8859_13", + "ISO_2022_KR", + "GBK", + "GB18030", + "BIG5_HKSCS", + "ISO_2022_CN", + "TSCII", + "TAMIL_MONO", + "TAMIL_BI", + "JAGRAN", + "MACINTOSH_ROMAN", + "UTF7", + "BHASKAR", + "HTCHANAKYA", + "UTF16BE", + "UTF16LE", + "UTF32BE", + "UTF32LE", + "BINARYENC", + "HZ_GB_2312", + "UTF8UTF8", + "TAM_ELANGO", + "TAM_LTTMBARANI", + "TAM_SHREE", + "TAM_TBOOMIS", + "TAM_TMNEWS", + "TAM_WEBTAMIL", + "KDDI_SHIFT_JIS", + "DOCOMO_SHIFT_JIS", + "SOFTBANK_SHIFT_JIS", + "KDDI_ISO_2022_JP", + "SOFTBANK_ISO_2022_JP" + ] +} diff --git a/lib/wasm-wrap.js b/lib/wasm-wrap.js new file mode 100644 index 0000000..28b00ac --- /dev/null +++ b/lib/wasm-wrap.js @@ -0,0 +1,29 @@ +// Wraps an instantiated Emscripten module (Node or browser target -- both +// expose the same ccall/UTF8ToString shape) into the {detectAsync(...)} +// interface shared with the native backend. Deliberately has no other +// requires: wasm/browser-entry.mjs imports this directly (not through +// lib/backend.js) so bundlers never see lib/backend.js's +// require('../build/Release/cld') at all. + +function wrapWasmModule(mod) { + return { + async detectAsync(text, isPlainText, languageHint, encodingHint, tldHint, httpHint, bestEffort) { + const numBytes = new TextEncoder().encode(text).length; + const ptr = mod.ccall( + 'cld_detect', 'number', + ['string', 'number', 'number', 'string', 'string', 'string', 'string', 'number'], + [text, numBytes, isPlainText ? 1 : 0, languageHint, encodingHint, tldHint, httpHint, bestEffort ? 1 : 0] + ); + try { + return JSON.parse(mod.UTF8ToString(ptr)); + } finally { + // Must free explicitly: ccall's 'number' return type gives us the + // raw pointer (not emscripten's 'string' convenience type, which + // decodes the string but never frees the underlying C buffer). + mod.ccall('cld_free', null, ['number'], [ptr]); + } + } + }; +} + +module.exports = { wrapWasmModule }; diff --git a/package-lock.json b/package-lock.json index baf194e..ad2bab2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,95 +1,248 @@ { "name": "cld", - "version": "2.10.0", - "lockfileVersion": 1, + "version": "2.10.2", + "lockfileVersion": 3, "requires": true, - "dependencies": { - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "node-addon-api": { + "packages": { + "": { + "name": "cld", + "version": "2.10.2", + "hasInstallScript": true, + "dependencies": { + "glob": "^12", + "node-addon-api": "^2.0.0", + "underscore": "^1.13.7" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/cliui": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", + "integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-12.0.0.tgz", + "integrity": "sha512-5Qcll1z7IKgHr5g485ePDdHcNQY0k2dtv/bjYy0iuyGxQw2qSOiiXUXJ+AYQpg3HNoUMHqAruX478Jeev7UULw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", + "integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^9.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/node-addon-api": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.0.tgz", "integrity": "sha512-ASCL5U13as7HhOExbT6OlWJJUV/lLzL2voOSP1UVehpRD8FbSrSDjfScK/KwAvVTI5AS6r4VwbOMlIqtvRidnA==" }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" } }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "underscore": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", - "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/underscore": { + "version": "1.13.8", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", + "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", + "license": "MIT" + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } } } } diff --git a/package.json b/package.json index 6313205..5083fc1 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,39 @@ "cld", "cld2" ], - "version": "2.10.1", + "version": "2.10.2", "main": "./index.js", "types": "./index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "browser": "./wasm/browser-entry.mjs", + "default": "./index.js" + } + }, + "files": [ + "index.js", + "index.d.ts", + "lib", + "wasm/dist", + "wasm/browser-entry.mjs", + "src", + "deps/cld/public", + "deps/cld/internal", + "deps/cld/binding.gyp", + "binding.gyp", + "bin" + ], "dependencies": { - "glob": "7", - "node-addon-api": "*", - "underscore": "^1.12.1" + "glob": "^12", + "node-addon-api": "^2.0.0", + "underscore": "^1.13.7" }, "scripts": { + "install": "node bin/build-native.js", "build": "node-gyp rebuild", - "test": "node test/runner.js", + "build:wasm": "scripts/build-wasm.sh", + "test": "node test/runner.js && node test/runner-wasm.js && node test/runner-wasm-browser.js", "postinstall": "node bin/postinstall.js" }, "author": { @@ -40,6 +62,6 @@ } ], "engines": { - "node": ">=12.0.0" + "node": "20 || >=22" } } diff --git a/scripts/build-wasm.sh b/scripts/build-wasm.sh new file mode 100755 index 0000000..96829f0 --- /dev/null +++ b/scripts/build-wasm.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Builds the WASM fallback backend from the same CLD2 sources used by +# binding.gyp's native build. Requires the Emscripten SDK on PATH (built and +# verified with emsdk 6.0.5 / emcc 6.0.5) -- see https://emscripten.org/docs/getting_started/downloads.html +# or run: source /path/to/emsdk/emsdk_env.sh +set -euo pipefail + +cd "$(dirname "$0")/.." + +if ! command -v emcc >/dev/null 2>&1; then + echo "error: emcc not found on PATH. Install/activate the Emscripten SDK first." >&2 + exit 1 +fi + +CLD_INTERNAL=deps/cld/internal + +# Mirrors deps/cld/binding.gyp's cld-c target source list exactly -- keep +# this in sync if that file's list ever changes. +CLD_SOURCES=( + "$CLD_INTERNAL/cldutil.cc" + "$CLD_INTERNAL/cldutil_shared.cc" + "$CLD_INTERNAL/compact_lang_det.cc" + "$CLD_INTERNAL/compact_lang_det_hint_code.cc" + "$CLD_INTERNAL/compact_lang_det_impl.cc" + "$CLD_INTERNAL/debug.cc" + "$CLD_INTERNAL/fixunicodevalue.cc" + "$CLD_INTERNAL/generated_entities.cc" + "$CLD_INTERNAL/generated_language.cc" + "$CLD_INTERNAL/generated_ulscript.cc" + "$CLD_INTERNAL/getonescriptspan.cc" + "$CLD_INTERNAL/lang_script.cc" + "$CLD_INTERNAL/offsetmap.cc" + "$CLD_INTERNAL/scoreonescriptspan.cc" + "$CLD_INTERNAL/tote.cc" + "$CLD_INTERNAL/utf8statetable.cc" + "$CLD_INTERNAL/cld_generated_cjk_uni_prop_80.cc" + "$CLD_INTERNAL/cld2_generated_cjk_compatible.cc" + "$CLD_INTERNAL/cld_generated_cjk_delta_bi_32.cc" + "$CLD_INTERNAL/generated_distinct_bi_0.cc" + "$CLD_INTERNAL/cld2_generated_quad0122.cc" + "$CLD_INTERNAL/cld2_generated_deltaocta0122.cc" + "$CLD_INTERNAL/cld2_generated_deltaoctachrome.cc" + "$CLD_INTERNAL/cld2_generated_distinctocta0122.cc" + "$CLD_INTERNAL/cld2_generated_distinctoctachrome.cc" + "$CLD_INTERNAL/cld2_generated_quadchrome_16.cc" + "$CLD_INTERNAL/cld2_generated_quadchrome_2.cc" + "$CLD_INTERNAL/cld_generated_score_quad_octa_0122.cc" + "$CLD_INTERNAL/cld_generated_score_quad_octa_2.cc" +) + +GLUE_SOURCES=( + src/constants.cc + src/cld_core.cc + wasm/glue.cc +) + +COMMON_FLAGS=( + -O3 + -std=gnu++98 -w + -fno-exceptions + -I deps/cld/public + -I "$CLD_INTERNAL" + -I src + -s MODULARIZE=1 + -s EXPORTED_FUNCTIONS=_cld_detect,_cld_free,_malloc,_free + -s EXPORTED_RUNTIME_METHODS=ccall,UTF8ToString + -s ALLOW_MEMORY_GROWTH=1 + -s WASM_ASYNC_COMPILATION=1 + # deps/cld's generated data-table files intentionally define overlapping + # symbols across translation units (the native build tolerates this via + # binding.gyp's "-z muldefs" ldflag) -- wasm-ld's equivalent: + -Wl,--allow-multiple-definition +) + +mkdir -p wasm/dist + +echo "Building Node target -> wasm/dist/cld.node.js" +emcc \ + "${COMMON_FLAGS[@]}" \ + -s ENVIRONMENT=node \ + -s EXPORT_NAME=createCldModule \ + "${CLD_SOURCES[@]}" "${GLUE_SOURCES[@]}" \ + -o wasm/dist/cld.node.js + +echo "Building browser target -> wasm/dist/cld.web.mjs" +emcc \ + "${COMMON_FLAGS[@]}" \ + -s ENVIRONMENT=web \ + -s EXPORT_ES6=1 \ + -s EXPORT_NAME=createCldModule \ + "${CLD_SOURCES[@]}" "${GLUE_SOURCES[@]}" \ + -o wasm/dist/cld.web.mjs + +echo "Done." diff --git a/scripts/generate-metadata.js b/scripts/generate-metadata.js new file mode 100644 index 0000000..1daaf17 --- /dev/null +++ b/scripts/generate-metadata.js @@ -0,0 +1,34 @@ +#!/usr/bin/env node +// Dev-only script, NOT part of npm install/publish. +// +// LANGUAGES/DETECTED_LANGUAGES/ENCODINGS are pure compile-time-static data +// (from Constants::init() in src/constants.cc), identical regardless of +// which backend (native or WASM) ends up running detect(). Rather than +// have index.js depend on either backend just to read constant strings +// (WASM instantiation is async, which would force async metadata exports +// too), snapshot them once here into a checked-in JSON file that both +// backends' index.js consumers read synchronously. +// +// Re-run this (`node scripts/generate-metadata.js`) only if deps/cld's +// language/encoding tables are ever upgraded -- CLD2 upstream is frozen, +// so in practice this should rarely, if ever, need to run again. Requires a +// native build to exist first (`npm run build`). + +const fs = require('fs'); +const path = require('path'); + +const cld2 = require('../build/Release/cld'); + +const metadata = { + LANGUAGES: cld2.LANGUAGES, + DETECTED_LANGUAGES: cld2.DETECTED_LANGUAGES, + ENCODINGS: cld2.ENCODINGS +}; + +const outPath = path.resolve(__dirname, '..', 'lib', 'metadata.json'); +fs.writeFileSync(outPath, JSON.stringify(metadata, null, 2) + '\n'); + +console.log(`Wrote ${outPath}`); +console.log(` LANGUAGES: ${Object.keys(metadata.LANGUAGES).length} entries`); +console.log(` DETECTED_LANGUAGES: ${metadata.DETECTED_LANGUAGES.length} entries`); +console.log(` ENCODINGS: ${metadata.ENCODINGS.length} entries`); diff --git a/scripts/generate-wasm-fixtures.js b/scripts/generate-wasm-fixtures.js new file mode 100644 index 0000000..e55ee3e --- /dev/null +++ b/scripts/generate-wasm-fixtures.js @@ -0,0 +1,43 @@ +#!/usr/bin/env node +// Dev-only, maintainer-run script -- NOT part of npm install/publish. +// +// Captures a golden snapshot of detect() results from the native build, +// for test/runner-wasm.js to compare the WASM backend against. This is +// deliberately a fixed, checked-in snapshot rather than a live comparison +// against "whatever native build happens to be present" -- native builds +// on different platforms/compilers (verified: MSVC vs GCC/Clang) can +// produce different scores/chunk boundaries for the same CLD2 source, a +// pre-existing quirk in CLD2's own algorithm unrelated to the WASM port. +// The WASM build is proven to match native exactly when built from the +// same compiler family (Clang, via emcc) as this snapshot was captured +// with, so comparing against a fixed snapshot -- rather than trusting +// whatever native binary a given CI runner happens to produce -- is what +// actually makes this test portable across platforms. +// +// Re-run this (`node scripts/generate-wasm-fixtures.js`) only if +// test/data.js's fixtures change. Requires a native build to exist first +// (`npm run build`), built on a GCC/Clang-family toolchain (Linux/macOS). + +const fs = require('fs'); +const path = require('path'); + +const data = require('../test/data'); +const meta = require('../lib/metadata.json'); +const { createDetect } = require('../lib/detect-shape'); + +const native = require('../build/Release/cld'); +const detect = createDetect(async () => native, meta); + +(async () => { + const fixtures = []; + for (const item of data.all) { + fixtures.push({ + default: await detect(item.sample), + bestEffort: await detect(item.sample, { bestEffort: true }) + }); + } + + const outPath = path.resolve(__dirname, '..', 'test', 'wasm-fixtures.json'); + fs.writeFileSync(outPath, JSON.stringify(fixtures, null, 2) + '\n'); + console.log(`Wrote ${outPath} (${fixtures.length} fixtures)`); +})(); diff --git a/src/cld.cc b/src/cld.cc index fc6d343..96ec485 100644 --- a/src/cld.cc +++ b/src/cld.cc @@ -4,6 +4,7 @@ #include "compact_lang_det.h" #include "encodings.h" #include "constants.h" +#include "cld_core.h" using std::terminate_handler; @@ -11,26 +12,6 @@ using std::terminate_handler; #include namespace NodeCld { - struct CLDInput { - std::string bytes, - languageHint, - encodingHint, - tldHint, - httpHint; - int numBytes; - bool isPlainText; - bool bestEffort; - }; - - struct CLDOutput { - CLD2::Language language3[3]; - int percent3[3]; - double normalized_score3[3]; - CLD2::ResultChunkVector resultChunkVector; - int textBytesFound; - bool isReliable; - }; - std::unique_ptr UnpackInputFromJSArgs(const Napi::CallbackInfo &info) { std::unique_ptr input(new CLDInput); @@ -58,51 +39,6 @@ namespace NodeCld { return input; } - std::unique_ptr DetectLanguage(std::unique_ptr input) { - std::unique_ptr output(new CLDOutput); - CLD2::CLDHints hints; - hints.tld_hint = 0; - hints.content_language_hint = 0; - hints.language_hint = CLD2::UNKNOWN_LANGUAGE; - hints.encoding_hint = CLD2::UNKNOWN_ENCODING; - - if (input->languageHint.length() > 0) { - hints.language_hint = Constants::getInstance().getLanguageFromName(input->languageHint.c_str()); - } - - if (input->encodingHint.length() > 0) { - hints.encoding_hint = Constants::getInstance().getEncodingFromName(input->encodingHint.c_str()); - } - - if (input->tldHint.length() > 0) { - hints.tld_hint = input->tldHint.c_str(); - } - - if (input->httpHint.length() > 0) { - hints.content_language_hint = input->httpHint.c_str(); - } - int flags = 0; - if (input->bestEffort) { - flags |= CLD2::kCLDFlagBestEffort; - } - - CLD2::ExtDetectLanguageSummary( - input->bytes.c_str(), - input->numBytes, - input->isPlainText, - &hints, - flags, - output->language3, - output->percent3, - output->normalized_score3, - &output->resultChunkVector, - &output->textBytesFound, - &output->isReliable - ); - - return output; - } - Napi::Object UnpackOutputToJS(const Napi::Env env, std::unique_ptr output) { size_t languageIdx = 0; auto languages = Napi::Array::New(env); @@ -159,7 +95,7 @@ namespace NodeCld { {} void Execute() { - mOutput = DetectLanguage(std::move(mInput)); + mOutput = DetectLanguage(*mInput); } void OnOK() { @@ -179,7 +115,7 @@ namespace NodeCld { Napi::Object Detect(const Napi::CallbackInfo &info) { auto input = UnpackInputFromJSArgs(info); - auto output = DetectLanguage(std::move(input)); + auto output = DetectLanguage(*input); return UnpackOutputToJS(info.Env(), std::move(output)); } @@ -218,6 +154,7 @@ namespace NodeCld { exports["detectAsync"] = Napi::Function::New(env, DetectAsync); return exports; } - - NODE_API_MODULE(cld, Init); } + +using NodeCld::Init; +NODE_API_MODULE(cld, Init); diff --git a/src/cld_core.cc b/src/cld_core.cc new file mode 100644 index 0000000..fcc5fda --- /dev/null +++ b/src/cld_core.cc @@ -0,0 +1,50 @@ +#include "cld_core.h" +#include "encodings.h" +#include "constants.h" + +namespace NodeCld { + std::unique_ptr DetectLanguage(const CLDInput &input) { + std::unique_ptr output(new CLDOutput); + CLD2::CLDHints hints; + hints.tld_hint = 0; + hints.content_language_hint = 0; + hints.language_hint = CLD2::UNKNOWN_LANGUAGE; + hints.encoding_hint = CLD2::UNKNOWN_ENCODING; + + if (input.languageHint.length() > 0) { + hints.language_hint = Constants::getInstance().getLanguageFromName(input.languageHint.c_str()); + } + + if (input.encodingHint.length() > 0) { + hints.encoding_hint = Constants::getInstance().getEncodingFromName(input.encodingHint.c_str()); + } + + if (input.tldHint.length() > 0) { + hints.tld_hint = input.tldHint.c_str(); + } + + if (input.httpHint.length() > 0) { + hints.content_language_hint = input.httpHint.c_str(); + } + int flags = 0; + if (input.bestEffort) { + flags |= CLD2::kCLDFlagBestEffort; + } + + CLD2::ExtDetectLanguageSummary( + input.bytes.c_str(), + input.numBytes, + input.isPlainText, + &hints, + flags, + output->language3, + output->percent3, + output->normalized_score3, + &output->resultChunkVector, + &output->textBytesFound, + &output->isReliable + ); + + return output; + } +} diff --git a/src/cld_core.h b/src/cld_core.h new file mode 100644 index 0000000..5805be2 --- /dev/null +++ b/src/cld_core.h @@ -0,0 +1,33 @@ +#ifndef NODE_CLD_CORE_H +#define NODE_CLD_CORE_H + +#include +#include + +#include "compact_lang_det.h" + +namespace NodeCld { + struct CLDInput { + std::string bytes, + languageHint, + encodingHint, + tldHint, + httpHint; + int numBytes; + bool isPlainText; + bool bestEffort; + }; + + struct CLDOutput { + CLD2::Language language3[3]; + int percent3[3]; + double normalized_score3[3]; + CLD2::ResultChunkVector resultChunkVector; + int textBytesFound; + bool isReliable; + }; + + std::unique_ptr DetectLanguage(const CLDInput &input); +} + +#endif diff --git a/test/runner-wasm-browser.js b/test/runner-wasm-browser.js new file mode 100644 index 0000000..5c01fc2 --- /dev/null +++ b/test/runner-wasm-browser.js @@ -0,0 +1,76 @@ +#!/usr/bin/env node +// Proves wasm/browser-entry.mjs's setWasmModuleOptions({ locateFile }) hook +// actually controls where the WASM backend fetches cld.web.wasm from -- +// not just that the option is accepted, but that redirecting it to a +// custom URL is what makes detection succeed. +// +// Each case runs in its own child process: browser-entry.mjs caches +// moduleOptions/modulePromise at module scope on first use, so "no +// override" and "with override" can't share one process/import. +// +// With no override, browser-entry.mjs's default WASM URL resolves against +// import.meta.url, which is a file:// URL here -- and Node's fetch() +// doesn't support file://, so detect() is expected to reject. That's not a +// Node quirk being worked around, it's exactly the gap the override exists +// for, and it doubles as proof that the override (not some other +// already-working path) is what makes the second case below succeed. + +const assert = require('assert'); +const http = require('http'); +const path = require('path'); +const fs = require('fs'); +const { execFile } = require('child_process'); +const { promisify } = require('util'); + +const execFileAsync = promisify(execFile); + +const SAMPLE = require('./data').basic[0]; +assert.equal(SAMPLE.name, 'ENGLISH'); + +const wasmBinary = fs.readFileSync(path.join(__dirname, '..', 'wasm', 'dist', 'cld.web.wasm')); + +// Runs the child asynchronously (not execFileSync): the second case below +// needs an HTTP server alive *in this same process* to answer the child's +// request, which a synchronous, event-loop-blocking child_process call +// would deadlock against. +function runChild(script) { + return execFileAsync(process.execPath, ['-e', script], { encoding: 'utf8', cwd: __dirname }); +} + +(async () => { + const withoutOverride = ` + import('../wasm/browser-entry.mjs') + .then(m => m.detect(${JSON.stringify(SAMPLE.sample)})) + .then(() => { console.log('UNEXPECTED_SUCCESS'); process.exit(0); }) + .catch(() => { console.log('EXPECTED_FAILURE'); process.exit(0); }); + `; + let out1; + try { + out1 = (await runChild(withoutOverride)).stdout; + } catch (e) { + throw new Error(`child process crashed running detect() without an override: ${e.message}`); + } + assert.match(out1, /EXPECTED_FAILURE/, 'detect() should fail without a locateFile override (file:// is not fetchable in Node)'); + + const server = http.createServer((req, res) => res.end(wasmBinary)); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const port = server.address().port; + + try { + const withOverride = ` + import('../wasm/browser-entry.mjs').then(async m => { + m.setWasmModuleOptions({ locateFile: () => 'http://127.0.0.1:${port}/custom-path/cld.web.wasm' }); + const result = await m.detect(${JSON.stringify(SAMPLE.sample)}); + console.log(JSON.stringify(result.languages[0])); + process.exit(0); + }).catch(e => { console.error(e.stack); process.exit(1); }); + `; + const out2 = (await runChild(withOverride)).stdout; + const topLanguage = JSON.parse(out2.trim().split('\n').pop()); + assert.equal(topLanguage.name, 'ENGLISH', 'detect() with a locateFile override should still detect correctly'); + } finally { + server.close(); + } + + console.log('Browser entry setWasmModuleOptions()/locateFile override verified'); +})(); diff --git a/test/runner-wasm.js b/test/runner-wasm.js new file mode 100644 index 0000000..6b809f6 --- /dev/null +++ b/test/runner-wasm.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// Confirms the WASM backend reproduces the reference CLD2 behavior, +// captured once as test/wasm-fixtures.json (see +// scripts/generate-wasm-fixtures.js for why this compares against a fixed +// snapshot rather than a live native build: native builds on different +// platforms/compilers -- verified: MSVC vs GCC/Clang -- can produce +// different results for the same CLD2 source and the same input, a +// pre-existing quirk in CLD2's own algorithm unrelated to the WASM port. +// Comparing against a fixed snapshot makes this test give the same answer +// on every platform, rather than depending on whatever native build a +// given CI runner happens to produce. + +const assert = require('assert'); +const data = require('./data'); +const fixtures = require('./wasm-fixtures.json'); +const meta = require('../lib/metadata.json'); + +const { createDetect } = require('../lib/detect-shape'); +const { loadWasmBackend } = require('../lib/backend'); + +const wasmBackendPromise = loadWasmBackend(); // instantiate once, reuse for every fixture +const detectWasm = createDetect(() => wasmBackendPromise, meta); + +(async () => { + assert.equal(fixtures.length, data.all.length, 'test/wasm-fixtures.json is out of sync with test/data.js -- re-run scripts/generate-wasm-fixtures.js'); + + for (let i = 0; i < data.all.length; i++) { + const item = data.all[i]; + const expected = fixtures[i]; + + const wasmResult = await detectWasm(item.sample); + assert.deepStrictEqual(wasmResult, expected.default, `WASM mismatch for ${item.name} (default options)`); + + const wasmBestEffort = await detectWasm(item.sample, { bestEffort: true }); + assert.deepStrictEqual(wasmBestEffort, expected.bestEffort, `WASM mismatch for ${item.name} (bestEffort)`); + } + + console.log(`WASM backend verified against ${data.all.length} reference fixtures`); +})(); diff --git a/test/wasm-fixtures.json b/test/wasm-fixtures.json new file mode 100644 index 0000000..90a15a5 --- /dev/null +++ b/test/wasm-fixtures.json @@ -0,0 +1,7870 @@ +[ + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "ENGLISH", + "code": "en", + "percent": 99, + "score": 1311 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "ENGLISH", + "code": "en", + "percent": 99, + "score": 1311 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "ARMENIAN", + "code": "hy", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "ARMENIAN", + "code": "hy", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "ARMENIAN", + "code": "hy", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "ARMENIAN", + "code": "hy", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 75, + "languages": [ + { + "name": "CHEROKEE", + "code": "chr", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "CHEROKEE", + "code": "chr", + "offset": 0, + "bytes": 73 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 75, + "languages": [ + { + "name": "CHEROKEE", + "code": "chr", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "CHEROKEE", + "code": "chr", + "offset": 0, + "bytes": 73 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "DHIVEHI", + "code": "dv", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "DHIVEHI", + "code": "dv", + "offset": 0, + "bytes": 248 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "DHIVEHI", + "code": "dv", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "DHIVEHI", + "code": "dv", + "offset": 0, + "bytes": 248 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 233, + "languages": [ + { + "name": "GEORGIAN", + "code": "ka", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "GEORGIAN", + "code": "ka", + "offset": 0, + "bytes": 232 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 233, + "languages": [ + { + "name": "GEORGIAN", + "code": "ka", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "GEORGIAN", + "code": "ka", + "offset": 0, + "bytes": 232 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "GREEK", + "code": "el", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "GREEK", + "code": "el", + "offset": 0, + "bytes": 241 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "GREEK", + "code": "el", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "GREEK", + "code": "el", + "offset": 0, + "bytes": 241 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "GUJARATI", + "code": "gu", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "GUJARATI", + "code": "gu", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "GUJARATI", + "code": "gu", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "GUJARATI", + "code": "gu", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "INUKTITUT", + "code": "iu", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "INUKTITUT", + "code": "iu", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "INUKTITUT", + "code": "iu", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "INUKTITUT", + "code": "iu", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "KANNADA", + "code": "kn", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "KANNADA", + "code": "kn", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "KANNADA", + "code": "kn", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "KANNADA", + "code": "kn", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 187, + "languages": [ + { + "name": "KHMER", + "code": "km", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "KHMER", + "code": "km", + "offset": 0, + "bytes": 186 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 187, + "languages": [ + { + "name": "KHMER", + "code": "km", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "KHMER", + "code": "km", + "offset": 0, + "bytes": 186 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "LAOTHIAN", + "code": "lo", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "LAOTHIAN", + "code": "lo", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "LAOTHIAN", + "code": "lo", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "LAOTHIAN", + "code": "lo", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 580, + "languages": [ + { + "name": "LIMBU", + "code": "lif", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "LIMBU", + "code": "lif", + "offset": 0, + "bytes": 615 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 580, + "languages": [ + { + "name": "LIMBU", + "code": "lif", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "LIMBU", + "code": "lif", + "offset": 0, + "bytes": 615 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "MALAYALAM", + "code": "ml", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "MALAYALAM", + "code": "ml", + "offset": 0, + "bytes": 246 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "MALAYALAM", + "code": "ml", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "MALAYALAM", + "code": "ml", + "offset": 0, + "bytes": 246 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 48, + "languages": [ + { + "name": "ORIYA", + "code": "or", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "ORIYA", + "code": "or", + "offset": 0, + "bytes": 46 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 48, + "languages": [ + { + "name": "ORIYA", + "code": "or", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "ORIYA", + "code": "or", + "offset": 0, + "bytes": 46 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "PUNJABI", + "code": "pa", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "PUNJABI", + "code": "pa", + "offset": 0, + "bytes": 246 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "PUNJABI", + "code": "pa", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "PUNJABI", + "code": "pa", + "offset": 0, + "bytes": 246 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 243, + "languages": [ + { + "name": "SINHALESE", + "code": "si", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "SINHALESE", + "code": "si", + "offset": 0, + "bytes": 242 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 243, + "languages": [ + { + "name": "SINHALESE", + "code": "si", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "SINHALESE", + "code": "si", + "offset": 0, + "bytes": 242 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 143, + "languages": [ + { + "name": "SYRIAC", + "code": "syr", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "SYRIAC", + "code": "syr", + "offset": 0, + "bytes": 141 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 143, + "languages": [ + { + "name": "SYRIAC", + "code": "syr", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "SYRIAC", + "code": "syr", + "offset": 0, + "bytes": 141 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 228, + "languages": [ + { + "name": "TAGALOG", + "code": "tl", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "TAGALOG", + "code": "tl", + "offset": 0, + "bytes": 227 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 228, + "languages": [ + { + "name": "TAGALOG", + "code": "tl", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "TAGALOG", + "code": "tl", + "offset": 0, + "bytes": 227 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 227, + "languages": [ + { + "name": "TAMIL", + "code": "ta", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "TAMIL", + "code": "ta", + "offset": 0, + "bytes": 226 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 227, + "languages": [ + { + "name": "TAMIL", + "code": "ta", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "TAMIL", + "code": "ta", + "offset": 0, + "bytes": 226 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "TELUGU", + "code": "te", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "TELUGU", + "code": "te", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "TELUGU", + "code": "te", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "TELUGU", + "code": "te", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "THAI", + "code": "th", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "THAI", + "code": "th", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "THAI", + "code": "th", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "THAI", + "code": "th", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "Chinese", + "code": "zh", + "percent": 99, + "score": 2031 + } + ], + "chunks": [ + { + "name": "Chinese", + "code": "zh", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "Chinese", + "code": "zh", + "percent": 99, + "score": 2031 + } + ], + "chunks": [ + { + "name": "Chinese", + "code": "zh", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 185, + "languages": [ + { + "name": "ChineseT", + "code": "zh-Hant", + "percent": 99, + "score": 1908 + } + ], + "chunks": [ + { + "name": "ChineseT", + "code": "zh-Hant", + "offset": 0, + "bytes": 184 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 185, + "languages": [ + { + "name": "ChineseT", + "code": "zh-Hant", + "percent": 99, + "score": 1908 + } + ], + "chunks": [ + { + "name": "ChineseT", + "code": "zh-Hant", + "offset": 0, + "bytes": 184 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 239, + "languages": [ + { + "name": "Japanese", + "code": "ja", + "percent": 99, + "score": 3295 + } + ], + "chunks": [ + { + "name": "Japanese", + "code": "ja", + "offset": 0, + "bytes": 238 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 239, + "languages": [ + { + "name": "Japanese", + "code": "ja", + "percent": 99, + "score": 3295 + } + ], + "chunks": [ + { + "name": "Japanese", + "code": "ja", + "offset": 0, + "bytes": 238 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "Korean", + "code": "ko", + "percent": 99, + "score": 3710 + } + ], + "chunks": [ + { + "name": "Korean", + "code": "ko", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "Korean", + "code": "ko", + "percent": 99, + "score": 3710 + } + ], + "chunks": [ + { + "name": "Korean", + "code": "ko", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "AFRIKAANS", + "code": "af", + "percent": 99, + "score": 886 + } + ], + "chunks": [ + { + "name": "AFRIKAANS", + "code": "af", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "AFRIKAANS", + "code": "af", + "percent": 99, + "score": 886 + } + ], + "chunks": [ + { + "name": "AFRIKAANS", + "code": "af", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "ALBANIAN", + "code": "sq", + "percent": 99, + "score": 1614 + } + ], + "chunks": [ + { + "name": "ALBANIAN", + "code": "sq", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "ALBANIAN", + "code": "sq", + "percent": 99, + "score": 1614 + } + ], + "chunks": [ + { + "name": "ALBANIAN", + "code": "sq", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 39, + "languages": [ + { + "name": "ARABIC", + "code": "ar", + "percent": 97, + "score": 889 + } + ], + "chunks": [] + }, + "bestEffort": { + "reliable": true, + "textBytes": 39, + "languages": [ + { + "name": "ARABIC", + "code": "ar", + "percent": 97, + "score": 889 + } + ], + "chunks": [] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "AZERBAIJANI", + "code": "az", + "percent": 99, + "score": 1336 + } + ], + "chunks": [ + { + "name": "AZERBAIJANI", + "code": "az", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "AZERBAIJANI", + "code": "az", + "percent": 99, + "score": 1336 + } + ], + "chunks": [ + { + "name": "AZERBAIJANI", + "code": "az", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "BASQUE", + "code": "eu", + "percent": 99, + "score": 1488 + } + ], + "chunks": [ + { + "name": "BASQUE", + "code": "eu", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "BASQUE", + "code": "eu", + "percent": 99, + "score": 1488 + } + ], + "chunks": [ + { + "name": "BASQUE", + "code": "eu", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "BELARUSIAN", + "code": "be", + "percent": 99, + "score": 1040 + } + ], + "chunks": [ + { + "name": "BELARUSIAN", + "code": "be", + "offset": 0, + "bytes": 248 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "BELARUSIAN", + "code": "be", + "percent": 99, + "score": 1040 + } + ], + "chunks": [ + { + "name": "BELARUSIAN", + "code": "be", + "offset": 0, + "bytes": 248 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 1965, + "languages": [ + { + "name": "BENGALI", + "code": "bn", + "percent": 99, + "score": 614 + } + ], + "chunks": [ + { + "name": "BENGALI", + "code": "bn", + "offset": 0, + "bytes": 2015 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 1965, + "languages": [ + { + "name": "BENGALI", + "code": "bn", + "percent": 99, + "score": 614 + } + ], + "chunks": [ + { + "name": "BENGALI", + "code": "bn", + "offset": 0, + "bytes": 2015 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 1999, + "languages": [ + { + "name": "BIHARI", + "code": "bh", + "percent": 99, + "score": 973 + } + ], + "chunks": [ + { + "name": "BIHARI", + "code": "bh", + "offset": 0, + "bytes": 2017 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 1999, + "languages": [ + { + "name": "BIHARI", + "code": "bh", + "percent": 99, + "score": 973 + } + ], + "chunks": [ + { + "name": "BIHARI", + "code": "bh", + "offset": 0, + "bytes": 2017 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 217, + "languages": [ + { + "name": "BULGARIAN", + "code": "bg", + "percent": 99, + "score": 692 + } + ], + "chunks": [ + { + "name": "BULGARIAN", + "code": "bg", + "offset": 0, + "bytes": 216 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 217, + "languages": [ + { + "name": "BULGARIAN", + "code": "bg", + "percent": 99, + "score": 692 + } + ], + "chunks": [ + { + "name": "BULGARIAN", + "code": "bg", + "offset": 0, + "bytes": 216 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "CATALAN", + "code": "ca", + "percent": 99, + "score": 795 + } + ], + "chunks": [ + { + "name": "CATALAN", + "code": "ca", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "CATALAN", + "code": "ca", + "percent": 99, + "score": 795 + } + ], + "chunks": [ + { + "name": "CATALAN", + "code": "ca", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 309, + "languages": [ + { + "name": "CEBUANO", + "code": "ceb", + "percent": 99, + "score": 721 + } + ], + "chunks": [ + { + "name": "CEBUANO", + "code": "ceb", + "offset": 87, + "bytes": 225 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 309, + "languages": [ + { + "name": "CEBUANO", + "code": "ceb", + "percent": 99, + "score": 721 + } + ], + "chunks": [ + { + "name": "CEBUANO", + "code": "ceb", + "offset": 87, + "bytes": 225 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 494, + "languages": [ + { + "name": "CROATIAN", + "code": "hr", + "percent": 95, + "score": 406 + }, + { + "name": "GREEK", + "code": "el", + "percent": 4, + "score": 1024 + } + ], + "chunks": [ + { + "name": "CROATIAN", + "code": "hr", + "offset": 0, + "bytes": 35 + }, + { + "name": "GREEK", + "code": "el", + "offset": 35, + "bytes": 26 + }, + { + "name": "CROATIAN", + "code": "hr", + "offset": 61, + "bytes": 453 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 494, + "languages": [ + { + "name": "CROATIAN", + "code": "hr", + "percent": 95, + "score": 406 + }, + { + "name": "GREEK", + "code": "el", + "percent": 4, + "score": 1024 + } + ], + "chunks": [ + { + "name": "CROATIAN", + "code": "hr", + "offset": 0, + "bytes": 35 + }, + { + "name": "GREEK", + "code": "el", + "offset": 35, + "bytes": 26 + }, + { + "name": "CROATIAN", + "code": "hr", + "offset": 61, + "bytes": 453 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "CZECH", + "code": "cs", + "percent": 99, + "score": 1357 + } + ], + "chunks": [ + { + "name": "CZECH", + "code": "cs", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "CZECH", + "code": "cs", + "percent": 99, + "score": 1357 + } + ], + "chunks": [ + { + "name": "CZECH", + "code": "cs", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "DANISH", + "code": "da", + "percent": 99, + "score": 930 + } + ], + "chunks": [ + { + "name": "DANISH", + "code": "da", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "DANISH", + "code": "da", + "percent": 99, + "score": 930 + } + ], + "chunks": [ + { + "name": "DANISH", + "code": "da", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "DUTCH", + "code": "nl", + "percent": 99, + "score": 970 + } + ], + "chunks": [ + { + "name": "DUTCH", + "code": "nl", + "offset": 0, + "bytes": 248 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "DUTCH", + "code": "nl", + "percent": 99, + "score": 970 + } + ], + "chunks": [ + { + "name": "DUTCH", + "code": "nl", + "offset": 0, + "bytes": 248 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "ENGLISH", + "code": "en", + "percent": 99, + "score": 962 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "ENGLISH", + "code": "en", + "percent": 99, + "score": 962 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 124, + "languages": [ + { + "name": "ESTONIAN", + "code": "et", + "percent": 99, + "score": 1048 + } + ], + "chunks": [ + { + "name": "ESTONIAN", + "code": "et", + "offset": 0, + "bytes": 123 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 124, + "languages": [ + { + "name": "ESTONIAN", + "code": "et", + "percent": 99, + "score": 1048 + } + ], + "chunks": [ + { + "name": "ESTONIAN", + "code": "et", + "offset": 0, + "bytes": 123 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "FINNISH", + "code": "fi", + "percent": 99, + "score": 1249 + } + ], + "chunks": [ + { + "name": "FINNISH", + "code": "fi", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "FINNISH", + "code": "fi", + "percent": 99, + "score": 1249 + } + ], + "chunks": [ + { + "name": "FINNISH", + "code": "fi", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "FRENCH", + "code": "fr", + "percent": 99, + "score": 983 + } + ], + "chunks": [ + { + "name": "FRENCH", + "code": "fr", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "FRENCH", + "code": "fr", + "percent": 99, + "score": 983 + } + ], + "chunks": [ + { + "name": "FRENCH", + "code": "fr", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 214, + "languages": [ + { + "name": "GALICIAN", + "code": "gl", + "percent": 99, + "score": 841 + } + ], + "chunks": [ + { + "name": "GALICIAN", + "code": "gl", + "offset": 0, + "bytes": 214 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 214, + "languages": [ + { + "name": "GALICIAN", + "code": "gl", + "percent": 99, + "score": 841 + } + ], + "chunks": [ + { + "name": "GALICIAN", + "code": "gl", + "offset": 0, + "bytes": 214 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "GANDA", + "code": "lg", + "percent": 99, + "score": 738 + } + ], + "chunks": [ + { + "name": "GANDA", + "code": "lg", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "GANDA", + "code": "lg", + "percent": 99, + "score": 738 + } + ], + "chunks": [ + { + "name": "GANDA", + "code": "lg", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "GERMAN", + "code": "de", + "percent": 99, + "score": 987 + } + ], + "chunks": [ + { + "name": "GERMAN", + "code": "de", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "GERMAN", + "code": "de", + "percent": 99, + "score": 987 + } + ], + "chunks": [ + { + "name": "GERMAN", + "code": "de", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "HAITIAN_CREOLE", + "code": "ht", + "percent": 99, + "score": 1344 + } + ], + "chunks": [ + { + "name": "HAITIAN_CREOLE", + "code": "ht", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "HAITIAN_CREOLE", + "code": "ht", + "percent": 99, + "score": 1344 + } + ], + "chunks": [ + { + "name": "HAITIAN_CREOLE", + "code": "ht", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 136, + "languages": [ + { + "name": "HEBREW", + "code": "iw", + "percent": 99, + "score": 834 + } + ], + "chunks": [ + { + "name": "HEBREW", + "code": "iw", + "offset": 0, + "bytes": 135 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 136, + "languages": [ + { + "name": "HEBREW", + "code": "iw", + "percent": 99, + "score": 834 + } + ], + "chunks": [ + { + "name": "HEBREW", + "code": "iw", + "offset": 0, + "bytes": 135 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "HINDI", + "code": "hi", + "percent": 99, + "score": 814 + } + ], + "chunks": [ + { + "name": "HINDI", + "code": "hi", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "HINDI", + "code": "hi", + "percent": 99, + "score": 814 + } + ], + "chunks": [ + { + "name": "HINDI", + "code": "hi", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 171, + "languages": [ + { + "name": "HMONG", + "code": "hmn", + "percent": 99, + "score": 2132 + } + ], + "chunks": [ + { + "name": "HMONG", + "code": "hmn", + "offset": 0, + "bytes": 170 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 171, + "languages": [ + { + "name": "HMONG", + "code": "hmn", + "percent": 99, + "score": 2132 + } + ], + "chunks": [ + { + "name": "HMONG", + "code": "hmn", + "offset": 0, + "bytes": 170 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "HUNGARIAN", + "code": "hu", + "percent": 99, + "score": 1436 + } + ], + "chunks": [ + { + "name": "HUNGARIAN", + "code": "hu", + "offset": 0, + "bytes": 246 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "HUNGARIAN", + "code": "hu", + "percent": 99, + "score": 1436 + } + ], + "chunks": [ + { + "name": "HUNGARIAN", + "code": "hu", + "offset": 0, + "bytes": 246 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "ICELANDIC", + "code": "is", + "percent": 99, + "score": 1160 + } + ], + "chunks": [ + { + "name": "ICELANDIC", + "code": "is", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "ICELANDIC", + "code": "is", + "percent": 99, + "score": 1160 + } + ], + "chunks": [ + { + "name": "ICELANDIC", + "code": "is", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 855, + "languages": [ + { + "name": "INDONESIAN", + "code": "id", + "percent": 99, + "score": 1123 + } + ], + "chunks": [ + { + "name": "INDONESIAN", + "code": "id", + "offset": 0, + "bytes": 873 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 855, + "languages": [ + { + "name": "INDONESIAN", + "code": "id", + "percent": 99, + "score": 1123 + } + ], + "chunks": [ + { + "name": "INDONESIAN", + "code": "id", + "offset": 0, + "bytes": 873 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "IRISH", + "code": "ga", + "percent": 99, + "score": 1397 + } + ], + "chunks": [ + { + "name": "IRISH", + "code": "ga", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "IRISH", + "code": "ga", + "percent": 99, + "score": 1397 + } + ], + "chunks": [ + { + "name": "IRISH", + "code": "ga", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "ITALIAN", + "code": "it", + "percent": 99, + "score": 489 + } + ], + "chunks": [ + { + "name": "ITALIAN", + "code": "it", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "ITALIAN", + "code": "it", + "percent": 99, + "score": 489 + } + ], + "chunks": [ + { + "name": "ITALIAN", + "code": "it", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "JAVANESE", + "code": "jw", + "percent": 99, + "score": 636 + } + ], + "chunks": [ + { + "name": "JAVANESE", + "code": "jw", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "JAVANESE", + "code": "jw", + "percent": 99, + "score": 636 + } + ], + "chunks": [ + { + "name": "JAVANESE", + "code": "jw", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "KINYARWANDA", + "code": "rw", + "percent": 99, + "score": 809 + } + ], + "chunks": [ + { + "name": "KINYARWANDA", + "code": "rw", + "offset": 0, + "bytes": 98 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "KINYARWANDA", + "code": "rw", + "percent": 99, + "score": 809 + } + ], + "chunks": [ + { + "name": "KINYARWANDA", + "code": "rw", + "offset": 0, + "bytes": 98 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "LATVIAN", + "code": "lv", + "percent": 99, + "score": 1397 + } + ], + "chunks": [ + { + "name": "LATVIAN", + "code": "lv", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "LATVIAN", + "code": "lv", + "percent": 99, + "score": 1397 + } + ], + "chunks": [ + { + "name": "LATVIAN", + "code": "lv", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "LITHUANIAN", + "code": "lt", + "percent": 99, + "score": 1064 + } + ], + "chunks": [ + { + "name": "LITHUANIAN", + "code": "lt", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "LITHUANIAN", + "code": "lt", + "percent": 99, + "score": 1064 + } + ], + "chunks": [ + { + "name": "LITHUANIAN", + "code": "lt", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 248, + "languages": [ + { + "name": "MACEDONIAN", + "code": "mk", + "percent": 99, + "score": 1015 + } + ], + "chunks": [ + { + "name": "MACEDONIAN", + "code": "mk", + "offset": 0, + "bytes": 247 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 248, + "languages": [ + { + "name": "MACEDONIAN", + "code": "mk", + "percent": 99, + "score": 1015 + } + ], + "chunks": [ + { + "name": "MACEDONIAN", + "code": "mk", + "offset": 0, + "bytes": 247 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 867, + "languages": [ + { + "name": "MALAY", + "code": "ms", + "percent": 99, + "score": 1372 + } + ], + "chunks": [ + { + "name": "MALAY", + "code": "ms", + "offset": 0, + "bytes": 883 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 867, + "languages": [ + { + "name": "MALAY", + "code": "ms", + "percent": 99, + "score": 1372 + } + ], + "chunks": [ + { + "name": "MALAY", + "code": "ms", + "offset": 0, + "bytes": 883 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "MALTESE", + "code": "mt", + "percent": 99, + "score": 1073 + } + ], + "chunks": [ + { + "name": "MALTESE", + "code": "mt", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "MALTESE", + "code": "mt", + "percent": 99, + "score": 1073 + } + ], + "chunks": [ + { + "name": "MALTESE", + "code": "mt", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 1256, + "languages": [ + { + "name": "MARATHI", + "code": "mr", + "percent": 95, + "score": 656 + }, + { + "name": "TELUGU", + "code": "te", + "percent": 3, + "score": 1024 + } + ], + "chunks": [ + { + "name": "MARATHI", + "code": "mr", + "offset": 0, + "bytes": 113 + }, + { + "name": "TELUGU", + "code": "te", + "offset": 113, + "bytes": 30 + }, + { + "name": "MARATHI", + "code": "mr", + "offset": 143, + "bytes": 17 + }, + { + "name": "MARATHI", + "code": "mr", + "offset": 178, + "bytes": 1120 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 1256, + "languages": [ + { + "name": "MARATHI", + "code": "mr", + "percent": 95, + "score": 656 + }, + { + "name": "TELUGU", + "code": "te", + "percent": 3, + "score": 1024 + }, + { + "name": "URDU", + "code": "ur", + "percent": 1, + "score": 682 + } + ], + "chunks": [ + { + "name": "MARATHI", + "code": "mr", + "offset": 0, + "bytes": 113 + }, + { + "name": "TELUGU", + "code": "te", + "offset": 113, + "bytes": 30 + }, + { + "name": "MARATHI", + "code": "mr", + "offset": 143, + "bytes": 17 + }, + { + "name": "MARATHI", + "code": "mr", + "offset": 178, + "bytes": 1120 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 187, + "languages": [ + { + "name": "NEPALI", + "code": "ne", + "percent": 99, + "score": 374 + } + ], + "chunks": [ + { + "name": "NEPALI", + "code": "ne", + "offset": 0, + "bytes": 185 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 187, + "languages": [ + { + "name": "NEPALI", + "code": "ne", + "percent": 99, + "score": 374 + } + ], + "chunks": [ + { + "name": "NEPALI", + "code": "ne", + "offset": 0, + "bytes": 185 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "NORWEGIAN", + "code": "no", + "percent": 99, + "score": 552 + } + ], + "chunks": [ + { + "name": "NORWEGIAN", + "code": "no", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "NORWEGIAN", + "code": "no", + "percent": 99, + "score": 552 + } + ], + "chunks": [ + { + "name": "NORWEGIAN", + "code": "no", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "PERSIAN", + "code": "fa", + "percent": 99, + "score": 695 + } + ], + "chunks": [ + { + "name": "PERSIAN", + "code": "fa", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "PERSIAN", + "code": "fa", + "percent": 99, + "score": 695 + } + ], + "chunks": [ + { + "name": "PERSIAN", + "code": "fa", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "POLISH", + "code": "pl", + "percent": 99, + "score": 1784 + } + ], + "chunks": [ + { + "name": "POLISH", + "code": "pl", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "POLISH", + "code": "pl", + "percent": 99, + "score": 1784 + } + ], + "chunks": [ + { + "name": "POLISH", + "code": "pl", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "PORTUGUESE", + "code": "pt", + "percent": 99, + "score": 860 + } + ], + "chunks": [ + { + "name": "PORTUGUESE", + "code": "pt", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "PORTUGUESE", + "code": "pt", + "percent": 99, + "score": 860 + } + ], + "chunks": [ + { + "name": "PORTUGUESE", + "code": "pt", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "ROMANIAN", + "code": "ro", + "percent": 99, + "score": 1118 + } + ], + "chunks": [ + { + "name": "ROMANIAN", + "code": "ro", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "ROMANIAN", + "code": "ro", + "percent": 99, + "score": 1118 + } + ], + "chunks": [ + { + "name": "ROMANIAN", + "code": "ro", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "ROMANIAN", + "code": "ro", + "percent": 99, + "score": 949 + } + ], + "chunks": [ + { + "name": "ROMANIAN", + "code": "ro", + "offset": 0, + "bytes": 245 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "ROMANIAN", + "code": "ro", + "percent": 99, + "score": 949 + } + ], + "chunks": [ + { + "name": "ROMANIAN", + "code": "ro", + "offset": 0, + "bytes": 245 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 87, + "languages": [ + { + "name": "RUSSIAN", + "code": "ru", + "percent": 98, + "score": 404 + } + ], + "chunks": [ + { + "name": "RUSSIAN", + "code": "ru", + "offset": 0, + "bytes": 86 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 87, + "languages": [ + { + "name": "RUSSIAN", + "code": "ru", + "percent": 98, + "score": 404 + } + ], + "chunks": [ + { + "name": "RUSSIAN", + "code": "ru", + "offset": 0, + "bytes": 86 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "SCOTS_GAELIC", + "code": "gd", + "percent": 99, + "score": 1370 + } + ], + "chunks": [ + { + "name": "SCOTS_GAELIC", + "code": "gd", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "SCOTS_GAELIC", + "code": "gd", + "percent": 99, + "score": 1370 + } + ], + "chunks": [ + { + "name": "SCOTS_GAELIC", + "code": "gd", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "SERBIAN", + "code": "sr", + "percent": 99, + "score": 836 + } + ], + "chunks": [ + { + "name": "SERBIAN", + "code": "sr", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "SERBIAN", + "code": "sr", + "percent": 99, + "score": 836 + } + ], + "chunks": [ + { + "name": "SERBIAN", + "code": "sr", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 209, + "languages": [ + { + "name": "SERBIAN", + "code": "sr", + "percent": 99, + "score": 551 + } + ], + "chunks": [ + { + "name": "SERBIAN", + "code": "sr", + "offset": 0, + "bytes": 241 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 209, + "languages": [ + { + "name": "SERBIAN", + "code": "sr", + "percent": 99, + "score": 551 + } + ], + "chunks": [ + { + "name": "SERBIAN", + "code": "sr", + "offset": 0, + "bytes": 241 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "SLOVAK", + "code": "sk", + "percent": 99, + "score": 1282 + } + ], + "chunks": [ + { + "name": "SLOVAK", + "code": "sk", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "SLOVAK", + "code": "sk", + "percent": 99, + "score": 1282 + } + ], + "chunks": [ + { + "name": "SLOVAK", + "code": "sk", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "SLOVENIAN", + "code": "sl", + "percent": 99, + "score": 377 + } + ], + "chunks": [ + { + "name": "SLOVENIAN", + "code": "sl", + "offset": 130, + "bytes": 125 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "SLOVENIAN", + "code": "sl", + "percent": 99, + "score": 377 + } + ], + "chunks": [ + { + "name": "SLOVENIAN", + "code": "sl", + "offset": 130, + "bytes": 125 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "SPANISH", + "code": "es", + "percent": 99, + "score": 738 + } + ], + "chunks": [ + { + "name": "SPANISH", + "code": "es", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "SPANISH", + "code": "es", + "percent": 99, + "score": 738 + } + ], + "chunks": [ + { + "name": "SPANISH", + "code": "es", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "SWAHILI", + "code": "sw", + "percent": 99, + "score": 1158 + } + ], + "chunks": [ + { + "name": "SWAHILI", + "code": "sw", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "SWAHILI", + "code": "sw", + "percent": 99, + "score": 1158 + } + ], + "chunks": [ + { + "name": "SWAHILI", + "code": "sw", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "SWEDISH", + "code": "sv", + "percent": 99, + "score": 688 + } + ], + "chunks": [ + { + "name": "SWEDISH", + "code": "sv", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "SWEDISH", + "code": "sv", + "percent": 99, + "score": 688 + } + ], + "chunks": [ + { + "name": "SWEDISH", + "code": "sv", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "TAGALOG", + "code": "tl", + "percent": 99, + "score": 798 + } + ], + "chunks": [ + { + "name": "TAGALOG", + "code": "tl", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "TAGALOG", + "code": "tl", + "percent": 99, + "score": 798 + } + ], + "chunks": [ + { + "name": "TAGALOG", + "code": "tl", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "TURKISH", + "code": "tr", + "percent": 99, + "score": 1359 + } + ], + "chunks": [ + { + "name": "TURKISH", + "code": "tr", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "TURKISH", + "code": "tr", + "percent": 99, + "score": 1359 + } + ], + "chunks": [ + { + "name": "TURKISH", + "code": "tr", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 245, + "languages": [ + { + "name": "UKRAINIAN", + "code": "uk", + "percent": 99, + "score": 919 + } + ], + "chunks": [ + { + "name": "UKRAINIAN", + "code": "uk", + "offset": 0, + "bytes": 244 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 245, + "languages": [ + { + "name": "UKRAINIAN", + "code": "uk", + "percent": 99, + "score": 919 + } + ], + "chunks": [ + { + "name": "UKRAINIAN", + "code": "uk", + "offset": 0, + "bytes": 244 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "URDU", + "code": "ur", + "percent": 99, + "score": 1100 + } + ], + "chunks": [ + { + "name": "URDU", + "code": "ur", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "URDU", + "code": "ur", + "percent": 99, + "score": 1100 + } + ], + "chunks": [ + { + "name": "URDU", + "code": "ur", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "VIETNAMESE", + "code": "vi", + "percent": 99, + "score": 979 + } + ], + "chunks": [ + { + "name": "VIETNAMESE", + "code": "vi", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "VIETNAMESE", + "code": "vi", + "percent": 99, + "score": 979 + } + ], + "chunks": [ + { + "name": "VIETNAMESE", + "code": "vi", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "WELSH", + "code": "cy", + "percent": 99, + "score": 1569 + } + ], + "chunks": [ + { + "name": "WELSH", + "code": "cy", + "offset": 0, + "bytes": 248 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "WELSH", + "code": "cy", + "percent": 99, + "score": 1569 + } + ], + "chunks": [ + { + "name": "WELSH", + "code": "cy", + "offset": 0, + "bytes": 248 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 246, + "languages": [ + { + "name": "YIDDISH", + "code": "yi", + "percent": 99, + "score": 969 + } + ], + "chunks": [ + { + "name": "YIDDISH", + "code": "yi", + "offset": 0, + "bytes": 244 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 246, + "languages": [ + { + "name": "YIDDISH", + "code": "yi", + "percent": 99, + "score": 969 + } + ], + "chunks": [ + { + "name": "YIDDISH", + "code": "yi", + "offset": 0, + "bytes": 244 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "SOMALI", + "code": "so", + "percent": 99, + "score": 1474 + } + ], + "chunks": [ + { + "name": "SOMALI", + "code": "so", + "offset": 0, + "bytes": 241 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "SOMALI", + "code": "so", + "percent": 99, + "score": 1474 + } + ], + "chunks": [ + { + "name": "SOMALI", + "code": "so", + "offset": 0, + "bytes": 241 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 540, + "languages": [ + { + "name": "IGBO", + "code": "ig", + "percent": 99, + "score": 1259 + } + ], + "chunks": [ + { + "name": "IGBO", + "code": "ig", + "offset": 0, + "bytes": 560 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 540, + "languages": [ + { + "name": "IGBO", + "code": "ig", + "percent": 99, + "score": 1259 + } + ], + "chunks": [ + { + "name": "IGBO", + "code": "ig", + "offset": 0, + "bytes": 560 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "HAUSA", + "code": "ha", + "percent": 99, + "score": 1163 + } + ], + "chunks": [ + { + "name": "HAUSA", + "code": "ha", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "HAUSA", + "code": "ha", + "percent": 99, + "score": 1163 + } + ], + "chunks": [ + { + "name": "HAUSA", + "code": "ha", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "YORUBA", + "code": "yo", + "percent": 99, + "score": 1011 + } + ], + "chunks": [ + { + "name": "YORUBA", + "code": "yo", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "YORUBA", + "code": "yo", + "percent": 99, + "score": 1011 + } + ], + "chunks": [ + { + "name": "YORUBA", + "code": "yo", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "ZULU", + "code": "zu", + "percent": 99, + "score": 1185 + } + ], + "chunks": [ + { + "name": "ZULU", + "code": "zu", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "ZULU", + "code": "zu", + "percent": 99, + "score": 1185 + } + ], + "chunks": [ + { + "name": "ZULU", + "code": "zu", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 834, + "languages": [ + { + "name": "BOSNIAN", + "code": "bs", + "percent": 99, + "score": 685 + } + ], + "chunks": [ + { + "name": "BOSNIAN", + "code": "bs", + "offset": 0, + "bytes": 847 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 834, + "languages": [ + { + "name": "BOSNIAN", + "code": "bs", + "percent": 99, + "score": 685 + } + ], + "chunks": [ + { + "name": "BOSNIAN", + "code": "bs", + "offset": 0, + "bytes": 847 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 312, + "languages": [ + { + "name": "INDONESIAN", + "code": "id", + "percent": 99, + "score": 714 + } + ], + "chunks": [ + { + "name": "INDONESIAN", + "code": "id", + "offset": 84, + "bytes": 226 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 312, + "languages": [ + { + "name": "INDONESIAN", + "code": "id", + "percent": 99, + "score": 714 + } + ], + "chunks": [ + { + "name": "INDONESIAN", + "code": "id", + "offset": 84, + "bytes": 226 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 310, + "languages": [ + { + "name": "MALAY", + "code": "ms", + "percent": 99, + "score": 768 + } + ], + "chunks": [ + { + "name": "MALAY", + "code": "ms", + "offset": 0, + "bytes": 309 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 310, + "languages": [ + { + "name": "MALAY", + "code": "ms", + "percent": 99, + "score": 768 + } + ], + "chunks": [ + { + "name": "MALAY", + "code": "ms", + "offset": 0, + "bytes": 309 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 553, + "languages": [ + { + "name": "FRENCH", + "code": "fr", + "percent": 58, + "score": 883 + }, + { + "name": "ENGLISH", + "code": "en", + "percent": 41, + "score": 1148 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 94 + }, + { + "name": "FRENCH", + "code": "fr", + "offset": 94, + "bytes": 329 + }, + { + "name": "ENGLISH", + "code": "en", + "offset": 423, + "bytes": 138 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 553, + "languages": [ + { + "name": "FRENCH", + "code": "fr", + "percent": 58, + "score": 883 + }, + { + "name": "ENGLISH", + "code": "en", + "percent": 41, + "score": 1148 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 94 + }, + { + "name": "FRENCH", + "code": "fr", + "offset": 94, + "bytes": 329 + }, + { + "name": "ENGLISH", + "code": "en", + "offset": 423, + "bytes": 138 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 83, + "languages": [ + { + "name": "MONGOLIAN", + "code": "mn", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "MONGOLIAN", + "code": "mn", + "offset": 0, + "bytes": 87 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 83, + "languages": [ + { + "name": "MONGOLIAN", + "code": "mn", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "MONGOLIAN", + "code": "mn", + "offset": 0, + "bytes": 87 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 91, + "languages": [ + { + "name": "X_Buginese", + "code": "xx-Bugi", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "X_Buginese", + "code": "xx-Bugi", + "offset": 0, + "bytes": 89 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 91, + "languages": [ + { + "name": "X_Buginese", + "code": "xx-Bugi", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "X_Buginese", + "code": "xx-Bugi", + "offset": 0, + "bytes": 89 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 142, + "languages": [ + { + "name": "X_Gothic", + "code": "xx-Goth", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "X_Gothic", + "code": "xx-Goth", + "offset": 0, + "bytes": 140 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 142, + "languages": [ + { + "name": "X_Gothic", + "code": "xx-Goth", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "X_Gothic", + "code": "xx-Goth", + "offset": 0, + "bytes": 140 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "ABKHAZIAN", + "code": "ab", + "percent": 99, + "score": 893 + } + ], + "chunks": [ + { + "name": "ABKHAZIAN", + "code": "ab", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "ABKHAZIAN", + "code": "ab", + "percent": 99, + "score": 893 + } + ], + "chunks": [ + { + "name": "ABKHAZIAN", + "code": "ab", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "AFAR", + "code": "aa", + "percent": 99, + "score": 837 + } + ], + "chunks": [ + { + "name": "AFAR", + "code": "aa", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "AFAR", + "code": "aa", + "percent": 99, + "score": 837 + } + ], + "chunks": [ + { + "name": "AFAR", + "code": "aa", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 365, + "languages": [ + { + "name": "AKAN", + "code": "ak", + "percent": 99, + "score": 565 + } + ], + "chunks": [ + { + "name": "AKAN", + "code": "ak", + "offset": 0, + "bytes": 204 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 365, + "languages": [ + { + "name": "AKAN", + "code": "ak", + "percent": 99, + "score": 565 + } + ], + "chunks": [ + { + "name": "AKAN", + "code": "ak", + "offset": 0, + "bytes": 204 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "AMHARIC", + "code": "am", + "percent": 99, + "score": 300 + } + ], + "chunks": [ + { + "name": "AMHARIC", + "code": "am", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "AMHARIC", + "code": "am", + "percent": 99, + "score": 300 + } + ], + "chunks": [ + { + "name": "AMHARIC", + "code": "am", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 258, + "languages": [ + { + "name": "ASSAMESE", + "code": "as", + "percent": 99, + "score": 482 + } + ], + "chunks": [ + { + "name": "ASSAMESE", + "code": "as", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 258, + "languages": [ + { + "name": "ASSAMESE", + "code": "as", + "percent": 99, + "score": 482 + } + ], + "chunks": [ + { + "name": "ASSAMESE", + "code": "as", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "AYMARA", + "code": "ay", + "percent": 99, + "score": 495 + } + ], + "chunks": [] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "AYMARA", + "code": "ay", + "percent": 99, + "score": 495 + } + ], + "chunks": [] + } + }, + { + "default": { + "reliable": true, + "textBytes": 243, + "languages": [ + { + "name": "BASHKIR", + "code": "ba", + "percent": 99, + "score": 380 + } + ], + "chunks": [] + }, + "bestEffort": { + "reliable": true, + "textBytes": 243, + "languages": [ + { + "name": "BASHKIR", + "code": "ba", + "percent": 99, + "score": 380 + } + ], + "chunks": [] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "BISLAMA", + "code": "bi", + "percent": 99, + "score": 1604 + } + ], + "chunks": [ + { + "name": "BISLAMA", + "code": "bi", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "BISLAMA", + "code": "bi", + "percent": 99, + "score": 1604 + } + ], + "chunks": [ + { + "name": "BISLAMA", + "code": "bi", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "BRETON", + "code": "br", + "percent": 99, + "score": 975 + } + ], + "chunks": [ + { + "name": "BRETON", + "code": "br", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "BRETON", + "code": "br", + "percent": 99, + "score": 975 + } + ], + "chunks": [ + { + "name": "BRETON", + "code": "br", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "BURMESE", + "code": "my", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "BURMESE", + "code": "my", + "offset": 0, + "bytes": 241 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "BURMESE", + "code": "my", + "percent": 100, + "score": 1024 + } + ], + "chunks": [ + { + "name": "BURMESE", + "code": "my", + "offset": 0, + "bytes": 241 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "CORSICAN", + "code": "co", + "percent": 99, + "score": 706 + } + ], + "chunks": [ + { + "name": "CORSICAN", + "code": "co", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "CORSICAN", + "code": "co", + "percent": 99, + "score": 706 + } + ], + "chunks": [ + { + "name": "CORSICAN", + "code": "co", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 258, + "languages": [ + { + "name": "DZONGKHA", + "code": "dz", + "percent": 99, + "score": 725 + } + ], + "chunks": [ + { + "name": "DZONGKHA", + "code": "dz", + "offset": 0, + "bytes": 257 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 258, + "languages": [ + { + "name": "DZONGKHA", + "code": "dz", + "percent": 99, + "score": 725 + } + ], + "chunks": [ + { + "name": "DZONGKHA", + "code": "dz", + "offset": 0, + "bytes": 257 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "ESPERANTO", + "code": "eo", + "percent": 99, + "score": 784 + } + ], + "chunks": [ + { + "name": "ESPERANTO", + "code": "eo", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "ESPERANTO", + "code": "eo", + "percent": 99, + "score": 784 + } + ], + "chunks": [ + { + "name": "ESPERANTO", + "code": "eo", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "FAROESE", + "code": "fo", + "percent": 99, + "score": 932 + } + ], + "chunks": [ + { + "name": "FAROESE", + "code": "fo", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "FAROESE", + "code": "fo", + "percent": 99, + "score": 932 + } + ], + "chunks": [ + { + "name": "FAROESE", + "code": "fo", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "FIJIAN", + "code": "fj", + "percent": 99, + "score": 991 + } + ], + "chunks": [ + { + "name": "FIJIAN", + "code": "fj", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "FIJIAN", + "code": "fj", + "percent": 99, + "score": 991 + } + ], + "chunks": [ + { + "name": "FIJIAN", + "code": "fj", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "FRISIAN", + "code": "fy", + "percent": 99, + "score": 1019 + } + ], + "chunks": [ + { + "name": "FRISIAN", + "code": "fy", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "FRISIAN", + "code": "fy", + "percent": 99, + "score": 1019 + } + ], + "chunks": [ + { + "name": "FRISIAN", + "code": "fy", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "GREENLANDIC", + "code": "kl", + "percent": 99, + "score": 1777 + } + ], + "chunks": [ + { + "name": "GREENLANDIC", + "code": "kl", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "GREENLANDIC", + "code": "kl", + "percent": 99, + "score": 1777 + } + ], + "chunks": [ + { + "name": "GREENLANDIC", + "code": "kl", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "GUARANI", + "code": "gn", + "percent": 99, + "score": 1191 + } + ], + "chunks": [ + { + "name": "GUARANI", + "code": "gn", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "GUARANI", + "code": "gn", + "percent": 99, + "score": 1191 + } + ], + "chunks": [ + { + "name": "GUARANI", + "code": "gn", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 458, + "languages": [ + { + "name": "HAWAIIAN", + "code": "haw", + "percent": 99, + "score": 956 + } + ], + "chunks": [ + { + "name": "HAWAIIAN", + "code": "haw", + "offset": 0, + "bytes": 510 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 458, + "languages": [ + { + "name": "HAWAIIAN", + "code": "haw", + "percent": 99, + "score": 956 + } + ], + "chunks": [ + { + "name": "HAWAIIAN", + "code": "haw", + "offset": 0, + "bytes": 510 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 540, + "languages": [ + { + "name": "IGBO", + "code": "ig", + "percent": 99, + "score": 1259 + } + ], + "chunks": [ + { + "name": "IGBO", + "code": "ig", + "offset": 0, + "bytes": 560 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 540, + "languages": [ + { + "name": "IGBO", + "code": "ig", + "percent": 99, + "score": 1259 + } + ], + "chunks": [ + { + "name": "IGBO", + "code": "ig", + "offset": 0, + "bytes": 560 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 224, + "languages": [ + { + "name": "INTERLINGUA", + "code": "ia", + "percent": 99, + "score": 537 + } + ], + "chunks": [ + { + "name": "INTERLINGUA", + "code": "ia", + "offset": 0, + "bytes": 223 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 224, + "languages": [ + { + "name": "INTERLINGUA", + "code": "ia", + "percent": 99, + "score": 537 + } + ], + "chunks": [ + { + "name": "INTERLINGUA", + "code": "ia", + "offset": 0, + "bytes": 223 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "INTERLINGUE", + "code": "ie", + "percent": 99, + "score": 444 + } + ], + "chunks": [] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "INTERLINGUE", + "code": "ie", + "percent": 99, + "score": 444 + } + ], + "chunks": [] + } + }, + { + "default": { + "reliable": true, + "textBytes": 310, + "languages": [ + { + "name": "INUPIAK", + "code": "ik", + "percent": 99, + "score": 507 + } + ], + "chunks": [ + { + "name": "INUPIAK", + "code": "ik", + "offset": 0, + "bytes": 103 + }, + { + "name": "INUPIAK", + "code": "ik", + "offset": 222, + "bytes": 92 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 310, + "languages": [ + { + "name": "INUPIAK", + "code": "ik", + "percent": 99, + "score": 507 + } + ], + "chunks": [ + { + "name": "INUPIAK", + "code": "ik", + "offset": 0, + "bytes": 103 + }, + { + "name": "INUPIAK", + "code": "ik", + "offset": 222, + "bytes": 92 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 1990, + "languages": [ + { + "name": "KASHMIRI", + "code": "ks", + "percent": 99, + "score": 878 + } + ], + "chunks": [ + { + "name": "KASHMIRI", + "code": "ks", + "offset": 0, + "bytes": 2036 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 1990, + "languages": [ + { + "name": "KASHMIRI", + "code": "ks", + "percent": 99, + "score": 878 + } + ], + "chunks": [ + { + "name": "KASHMIRI", + "code": "ks", + "offset": 0, + "bytes": 2036 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "KAZAKH", + "code": "kk", + "percent": 99, + "score": 372 + } + ], + "chunks": [ + { + "name": "KAZAKH", + "code": "kk", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "KAZAKH", + "code": "kk", + "percent": 99, + "score": 372 + } + ], + "chunks": [ + { + "name": "KAZAKH", + "code": "kk", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "KAZAKH", + "code": "kk", + "percent": 99, + "score": 950 + } + ], + "chunks": [ + { + "name": "KAZAKH", + "code": "kk", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "KAZAKH", + "code": "kk", + "percent": 99, + "score": 950 + } + ], + "chunks": [ + { + "name": "KAZAKH", + "code": "kk", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "KHASI", + "code": "kha", + "percent": 99, + "score": 1104 + } + ], + "chunks": [ + { + "name": "KHASI", + "code": "kha", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "KHASI", + "code": "kha", + "percent": 99, + "score": 1104 + } + ], + "chunks": [ + { + "name": "KHASI", + "code": "kha", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 210, + "languages": [ + { + "name": "KURDISH", + "code": "ku", + "percent": 99, + "score": 975 + } + ], + "chunks": [ + { + "name": "KURDISH", + "code": "ku", + "offset": 0, + "bytes": 209 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 210, + "languages": [ + { + "name": "KURDISH", + "code": "ku", + "percent": 99, + "score": 975 + } + ], + "chunks": [ + { + "name": "KURDISH", + "code": "ku", + "offset": 0, + "bytes": 209 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "KYRGYZ", + "code": "ky", + "percent": 99, + "score": 756 + } + ], + "chunks": [ + { + "name": "KYRGYZ", + "code": "ky", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "KYRGYZ", + "code": "ky", + "percent": 99, + "score": 756 + } + ], + "chunks": [ + { + "name": "KYRGYZ", + "code": "ky", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "KYRGYZ", + "code": "ky", + "percent": 99, + "score": 882 + } + ], + "chunks": [ + { + "name": "KYRGYZ", + "code": "ky", + "offset": 0, + "bytes": 246 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 247, + "languages": [ + { + "name": "KYRGYZ", + "code": "ky", + "percent": 99, + "score": 882 + } + ], + "chunks": [ + { + "name": "KYRGYZ", + "code": "ky", + "offset": 0, + "bytes": 246 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "LATIN", + "code": "la", + "percent": 99, + "score": 996 + } + ], + "chunks": [ + { + "name": "LATIN", + "code": "la", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "LATIN", + "code": "la", + "percent": 99, + "score": 996 + } + ], + "chunks": [ + { + "name": "LATIN", + "code": "la", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "LINGALA", + "code": "ln", + "percent": 99, + "score": 1258 + } + ], + "chunks": [ + { + "name": "LINGALA", + "code": "ln", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "LINGALA", + "code": "ln", + "percent": 99, + "score": 1258 + } + ], + "chunks": [ + { + "name": "LINGALA", + "code": "ln", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "LUXEMBOURGISH", + "code": "lb", + "percent": 99, + "score": 1007 + } + ], + "chunks": [ + { + "name": "LUXEMBOURGISH", + "code": "lb", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "LUXEMBOURGISH", + "code": "lb", + "percent": 99, + "score": 1007 + } + ], + "chunks": [ + { + "name": "LUXEMBOURGISH", + "code": "lb", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "MALAGASY", + "code": "mg", + "percent": 99, + "score": 1339 + } + ], + "chunks": [ + { + "name": "MALAGASY", + "code": "mg", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "MALAGASY", + "code": "mg", + "percent": 99, + "score": 1339 + } + ], + "chunks": [ + { + "name": "MALAGASY", + "code": "mg", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "MALAY", + "code": "ms", + "percent": 99, + "score": 1342 + } + ], + "chunks": [ + { + "name": "MALAY", + "code": "ms", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "MALAY", + "code": "ms", + "percent": 99, + "score": 1342 + } + ], + "chunks": [ + { + "name": "MALAY", + "code": "ms", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "MANX", + "code": "gv", + "percent": 65, + "score": 1311 + }, + { + "name": "ENGLISH", + "code": "en", + "percent": 34, + "score": 870 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 88 + }, + { + "name": "MANX", + "code": "gv", + "offset": 88, + "bytes": 163 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "MANX", + "code": "gv", + "percent": 65, + "score": 1311 + }, + { + "name": "ENGLISH", + "code": "en", + "percent": 34, + "score": 870 + } + ], + "chunks": [ + { + "name": "ENGLISH", + "code": "en", + "offset": 0, + "bytes": 88 + }, + { + "name": "MANX", + "code": "gv", + "offset": 88, + "bytes": 163 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "MAORI", + "code": "mi", + "percent": 99, + "score": 992 + } + ], + "chunks": [ + { + "name": "MAORI", + "code": "mi", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "MAORI", + "code": "mi", + "percent": 99, + "score": 992 + } + ], + "chunks": [ + { + "name": "MAORI", + "code": "mi", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 453, + "languages": [ + { + "name": "MAURITIAN_CREOLE", + "code": "mfe", + "percent": 99, + "score": 926 + } + ], + "chunks": [ + { + "name": "MAURITIAN_CREOLE", + "code": "mfe", + "offset": 0, + "bytes": 464 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 453, + "languages": [ + { + "name": "MAURITIAN_CREOLE", + "code": "mfe", + "percent": 99, + "score": 926 + } + ], + "chunks": [ + { + "name": "MAURITIAN_CREOLE", + "code": "mfe", + "offset": 0, + "bytes": 464 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "MONGOLIAN", + "code": "mn", + "percent": 99, + "score": 1329 + } + ], + "chunks": [ + { + "name": "MONGOLIAN", + "code": "mn", + "offset": 0, + "bytes": 241 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 242, + "languages": [ + { + "name": "MONGOLIAN", + "code": "mn", + "percent": 99, + "score": 1329 + } + ], + "chunks": [ + { + "name": "MONGOLIAN", + "code": "mn", + "offset": 0, + "bytes": 241 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "NAURU", + "code": "na", + "percent": 99, + "score": 1056 + } + ], + "chunks": [ + { + "name": "NAURU", + "code": "na", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "NAURU", + "code": "na", + "percent": 99, + "score": 1056 + } + ], + "chunks": [ + { + "name": "NAURU", + "code": "na", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 170, + "languages": [ + { + "name": "NDEBELE", + "code": "nr", + "percent": 99, + "score": 757 + } + ], + "chunks": [ + { + "name": "NDEBELE", + "code": "nr", + "offset": 0, + "bytes": 173 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 170, + "languages": [ + { + "name": "NDEBELE", + "code": "nr", + "percent": 99, + "score": 757 + } + ], + "chunks": [ + { + "name": "NDEBELE", + "code": "nr", + "offset": 0, + "bytes": 173 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "NORWEGIAN_N", + "code": "nn", + "percent": 99, + "score": 1048 + } + ], + "chunks": [ + { + "name": "NORWEGIAN_N", + "code": "nn", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "NORWEGIAN_N", + "code": "nn", + "percent": 99, + "score": 1048 + } + ], + "chunks": [ + { + "name": "NORWEGIAN_N", + "code": "nn", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 163, + "languages": [ + { + "name": "NYANJA", + "code": "ny", + "percent": 99, + "score": 973 + } + ], + "chunks": [ + { + "name": "NYANJA", + "code": "ny", + "offset": 0, + "bytes": 164 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 163, + "languages": [ + { + "name": "NYANJA", + "code": "ny", + "percent": 99, + "score": 973 + } + ], + "chunks": [ + { + "name": "NYANJA", + "code": "ny", + "offset": 0, + "bytes": 164 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 166, + "languages": [ + { + "name": "OCCITAN", + "code": "oc", + "percent": 99, + "score": 875 + } + ], + "chunks": [ + { + "name": "OCCITAN", + "code": "oc", + "offset": 0, + "bytes": 168 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 166, + "languages": [ + { + "name": "OCCITAN", + "code": "oc", + "percent": 99, + "score": 875 + } + ], + "chunks": [ + { + "name": "OCCITAN", + "code": "oc", + "offset": 0, + "bytes": 168 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "OROMO", + "code": "om", + "percent": 99, + "score": 1108 + } + ], + "chunks": [ + { + "name": "OROMO", + "code": "om", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "OROMO", + "code": "om", + "percent": 99, + "score": 1108 + } + ], + "chunks": [ + { + "name": "OROMO", + "code": "om", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "PASHTO", + "code": "ps", + "percent": 99, + "score": 893 + } + ], + "chunks": [ + { + "name": "PASHTO", + "code": "ps", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "PASHTO", + "code": "ps", + "percent": 99, + "score": 893 + } + ], + "chunks": [ + { + "name": "PASHTO", + "code": "ps", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 353, + "languages": [ + { + "name": "PEDI", + "code": "nso", + "percent": 99, + "score": 753 + } + ], + "chunks": [ + { + "name": "PEDI", + "code": "nso", + "offset": 0, + "bytes": 377 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 353, + "languages": [ + { + "name": "PEDI", + "code": "nso", + "percent": 99, + "score": 753 + } + ], + "chunks": [ + { + "name": "PEDI", + "code": "nso", + "offset": 0, + "bytes": 377 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "QUECHUA", + "code": "qu", + "percent": 99, + "score": 1590 + } + ], + "chunks": [ + { + "name": "QUECHUA", + "code": "qu", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "QUECHUA", + "code": "qu", + "percent": 99, + "score": 1590 + } + ], + "chunks": [ + { + "name": "QUECHUA", + "code": "qu", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 407, + "languages": [ + { + "name": "RHAETO_ROMANCE", + "code": "rm", + "percent": 99, + "score": 807 + } + ], + "chunks": [ + { + "name": "RHAETO_ROMANCE", + "code": "rm", + "offset": 0, + "bytes": 449 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 407, + "languages": [ + { + "name": "RHAETO_ROMANCE", + "code": "rm", + "percent": 99, + "score": 807 + } + ], + "chunks": [ + { + "name": "RHAETO_ROMANCE", + "code": "rm", + "offset": 0, + "bytes": 449 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "RUNDI", + "code": "rn", + "percent": 99, + "score": 1089 + } + ], + "chunks": [ + { + "name": "RUNDI", + "code": "rn", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "RUNDI", + "code": "rn", + "percent": 99, + "score": 1089 + } + ], + "chunks": [ + { + "name": "RUNDI", + "code": "rn", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SAMOAN", + "code": "sm", + "percent": 99, + "score": 988 + } + ], + "chunks": [ + { + "name": "SAMOAN", + "code": "sm", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SAMOAN", + "code": "sm", + "percent": 99, + "score": 988 + } + ], + "chunks": [ + { + "name": "SAMOAN", + "code": "sm", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 248, + "languages": [ + { + "name": "SANGO", + "code": "sg", + "percent": 99, + "score": 1600 + } + ], + "chunks": [ + { + "name": "SANGO", + "code": "sg", + "offset": 0, + "bytes": 247 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 248, + "languages": [ + { + "name": "SANGO", + "code": "sg", + "percent": 99, + "score": 1600 + } + ], + "chunks": [ + { + "name": "SANGO", + "code": "sg", + "offset": 0, + "bytes": 247 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 246, + "languages": [ + { + "name": "SANSKRIT", + "code": "sa", + "percent": 99, + "score": 188 + } + ], + "chunks": [] + }, + "bestEffort": { + "reliable": true, + "textBytes": 246, + "languages": [ + { + "name": "SANSKRIT", + "code": "sa", + "percent": 99, + "score": 188 + } + ], + "chunks": [] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SANSKRIT", + "code": "sa", + "percent": 99, + "score": 932 + } + ], + "chunks": [ + { + "name": "SANSKRIT", + "code": "sa", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SANSKRIT", + "code": "sa", + "percent": 99, + "score": 932 + } + ], + "chunks": [ + { + "name": "SANSKRIT", + "code": "sa", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SCOTS", + "code": "sco", + "percent": 90, + "score": 610 + }, + { + "name": "ENGLISH", + "code": "en", + "percent": 9, + "score": 2893 + } + ], + "chunks": [ + { + "name": "SCOTS", + "code": "sco", + "offset": 0, + "bytes": 36 + }, + { + "name": "ENGLISH", + "code": "en", + "offset": 36, + "bytes": 26 + }, + { + "name": "SCOTS", + "code": "sco", + "offset": 62, + "bytes": 194 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SCOTS", + "code": "sco", + "percent": 90, + "score": 610 + }, + { + "name": "ENGLISH", + "code": "en", + "percent": 9, + "score": 2893 + } + ], + "chunks": [ + { + "name": "SCOTS", + "code": "sco", + "offset": 0, + "bytes": 36 + }, + { + "name": "ENGLISH", + "code": "en", + "offset": 36, + "bytes": 26 + }, + { + "name": "SCOTS", + "code": "sco", + "offset": 62, + "bytes": 194 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 268, + "languages": [ + { + "name": "SESELWA", + "code": "crs", + "percent": 99, + "score": 1150 + } + ], + "chunks": [ + { + "name": "SESELWA", + "code": "crs", + "offset": 0, + "bytes": 279 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 268, + "languages": [ + { + "name": "SESELWA", + "code": "crs", + "percent": 99, + "score": 1150 + } + ], + "chunks": [ + { + "name": "SESELWA", + "code": "crs", + "offset": 0, + "bytes": 279 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "SESOTHO", + "code": "st", + "percent": 99, + "score": 916 + } + ], + "chunks": [ + { + "name": "SESOTHO", + "code": "st", + "offset": 0, + "bytes": 248 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "SESOTHO", + "code": "st", + "percent": 99, + "score": 916 + } + ], + "chunks": [ + { + "name": "SESOTHO", + "code": "st", + "offset": 0, + "bytes": 248 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "SHONA", + "code": "sn", + "percent": 99, + "score": 1092 + } + ], + "chunks": [ + { + "name": "SHONA", + "code": "sn", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "SHONA", + "code": "sn", + "percent": 99, + "score": 1092 + } + ], + "chunks": [ + { + "name": "SHONA", + "code": "sn", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SINDHI", + "code": "sd", + "percent": 99, + "score": 708 + } + ], + "chunks": [ + { + "name": "SINDHI", + "code": "sd", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "SINDHI", + "code": "sd", + "percent": 99, + "score": 708 + } + ], + "chunks": [ + { + "name": "SINDHI", + "code": "sd", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "SISWANT", + "code": "ss", + "percent": 99, + "score": 1188 + } + ], + "chunks": [ + { + "name": "SISWANT", + "code": "ss", + "offset": 0, + "bytes": 249 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 250, + "languages": [ + { + "name": "SISWANT", + "code": "ss", + "percent": 99, + "score": 1188 + } + ], + "chunks": [ + { + "name": "SISWANT", + "code": "ss", + "offset": 0, + "bytes": 249 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 329, + "languages": [ + { + "name": "SUNDANESE", + "code": "su", + "percent": 99, + "score": 789 + } + ], + "chunks": [ + { + "name": "SUNDANESE", + "code": "su", + "offset": 0, + "bytes": 333 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 329, + "languages": [ + { + "name": "SUNDANESE", + "code": "su", + "percent": 99, + "score": 789 + } + ], + "chunks": [ + { + "name": "SUNDANESE", + "code": "su", + "offset": 0, + "bytes": 333 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 245, + "languages": [ + { + "name": "TAJIK", + "code": "tg", + "percent": 99, + "score": 1019 + } + ], + "chunks": [ + { + "name": "TAJIK", + "code": "tg", + "offset": 0, + "bytes": 244 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 245, + "languages": [ + { + "name": "TAJIK", + "code": "tg", + "percent": 99, + "score": 1019 + } + ], + "chunks": [ + { + "name": "TAJIK", + "code": "tg", + "offset": 0, + "bytes": 244 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 133, + "languages": [ + { + "name": "TATAR", + "code": "tt", + "percent": 99, + "score": 768 + } + ], + "chunks": [ + { + "name": "TATAR", + "code": "tt", + "offset": 0, + "bytes": 131 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 133, + "languages": [ + { + "name": "TATAR", + "code": "tt", + "percent": 99, + "score": 768 + } + ], + "chunks": [ + { + "name": "TATAR", + "code": "tt", + "offset": 0, + "bytes": 131 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "TATAR", + "code": "tt", + "percent": 99, + "score": 1450 + } + ], + "chunks": [ + { + "name": "TATAR", + "code": "tt", + "offset": 0, + "bytes": 252 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 253, + "languages": [ + { + "name": "TATAR", + "code": "tt", + "percent": 99, + "score": 1450 + } + ], + "chunks": [ + { + "name": "TATAR", + "code": "tt", + "offset": 0, + "bytes": 252 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 1961, + "languages": [ + { + "name": "TIBETAN", + "code": "bo", + "percent": 99, + "score": 695 + } + ], + "chunks": [ + { + "name": "TIBETAN", + "code": "bo", + "offset": 0, + "bytes": 2384 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 1961, + "languages": [ + { + "name": "TIBETAN", + "code": "bo", + "percent": 99, + "score": 695 + } + ], + "chunks": [ + { + "name": "TIBETAN", + "code": "bo", + "offset": 0, + "bytes": 2384 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "TIGRINYA", + "code": "ti", + "percent": 99, + "score": 478 + } + ], + "chunks": [ + { + "name": "TIGRINYA", + "code": "ti", + "offset": 0, + "bytes": 248 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 249, + "languages": [ + { + "name": "TIGRINYA", + "code": "ti", + "percent": 99, + "score": 478 + } + ], + "chunks": [ + { + "name": "TIGRINYA", + "code": "ti", + "offset": 0, + "bytes": 248 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "TONGA", + "code": "to", + "percent": 99, + "score": 1142 + } + ], + "chunks": [ + { + "name": "TONGA", + "code": "to", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "TONGA", + "code": "to", + "percent": 99, + "score": 1142 + } + ], + "chunks": [ + { + "name": "TONGA", + "code": "to", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "TSONGA", + "code": "ts", + "percent": 99, + "score": 1180 + } + ], + "chunks": [ + { + "name": "TSONGA", + "code": "ts", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "TSONGA", + "code": "ts", + "percent": 99, + "score": 1180 + } + ], + "chunks": [ + { + "name": "TSONGA", + "code": "ts", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "TSWANA", + "code": "tn", + "percent": 99, + "score": 1068 + } + ], + "chunks": [ + { + "name": "TSWANA", + "code": "tn", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "TSWANA", + "code": "tn", + "percent": 99, + "score": 1068 + } + ], + "chunks": [ + { + "name": "TSWANA", + "code": "tn", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "TURKMEN", + "code": "tk", + "percent": 99, + "score": 636 + } + ], + "chunks": [ + { + "name": "TURKMEN", + "code": "tk", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "TURKMEN", + "code": "tk", + "percent": 99, + "score": 636 + } + ], + "chunks": [ + { + "name": "TURKMEN", + "code": "tk", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "TURKMEN", + "code": "tk", + "percent": 99, + "score": 1203 + } + ], + "chunks": [ + { + "name": "TURKMEN", + "code": "tk", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "TURKMEN", + "code": "tk", + "percent": 99, + "score": 1203 + } + ], + "chunks": [ + { + "name": "TURKMEN", + "code": "tk", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 248, + "languages": [ + { + "name": "AKAN", + "code": "ak", + "percent": 99, + "score": 1488 + } + ], + "chunks": [ + { + "name": "AKAN", + "code": "ak", + "offset": 0, + "bytes": 247 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 248, + "languages": [ + { + "name": "AKAN", + "code": "ak", + "percent": 99, + "score": 1488 + } + ], + "chunks": [ + { + "name": "AKAN", + "code": "ak", + "offset": 0, + "bytes": 247 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "UIGHUR", + "code": "ug", + "percent": 99, + "score": 1298 + } + ], + "chunks": [ + { + "name": "UIGHUR", + "code": "ug", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "UIGHUR", + "code": "ug", + "percent": 99, + "score": 1298 + } + ], + "chunks": [ + { + "name": "UIGHUR", + "code": "ug", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 246, + "languages": [ + { + "name": "UIGHUR", + "code": "ug", + "percent": 99, + "score": 1337 + } + ], + "chunks": [ + { + "name": "UIGHUR", + "code": "ug", + "offset": 0, + "bytes": 245 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 246, + "languages": [ + { + "name": "UIGHUR", + "code": "ug", + "percent": 99, + "score": 1337 + } + ], + "chunks": [ + { + "name": "UIGHUR", + "code": "ug", + "offset": 0, + "bytes": 245 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "UZBEK", + "code": "uz", + "percent": 99, + "score": 837 + } + ], + "chunks": [ + { + "name": "UZBEK", + "code": "uz", + "offset": 0, + "bytes": 253 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 254, + "languages": [ + { + "name": "UZBEK", + "code": "uz", + "percent": 99, + "score": 837 + } + ], + "chunks": [ + { + "name": "UZBEK", + "code": "uz", + "offset": 0, + "bytes": 253 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "UZBEK", + "code": "uz", + "percent": 99, + "score": 905 + } + ], + "chunks": [ + { + "name": "UZBEK", + "code": "uz", + "offset": 0, + "bytes": 250 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 251, + "languages": [ + { + "name": "UZBEK", + "code": "uz", + "percent": 99, + "score": 905 + } + ], + "chunks": [ + { + "name": "UZBEK", + "code": "uz", + "offset": 0, + "bytes": 250 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "UZBEK", + "code": "uz", + "percent": 99, + "score": 1460 + } + ], + "chunks": [ + { + "name": "UZBEK", + "code": "uz", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "UZBEK", + "code": "uz", + "percent": 99, + "score": 1460 + } + ], + "chunks": [ + { + "name": "UZBEK", + "code": "uz", + "offset": 0, + "bytes": 251 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 362, + "languages": [ + { + "name": "VENDA", + "code": "ve", + "percent": 99, + "score": 1375 + } + ], + "chunks": [ + { + "name": "VENDA", + "code": "ve", + "offset": 0, + "bytes": 365 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 362, + "languages": [ + { + "name": "VENDA", + "code": "ve", + "percent": 99, + "score": 1375 + } + ], + "chunks": [ + { + "name": "VENDA", + "code": "ve", + "offset": 0, + "bytes": 365 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "VOLAPUK", + "code": "vo", + "percent": 99, + "score": 1548 + } + ], + "chunks": [ + { + "name": "VOLAPUK", + "code": "vo", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "VOLAPUK", + "code": "vo", + "percent": 99, + "score": 1548 + } + ], + "chunks": [ + { + "name": "VOLAPUK", + "code": "vo", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 655, + "languages": [ + { + "name": "WARAY_PHILIPPINES", + "code": "war", + "percent": 99, + "score": 677 + } + ], + "chunks": [ + { + "name": "WARAY_PHILIPPINES", + "code": "war", + "offset": 0, + "bytes": 520 + }, + { + "name": "WARAY_PHILIPPINES", + "code": "war", + "offset": 608, + "bytes": 98 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 655, + "languages": [ + { + "name": "WARAY_PHILIPPINES", + "code": "war", + "percent": 99, + "score": 677 + } + ], + "chunks": [ + { + "name": "WARAY_PHILIPPINES", + "code": "war", + "offset": 0, + "bytes": 520 + }, + { + "name": "WARAY_PHILIPPINES", + "code": "war", + "offset": 608, + "bytes": 98 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "WOLOF", + "code": "wo", + "percent": 99, + "score": 1261 + } + ], + "chunks": [ + { + "name": "WOLOF", + "code": "wo", + "offset": 0, + "bytes": 254 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 255, + "languages": [ + { + "name": "WOLOF", + "code": "wo", + "percent": 99, + "score": 1261 + } + ], + "chunks": [ + { + "name": "WOLOF", + "code": "wo", + "offset": 0, + "bytes": 254 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "XHOSA", + "code": "xh", + "percent": 99, + "score": 1144 + } + ], + "chunks": [ + { + "name": "XHOSA", + "code": "xh", + "offset": 0, + "bytes": 255 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 256, + "languages": [ + { + "name": "XHOSA", + "code": "xh", + "percent": 99, + "score": 1144 + } + ], + "chunks": [ + { + "name": "XHOSA", + "code": "xh", + "offset": 0, + "bytes": 255 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "X_KLINGON", + "code": "tlh", + "percent": 99, + "score": 1536 + } + ], + "chunks": [ + { + "name": "X_KLINGON", + "code": "tlh", + "offset": 0, + "bytes": 256 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 257, + "languages": [ + { + "name": "X_KLINGON", + "code": "tlh", + "percent": 99, + "score": 1536 + } + ], + "chunks": [ + { + "name": "X_KLINGON", + "code": "tlh", + "offset": 0, + "bytes": 256 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 191, + "languages": [ + { + "name": "X_PIG_LATIN", + "code": "zzp", + "percent": 99, + "score": 1654 + } + ], + "chunks": [ + { + "name": "X_PIG_LATIN", + "code": "zzp", + "offset": 0, + "bytes": 190 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 191, + "languages": [ + { + "name": "X_PIG_LATIN", + "code": "zzp", + "percent": 99, + "score": 1654 + } + ], + "chunks": [ + { + "name": "X_PIG_LATIN", + "code": "zzp", + "offset": 0, + "bytes": 190 + } + ] + } + }, + { + "default": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "ZHUANG", + "code": "za", + "percent": 99, + "score": 2137 + } + ], + "chunks": [ + { + "name": "ZHUANG", + "code": "za", + "offset": 0, + "bytes": 251 + } + ] + }, + "bestEffort": { + "reliable": true, + "textBytes": 252, + "languages": [ + { + "name": "ZHUANG", + "code": "za", + "percent": 99, + "score": 2137 + } + ], + "chunks": [ + { + "name": "ZHUANG", + "code": "za", + "offset": 0, + "bytes": 251 + } + ] + } + } +] diff --git a/wasm/browser-entry.mjs b/wasm/browser-entry.mjs new file mode 100644 index 0000000..31fbc90 --- /dev/null +++ b/wasm/browser-entry.mjs @@ -0,0 +1,34 @@ +// Entry point for bundlers/browsers (wired via package.json's "exports" +// "browser" condition). Deliberately a separate file from index.js rather +// than a runtime branch: bundlers statically choke on index.js's +// require('../build/Release/cld') even inside an unreachable branch, since +// a .node file can't be resolved at bundle time. This file never attempts +// the native path at all -- it goes straight to the WASM backend. + +import createCldModule from './dist/cld.web.mjs'; +import meta from '../lib/metadata.json' with { type: 'json' }; +import { createDetect } from '../lib/detect-shape.js'; +import { wrapWasmModule } from '../lib/wasm-wrap.js'; + +let moduleOptions = null; +let modulePromise = null; +function loadBackend() { + if (!modulePromise) { + modulePromise = createCldModule(moduleOptions ?? {}).then(wrapWasmModule); + } + return modulePromise; +} + +// Lets consumers override where cld.web.wasm is fetched from (e.g. when a +// bundler moves wasm assets to a different path/CDN than the JS glue +// expects by default). Must be called before the first detect() call -- +// loadBackend() only reads moduleOptions the first time it instantiates +// the module. +export function setWasmModuleOptions(options) { + moduleOptions = options; +} + +export const LANGUAGES = meta.LANGUAGES; +export const DETECTED_LANGUAGES = meta.DETECTED_LANGUAGES; +export const ENCODINGS = meta.ENCODINGS; +export const detect = createDetect(loadBackend, meta); diff --git a/wasm/dist/cld.node.js b/wasm/dist/cld.node.js new file mode 100644 index 0000000..d4ff556 --- /dev/null +++ b/wasm/dist/cld.node.js @@ -0,0 +1,2 @@ +async function createCldModule(moduleArg={}){var Module=moduleArg;var ENVIRONMENT_IS_NODE=true;var programArgs=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var _scriptName;if(typeof __filename!="undefined"){_scriptName=__filename}else{}var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("node:fs");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename);return ret};readAsync=async(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename,binary?undefined:"utf8");return ret};if(process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}programArgs=process.argv.slice(2);quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;var isFileURI=filename=>filename.startsWith("file://");class EmscriptenEH{}class EmscriptenSjLj extends EmscriptenEH{}var runtimeInitialized=false;function getMemoryBuffer(){return wasmMemory.buffer}function updateMemoryViews(){if(HEAP8?.buffer?.resizable)return;var b=getMemoryBuffer();HEAP8=new Int8Array(b);HEAPU8=new Uint8Array(b);HEAP32=new Int32Array(b);HEAPU32=new Uint32Array(b)}function preRun(){var preRun=Module["preRun"];if(preRun){if(typeof preRun=="function")preRun=[preRun];onPreRuns.push(...preRun)}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;wasmExports["h"]()}function postRun(){var postRun=Module["postRun"];if(postRun){if(typeof postRun=="function")postRun=[postRun];onPostRuns.push(...postRun)}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what=`Aborted(${what})`;err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var wasmBinaryFile;function findWasmBinary(){return locateFile("cld.node.wasm")}function getBinarySync(file){if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary&&!ENVIRONMENT_IS_NODE){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance){wasmExports=instance.exports;assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();var instantiateWasm=Module["instantiateWasm"];if(instantiateWasm){return new Promise(resolve=>{instantiateWasm(info,inst=>resolve(receiveInstance(inst)))})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var HEAP8;var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var onPreRuns=[];var noExitRuntime=true;var stackRestore=val=>__emscripten_stack_restore(val);var stackSave=()=>_emscripten_stack_get_current();var __abort_js=()=>abort("");var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var HEAPU8;var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);var HEAP32;var HEAPU32;var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var ENV={};var getExecutableName=()=>thisProgram;var getEnvStrings=()=>{if(!getEnvStrings.strings){var lang=(globalThis.navigator?.language??"C").replace("-","_")+".UTF-8";var env={USER:"web_user",LOGNAME:"web_user",PATH:"/",PWD:"/",HOME:"/home/web_user",LANG:lang,_:getExecutableName()};for(var x in ENV){if(ENV[x]===undefined)delete env[x];else env[x]=ENV[x]}var strings=[];for(var x in env){strings.push(`${x}=${env[x]}`)}getEnvStrings.strings=strings}return getEnvStrings.strings};var _environ_get=(__environ,environ_buf)=>{var bufSize=0;var envp=0;for(var string of getEnvStrings()){var ptr=environ_buf+bufSize;HEAPU32[__environ+envp>>2]=ptr;bufSize+=stringToUTF8(string,ptr,Infinity)+1;envp+=4}return 0};var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var _environ_sizes_get=(penviron_count,penviron_buf_size)=>{var strings=getEnvStrings();HEAPU32[penviron_count>>2]=strings.length;var bufSize=0;for(var string of strings){bufSize+=lengthBytesUTF8(string)+1}HEAPU32[penviron_buf_size>>2]=bufSize;return 0};var _fd_close=fd=>52;var printCharBuffers=[null,[],[]];var UTF8Decoder=globalThis.TextDecoder&&new TextDecoder;var findStringEnd=(heapOrArray,idx,maxBytesToRead,ignoreNul)=>{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var UTF8ArrayToString=(heapOrArray,idx=0,maxBytesToRead,ignoreNul)=>{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.subarray(idx,endPtr))}var str="";while(idx>10,56320|ch&1023)}}return str};var printChar=(stream,curr)=>{var buffer=printCharBuffers[stream];if(curr===0||curr===10){(stream===1?out:err)(UTF8ArrayToString(buffer));buffer.length=0}else{buffer.push(curr)}};var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead,ignoreNul):"";var _fd_write=(fd,iov,iovcnt,pnum)=>{var num=0;for(var i=0;i>2];var len=HEAPU32[iov+4>>2];iov+=8;for(var j=0;j>2]=num;return 0};var getCFunc=ident=>{var func=Module["_"+ident];return func};var writeArrayToMemory=(array,buffer)=>{HEAP8.set(array,buffer)};var stackAlloc=sz=>__emscripten_stack_alloc(sz);var stringToUTF8OnStack=str=>{var size=lengthBytesUTF8(str)+1;var ret=stackAlloc(size);stringToUTF8(str,ret,size);return ret};var ccall=(ident,returnType,argTypes,args,opts)=>{var toC={string:str=>{var ret=0;if(str!==null&&str!==undefined&&str!==0){ret=stringToUTF8OnStack(str)}return ret},array:arr=>{var ret=stackAlloc(arr.length);writeArrayToMemory(arr,ret);return ret}};function convertReturnValue(ret){if(returnType==="string"){return UTF8ToString(ret)}if(returnType==="boolean")return Boolean(ret);return ret}var func=getCFunc(ident);var cArgs=[];var stack=0;if(args){for(var i=0;i0){preInit.shift()()}}}Module["ccall"]=ccall;Module["UTF8ToString"]=UTF8ToString;var _cld_detect,_malloc,_cld_free,_free,__emscripten_stack_restore,__emscripten_stack_alloc,_emscripten_stack_get_current,memory,__indirect_function_table,wasmMemory;function assignWasmExports(wasmExports){_cld_detect=Module["_cld_detect"]=wasmExports["i"];_malloc=Module["_malloc"]=wasmExports["j"];_cld_free=Module["_cld_free"]=wasmExports["k"];_free=Module["_free"]=wasmExports["l"];__emscripten_stack_restore=wasmExports["m"];__emscripten_stack_alloc=wasmExports["n"];_emscripten_stack_get_current=wasmExports["o"];memory=wasmMemory=wasmExports["g"];__indirect_function_table=wasmExports["__indirect_function_table"]}var wasmImports={e:__abort_js,d:_emscripten_resize_heap,b:_environ_get,c:_environ_sizes_get,f:_fd_close,a:_fd_write};async function run(){preRun();var setStatus=Module["setStatus"];if(setStatus){setStatus("Running...");await new Promise(resolve=>setTimeout(resolve,1));setTimeout(setStatus,1,"")}if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}var wasmExports;wasmExports=await createWasm();await run(); +;return Module}if(typeof exports==="object"&&typeof module==="object"){module.exports=createCldModule;module.exports.default=createCldModule}else if(typeof define==="function"&&define["amd"])define([],()=>createCldModule); diff --git a/wasm/dist/cld.node.wasm b/wasm/dist/cld.node.wasm new file mode 100755 index 0000000..c1e72fc Binary files /dev/null and b/wasm/dist/cld.node.wasm differ diff --git a/wasm/dist/cld.web.mjs b/wasm/dist/cld.web.mjs new file mode 100644 index 0000000..f7e0ec9 --- /dev/null +++ b/wasm/dist/cld.web.mjs @@ -0,0 +1,2 @@ +async function createCldModule(moduleArg={}){var Module=moduleArg;var ENVIRONMENT_IS_WEB=true;var ENVIRONMENT_IS_WORKER=false;var programArgs=[];var thisProgram="./this.program";var _scriptName=import.meta.url;var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){try{scriptDirectory=new URL(".",_scriptName).href}catch{}{readAsync=async url=>{var response=await fetch(url,{credentials:"same-origin"});if(response.ok){return response.arrayBuffer()}throw new Error(response.status+" : "+response.url)}}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;class EmscriptenEH{}class EmscriptenSjLj extends EmscriptenEH{}var runtimeInitialized=false;function getMemoryBuffer(){return wasmMemory.buffer}function updateMemoryViews(){if(HEAP8?.buffer?.resizable)return;var b=getMemoryBuffer();HEAP8=new Int8Array(b);HEAPU8=new Uint8Array(b);HEAP32=new Int32Array(b);HEAPU32=new Uint32Array(b)}function preRun(){var preRun=Module["preRun"];if(preRun){if(typeof preRun=="function")preRun=[preRun];onPreRuns.push(...preRun)}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;wasmExports["h"]()}function postRun(){var postRun=Module["postRun"];if(postRun){if(typeof postRun=="function")postRun=[postRun];onPostRuns.push(...postRun)}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what=`Aborted(${what})`;err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var wasmBinaryFile;function findWasmBinary(){if(Module["locateFile"]){return locateFile("cld.web.wasm")}return new URL("cld.web.wasm",import.meta.url).href}function getBinarySync(file){if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance){wasmExports=instance.exports;assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();var instantiateWasm=Module["instantiateWasm"];if(instantiateWasm){return new Promise(resolve=>{instantiateWasm(info,inst=>resolve(receiveInstance(inst)))})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var HEAP8;var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var onPreRuns=[];var noExitRuntime=true;var stackRestore=val=>__emscripten_stack_restore(val);var stackSave=()=>_emscripten_stack_get_current();var __abort_js=()=>abort("");var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var HEAPU8;var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);var HEAP32;var HEAPU32;var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var ENV={};var getExecutableName=()=>thisProgram;var getEnvStrings=()=>{if(!getEnvStrings.strings){var lang=(globalThis.navigator?.language??"C").replace("-","_")+".UTF-8";var env={USER:"web_user",LOGNAME:"web_user",PATH:"/",PWD:"/",HOME:"/home/web_user",LANG:lang,_:getExecutableName()};for(var x in ENV){if(ENV[x]===undefined)delete env[x];else env[x]=ENV[x]}var strings=[];for(var x in env){strings.push(`${x}=${env[x]}`)}getEnvStrings.strings=strings}return getEnvStrings.strings};var _environ_get=(__environ,environ_buf)=>{var bufSize=0;var envp=0;for(var string of getEnvStrings()){var ptr=environ_buf+bufSize;HEAPU32[__environ+envp>>2]=ptr;bufSize+=stringToUTF8(string,ptr,Infinity)+1;envp+=4}return 0};var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var _environ_sizes_get=(penviron_count,penviron_buf_size)=>{var strings=getEnvStrings();HEAPU32[penviron_count>>2]=strings.length;var bufSize=0;for(var string of strings){bufSize+=lengthBytesUTF8(string)+1}HEAPU32[penviron_buf_size>>2]=bufSize;return 0};var _fd_close=fd=>52;var printCharBuffers=[null,[],[]];var UTF8Decoder=globalThis.TextDecoder&&new TextDecoder;var findStringEnd=(heapOrArray,idx,maxBytesToRead,ignoreNul)=>{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var UTF8ArrayToString=(heapOrArray,idx=0,maxBytesToRead,ignoreNul)=>{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.subarray(idx,endPtr))}var str="";while(idx>10,56320|ch&1023)}}return str};var printChar=(stream,curr)=>{var buffer=printCharBuffers[stream];if(curr===0||curr===10){(stream===1?out:err)(UTF8ArrayToString(buffer));buffer.length=0}else{buffer.push(curr)}};var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead,ignoreNul):"";var _fd_write=(fd,iov,iovcnt,pnum)=>{var num=0;for(var i=0;i>2];var len=HEAPU32[iov+4>>2];iov+=8;for(var j=0;j>2]=num;return 0};var getCFunc=ident=>{var func=Module["_"+ident];return func};var writeArrayToMemory=(array,buffer)=>{HEAP8.set(array,buffer)};var stackAlloc=sz=>__emscripten_stack_alloc(sz);var stringToUTF8OnStack=str=>{var size=lengthBytesUTF8(str)+1;var ret=stackAlloc(size);stringToUTF8(str,ret,size);return ret};var ccall=(ident,returnType,argTypes,args,opts)=>{var toC={string:str=>{var ret=0;if(str!==null&&str!==undefined&&str!==0){ret=stringToUTF8OnStack(str)}return ret},array:arr=>{var ret=stackAlloc(arr.length);writeArrayToMemory(arr,ret);return ret}};function convertReturnValue(ret){if(returnType==="string"){return UTF8ToString(ret)}if(returnType==="boolean")return Boolean(ret);return ret}var func=getCFunc(ident);var cArgs=[];var stack=0;if(args){for(var i=0;i0){preInit.shift()()}}}Module["ccall"]=ccall;Module["UTF8ToString"]=UTF8ToString;var _cld_detect,_malloc,_cld_free,_free,__emscripten_stack_restore,__emscripten_stack_alloc,_emscripten_stack_get_current,memory,__indirect_function_table,wasmMemory;function assignWasmExports(wasmExports){_cld_detect=Module["_cld_detect"]=wasmExports["i"];_malloc=Module["_malloc"]=wasmExports["j"];_cld_free=Module["_cld_free"]=wasmExports["k"];_free=Module["_free"]=wasmExports["l"];__emscripten_stack_restore=wasmExports["m"];__emscripten_stack_alloc=wasmExports["n"];_emscripten_stack_get_current=wasmExports["o"];memory=wasmMemory=wasmExports["g"];__indirect_function_table=wasmExports["__indirect_function_table"]}var wasmImports={e:__abort_js,d:_emscripten_resize_heap,b:_environ_get,c:_environ_sizes_get,f:_fd_close,a:_fd_write};async function run(){preRun();var setStatus=Module["setStatus"];if(setStatus){setStatus("Running...");await new Promise(resolve=>setTimeout(resolve,1));setTimeout(setStatus,1,"")}if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}var wasmExports;wasmExports=await createWasm();await run(); +;return Module}export default createCldModule; diff --git a/wasm/dist/cld.web.wasm b/wasm/dist/cld.web.wasm new file mode 100755 index 0000000..c1e72fc Binary files /dev/null and b/wasm/dist/cld.web.wasm differ diff --git a/wasm/glue.cc b/wasm/glue.cc new file mode 100644 index 0000000..8d758a3 --- /dev/null +++ b/wasm/glue.cc @@ -0,0 +1,125 @@ +#include +#include +#include +#include + +#include + +#include "cld_core.h" +#include "constants.h" + +namespace { + // Only CLD2's own ASCII language/encoding names ever end up as string + // values here (the detected input text itself is never echoed back), but + // escape defensively anyway since this is a JSON boundary. + void appendEscapedJSON(std::ostringstream &out, const char *s) { + out << '"'; + for (const char *p = s; *p; ++p) { + unsigned char c = static_cast(*p); + switch (c) { + case '"': out << "\\\""; break; + case '\\': out << "\\\\"; break; + case '\n': out << "\\n"; break; + case '\r': out << "\\r"; break; + case '\t': out << "\\t"; break; + default: + if (c < 0x20) { + char buf[8]; + snprintf(buf, sizeof(buf), "\\u%04x", c); + out << buf; + } else { + out << static_cast(c); + } + } + } + out << '"'; + } + + std::string serialize(const NodeCld::CLDOutput &output) { + std::ostringstream out; + // default ostream precision is 6 significant digits, which would + // silently truncate score (a double) for any value with more digits + // than that; 17 is enough to round-trip any double exactly, matching + // what Napi::Number::New does natively on the native addon side. + out.precision(17); + out << "{\"reliable\":" << (output.isReliable ? "true" : "false") + << ",\"textBytes\":" << output.textBytesFound + << ",\"languages\":["; + + bool first = true; + for (size_t i = 0; i < 3; i++) { + CLD2::Language lang = output.language3[i]; + if (lang == CLD2::UNKNOWN_LANGUAGE) { + continue; + } + if (!first) out << ","; + first = false; + out << "{\"name\":"; + appendEscapedJSON(out, NodeCld::Constants::getInstance().getLanguageName(lang)); + out << ",\"code\":"; + appendEscapedJSON(out, NodeCld::Constants::getInstance().getLanguageCode(lang)); + out << ",\"percent\":" << output.percent3[i] + << ",\"score\":" << output.normalized_score3[i] << "}"; + } + out << "],\"chunks\":["; + + first = true; + for (size_t i = 0; i < output.resultChunkVector.size(); i++) { + const CLD2::ResultChunk &chunk = output.resultChunkVector.at(i); + CLD2::Language lang = static_cast(chunk.lang1); + if (lang == CLD2::UNKNOWN_LANGUAGE) { + continue; + } + if (!first) out << ","; + first = false; + out << "{\"name\":"; + appendEscapedJSON(out, NodeCld::Constants::getInstance().getLanguageName(lang)); + out << ",\"code\":"; + appendEscapedJSON(out, NodeCld::Constants::getInstance().getLanguageCode(lang)); + out << ",\"offset\":" << chunk.offset + << ",\"bytes\":" << chunk.bytes << "}"; + } + out << "]}"; + + return out.str(); + } +} + +extern "C" { + // Returns a heap-allocated, NUL-terminated JSON string. Callers must use + // ccall(..., 'number', ...) (NOT the 'string' convenience return type, + // which decodes the string but never frees the underlying buffer) and + // pass the returned pointer to cld_free() once done reading it. + EMSCRIPTEN_KEEPALIVE + char* cld_detect( + const char* bytes, int numBytes, + int isPlainText, + const char* languageHint, + const char* encodingHint, + const char* tldHint, + const char* httpHint, + int bestEffort + ) { + NodeCld::CLDInput input; + input.bytes = std::string(bytes, numBytes); + input.numBytes = numBytes; + input.isPlainText = isPlainText != 0; + input.languageHint = languageHint ? languageHint : ""; + input.encodingHint = encodingHint ? encodingHint : ""; + input.tldHint = tldHint ? tldHint : ""; + input.httpHint = httpHint ? httpHint : ""; + input.bestEffort = bestEffort != 0; + + auto output = NodeCld::DetectLanguage(input); + std::string json = serialize(*output); + + char *result = static_cast(malloc(json.size() + 1)); + memcpy(result, json.c_str(), json.size() + 1); + return result; + } + + EMSCRIPTEN_KEEPALIVE + void cld_free(char* ptr) { + free(ptr); + } +}