diff --git a/benchmark/readline/readline-stream-split.js b/benchmark/readline/readline-stream-split.js new file mode 100644 index 000000000000..2e5bd8ef5f10 --- /dev/null +++ b/benchmark/readline/readline-stream-split.js @@ -0,0 +1,35 @@ +'use strict'; +const common = require('../common.js'); +const readline = require('readline'); +const { Readable } = require('stream'); + +const bench = common.createBenchmark(main, { + n: [1e4], + unicodeLineSeparators: [0, 1], +}); + +function main({ n, unicodeLineSeparators }) { + const line = '{"item":"data","value":42}\n'; + const chunks = Array.from({ length: 100 }, () => line); + + bench.start(); + let remaining = n; + + function run() { + if (remaining-- === 0) { + bench.end(n); + return; + } + + const input = Readable.from(chunks); + const rl = readline.createInterface({ + input, + unicodeLineSeparators: Boolean(unicodeLineSeparators), + }); + + rl.on('line', () => {}); + rl.on('close', run); + } + + run(); +} diff --git a/doc/api/readline.md b/doc/api/readline.md index 02e56e4c6a32..be6d689ae188 100644 --- a/doc/api/readline.md +++ b/doc/api/readline.md @@ -724,6 +724,10 @@ added: v17.0.0 * `tabSize` {integer} The number of spaces a tab is equal to (minimum 1). **Default:** `8`. * `signal` {AbortSignal} Allows closing the interface using an AbortSignal. + * `unicodeLineSeparators` {boolean} If `true`, `\u2028` (LINE SEPARATOR) and + `\u2029` (PARAGRAPH SEPARATOR) are treated as line endings. If `false`, + only `\r\n`, `\n`, and `\r` are treated as line endings. + **Default:** `true`. * Returns: {readlinePromises.Interface} The `readlinePromises.createInterface()` method creates a new `readlinePromises.Interface` @@ -990,6 +994,10 @@ changes: **Default:** `8`. * `signal` {AbortSignal} Allows closing the interface using an AbortSignal. Aborting the signal will internally call `close` on the interface. + * `unicodeLineSeparators` {boolean} If `true`, `\u2028` (LINE SEPARATOR) and + `\u2029` (PARAGRAPH SEPARATOR) are treated as line endings. If `false`, + only `\r\n`, `\n`, and `\r` are treated as line endings. + **Default:** `true`. * Returns: {readline.Interface} The `readline.createInterface()` method creates a new `readline.Interface` diff --git a/lib/internal/readline/interface.js b/lib/internal/readline/interface.js index bc8f30427616..580ab7335eb1 100644 --- a/lib/internal/readline/interface.js +++ b/lib/internal/readline/interface.js @@ -45,6 +45,7 @@ const { const { validateAbortSignal, + validateBoolean, validateString, validateUint32, } = require('internal/validators'); @@ -86,7 +87,8 @@ const kMincrlfDelay = 100; * - \u2028 (Unicode 'LINE SEPARATOR') * - \u2029 (Unicode 'PARAGRAPH SEPARATOR') */ -const lineEnding = /\r?\n|\r(?!\n)|\u2028|\u2029/g; +const lineEndingWithUnicode = /\r?\n|\r(?!\n)|\u2028|\u2029/g; +const lineEndingWithoutUnicode = /\r?\n|\r(?!\n)/g; const kLineObjectStream = Symbol('line object stream'); const kQuestionCancel = Symbol('kQuestionCancel'); @@ -117,6 +119,7 @@ const kMoveUpOrHistoryPrev = Symbol('_moveUpOrHistoryPrev'); const kInsertString = Symbol('_insertString'); const kLine = Symbol('_line'); const kLine_buffer = Symbol('_line_buffer'); +const kLineEnding = Symbol('_lineEnding'); const kKillRing = Symbol('_killRing'); const kKillRingCursor = Symbol('_killRingCursor'); const kMoveCursor = Symbol('_moveCursor'); @@ -176,6 +179,7 @@ function InterfaceConstructor(input, output, completer, terminal) { let prompt = '> '; let signal; let historyOptions; + let unicodeLineSeparators = true; if (input?.input) { // An options object was given @@ -208,6 +212,11 @@ function InterfaceConstructor(input, output, completer, terminal) { } } + if (input.unicodeLineSeparators !== undefined) { + validateBoolean(input.unicodeLineSeparators, 'options.unicodeLineSeparators'); + unicodeLineSeparators = input.unicodeLineSeparators; + } + if (signal) { validateAbortSignal(signal, 'options.signal'); } @@ -262,6 +271,10 @@ function InterfaceConstructor(input, output, completer, terminal) { this.setPrompt(prompt); this.terminal = !!terminal; + this.unicodeLineSeparators = unicodeLineSeparators; + this[kLineEnding] = unicodeLineSeparators ? + lineEndingWithUnicode : + lineEndingWithoutUnicode; function onerror(err) { self.emit('error', err); @@ -647,9 +660,11 @@ class Interface extends InterfaceConstructor { // string split is much cheaper than the regular expression. const lines = StringPrototypeIncludes(string, '\r') || - StringPrototypeIncludes(string, '\u2028') || - StringPrototypeIncludes(string, '\u2029') ? - RegExpPrototypeSymbolSplit(lineEnding, string) : + (this.unicodeLineSeparators && ( + StringPrototypeIncludes(string, '\u2028') || + StringPrototypeIncludes(string, '\u2029') + )) ? + RegExpPrototypeSymbolSplit(this[kLineEnding], string) : StringPrototypeSplit(string, '\n'); const lastIndex = lines.length - 1; if (lastIndex === 0) { @@ -1556,16 +1571,16 @@ class Interface extends InterfaceConstructor { default: if (typeof s === 'string' && s) { // Erase state of previous searches. - lineEnding.lastIndex = 0; + this[kLineEnding].lastIndex = 0; let nextMatch; // Keep track of the end of the last match. let lastIndex = 0; - while ((nextMatch = RegExpPrototypeExec(lineEnding, s)) !== null) { + while ((nextMatch = RegExpPrototypeExec(this[kLineEnding], s)) !== null) { this[kInsertString](StringPrototypeSlice(s, lastIndex, nextMatch.index)); - ({ lastIndex } = lineEnding); + ({ lastIndex } = this[kLineEnding]); this[kLine](); // Restore lastIndex as the call to kLine could have mutated it. - lineEnding.lastIndex = lastIndex; + this[kLineEnding].lastIndex = lastIndex; } // This ensures that the last line is written if it doesn't end in a newline. // Note that the last line may be the first line, in which case this still works. diff --git a/test/parallel/test-readline-async-iterators-unicode.js b/test/parallel/test-readline-async-iterators-unicode.js new file mode 100644 index 000000000000..1bd1adc1a113 --- /dev/null +++ b/test/parallel/test-readline-async-iterators-unicode.js @@ -0,0 +1,25 @@ +'use strict'; +const common = require('../common'); +const assert = require('node:assert'); +const readline = require('node:readline'); +const { Readable } = require('node:stream'); + +// Test async iterator on readline interface with unicodeLineSeparators: false +(async () => { + const input = 'line 1\u2028continued\nline 2\u2029continued\nline 3'; + const rli = readline.createInterface({ + input: Readable.from(input), + unicodeLineSeparators: false, + }); + + const lines = []; + for await (const line of rli) { + lines.push(line); + } + + assert.deepStrictEqual(lines, [ + 'line 1\u2028continued', + 'line 2\u2029continued', + 'line 3', + ]); +})().then(common.mustCall()); diff --git a/test/parallel/test-readline-promises-unicode.js b/test/parallel/test-readline-promises-unicode.js new file mode 100644 index 000000000000..9adeec1f7d7b --- /dev/null +++ b/test/parallel/test-readline-promises-unicode.js @@ -0,0 +1,21 @@ +'use strict'; +const common = require('../common'); +const assert = require('node:assert'); +const readlinePromises = require('node:readline/promises'); +const { Readable } = require('node:stream'); + +// Test readlinePromises.createInterface with unicodeLineSeparators: false +{ + const input = '{"a":1}\u2028{"b":2}\n{"c":3}'; + const rli = readlinePromises.createInterface({ + input: Readable.from(input), + unicodeLineSeparators: false, + }); + + const lines = []; + rli.on('line', (line) => lines.push(line)); + + rli.on('close', common.mustCall(() => { + assert.deepStrictEqual(lines, ['{"a":1}\u2028{"b":2}', '{"c":3}']); + })); +} diff --git a/test/parallel/test-readline-tty-unicode-separators.js b/test/parallel/test-readline-tty-unicode-separators.js new file mode 100644 index 000000000000..e093da7fbb6f --- /dev/null +++ b/test/parallel/test-readline-tty-unicode-separators.js @@ -0,0 +1,46 @@ +'use strict'; +require('../common'); +const assert = require('node:assert'); +const readline = require('node:readline'); +const { PassThrough } = require('node:stream'); + +// Test terminal write with unicodeLineSeparators: false +{ + const input = new PassThrough(); + const output = new PassThrough(); + + const rli = readline.createInterface({ + input, + output, + terminal: true, + unicodeLineSeparators: false, + }); + + const lines = []; + rli.on('line', (line) => lines.push(line)); + + input.write('foo\u2028bar\n'); + rli.close(); + + assert.deepStrictEqual(lines, ['foo\u2028bar']); +} + +// Test terminal write with default unicodeLineSeparators: true +{ + const input = new PassThrough(); + const output = new PassThrough(); + + const rli = readline.createInterface({ + input, + output, + terminal: true, + }); + + const lines = []; + rli.on('line', (line) => lines.push(line)); + + input.write('foo\u2028bar\n'); + rli.close(); + + assert.deepStrictEqual(lines, ['foo', 'bar']); +} diff --git a/test/parallel/test-readline-unicode-line-separators-opt-out.js b/test/parallel/test-readline-unicode-line-separators-opt-out.js new file mode 100644 index 000000000000..8b8f373eba4d --- /dev/null +++ b/test/parallel/test-readline-unicode-line-separators-opt-out.js @@ -0,0 +1,77 @@ +'use strict'; +const common = require('../common'); +const assert = require('node:assert'); +const readline = require('node:readline'); +const { Readable } = require('node:stream'); + +// Test 1: JSONL with Unicode line/paragraph separators inside JSON strings +{ + const jsonlData = '{"text":"Hello\\u2028World"}\n{"text":"Foo\\u2029Bar"}\n'; + + const rli = readline.createInterface({ + input: Readable.from(jsonlData), + unicodeLineSeparators: false, + }); + + const parsed = []; + rli.on('line', (line) => { + parsed.push(JSON.parse(line)); + }); + + rli.on('close', common.mustCall(() => { + assert.strictEqual(parsed.length, 2); + assert.deepStrictEqual(parsed[0], { text: 'Hello\u2028World' }); + assert.deepStrictEqual(parsed[1], { text: 'Foo\u2029Bar' }); + })); +} + +// Test 2: Default behavior remains unicodeLineSeparators: true +{ + const input = '012\n345\r67\r\n89\u{2028}ABC\u{2029}DEF'; + + const rli = readline.createInterface({ + input: Readable.from(input), + }); + + const lines = []; + rli.on('line', (line) => lines.push(line)); + + rli.on('close', common.mustCall(() => { + assert.deepStrictEqual(lines, ['012', '345', '67', '89', 'ABC', 'DEF']); + })); +} + +// Test 3: Standard CRLF/LF/CR still split, but U+2028 and U+2029 do not split +{ + const input = '012\n345\r67\r\n89\u{2028}ABC\u{2029}DEF'; + + const rli = readline.createInterface({ + input: Readable.from(input), + unicodeLineSeparators: false, + }); + + const lines = []; + rli.on('line', (line) => lines.push(line)); + + rli.on('close', common.mustCall(() => { + assert.deepStrictEqual(lines, ['012', '345', '67', '89\u{2028}ABC\u{2029}DEF']); + })); +} + +// Test 4: Argument type validation for unicodeLineSeparators +{ + for (const badValue of ['false', 0, 1, {}, [], () => {}]) { + assert.throws( + () => { + readline.createInterface({ + input: Readable.from(''), + unicodeLineSeparators: badValue, + }); + }, + { + code: 'ERR_INVALID_ARG_TYPE', + name: 'TypeError', + }, + ); + } +}