From 1e6635ffc61ed49b106f07584a575c50cbd8b960 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 22 Oct 2026 05:54:05 -0600 Subject: [PATCH] Check a quoted field for inner quotes with one vector compare Every quoted cell's content is found by scanning it for the close quote. The common field ends with its close quote and holds no other one; for up to 63 content bytes, one 64-byte compare (the scanner's byte mask) now proves that, and every other field keeps the byte loop. Results are the same by construction (2M random fields matched main's loop). findcontent: 20-28-byte clean fields 11-13 -> 6.2 ns; escaped fields about 1 ns slower. quoted.csv (1.5M rows, all fields quoted): about 10% less CPU single-threaded, 15-20% less wall at 8 tasks. Co-Authored-By: Claude Opus 5.5 --- src/core.jl | 8 ++++++++ test/kernel.jl | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/core.jl b/src/core.jl index 314e3ef7..2ee36b67 100644 --- a/src/core.jl +++ b/src/core.jl @@ -526,6 +526,14 @@ function findcontent(buf::Vector{UInt8}, i::Int, j::Int, k = i + 1 escaped = false if escape == closequote + # The common quoted field ends with its close quote and holds no other + # close quote: one 64-byte compare proves it for up to 63 content bytes. + n = j - i - 1 + if 0 <= n <= 63 && i + 64 <= length(buf) && @inbounds(buf[j]) == closequote + m = GC.@preserve buf byte_mask_vec(pointer(buf, i + 1), closequote) + m & ((one(UInt64) << n) - one(UInt64)) == zero(UInt64) && + return (i + 1, n, false, Parsers.RC_OK) + end GC.@preserve buf begin p = pointer(buf) @inbounds while k <= j diff --git a/test/kernel.jl b/test/kernel.jl index 8ae77867..f2b62c3a 100644 --- a/test/kernel.jl +++ b/test/kernel.jl @@ -741,6 +741,41 @@ end # --------------------------------------------------------------------------- +@testset "quoted field content around the 64-byte check" begin + # A clean quoted field of up to 63 content bytes with 64 bytes of buffer + # after its start is decided by one vector compare; every other field takes + # the byte loop. Both must give the exact rule's answer. + q = UInt8('"') + function reference(buf, i, j) + k = i + 1 + esc = false + while k <= j + if buf[k] == q + if k < j && buf[k + 1] == q + esc = true + k += 2 + continue + end + return (i + 1, j - i - 1, esc, k == j ? Parsers.RC_OK : Parsers.RC_INVALID) + end + k += 1 + end + return (i + 1, j - i, esc, Parsers.RC_INVALID) + end + for n in (0, 1, 7, 8, 62, 63, 64, 65, 130), tail in (0, 1, 62, 63, 64, 65), + inner in (:none, :lone_first, :lone_last, :doubled_first, :doubled_last) + content = fill(UInt8('a'), n) + inner === :lone_first && n >= 1 && (content[1] = q) + inner === :lone_last && n >= 1 && (content[n] = q) + inner === :doubled_first && n >= 2 && (content[1:2] .= q) + inner === :doubled_last && n >= 2 && (content[n-1:n] .= q) + field = [q; content; q] + buf = [UInt8('x'); field; fill(UInt8('y'), tail)] + i, j = 2, 1 + length(field) + @test K.findcontent(buf, i, j, q, q, q) == reference(buf, i, j) + end +end + @testset "Parsers 3 integration" begin quotechar = UInt8('"') quoted = Vector{UInt8}(codeunits("\"a\"\"b\""))