diff --git a/Project.toml b/Project.toml index dc3443a1..26180419 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "CSV" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -version = "1.1.0" +version = "1.2.0" authors = ["Jacob Quinn "] [deps] diff --git a/docs/src/examples.md b/docs/src/examples.md index 74123111..0cf934d3 100644 --- a/docs/src/examples.md +++ b/docs/src/examples.md @@ -391,6 +391,7 @@ file = CSV.File(IOBuffer(data); select=[:a, :c]) file = CSV.File(IOBuffer(data); select=["a", "c"]) file = CSV.File(IOBuffer(data); select=[true, false, true]) file = CSV.File(IOBuffer(data); select=r"[ac]") +file = CSV.File(IOBuffer(data); select=(i, name) -> name in (:a, :c)) # now examples of dropping, when we'd rather specify the column(s) # we'd like to drop/exclude from parsing file = CSV.File(IOBuffer(data); drop=[2]) @@ -398,10 +399,11 @@ file = CSV.File(IOBuffer(data); drop=[:b]) file = CSV.File(IOBuffer(data); drop=["b"]) file = CSV.File(IOBuffer(data); drop=[false, true, false]) file = CSV.File(IOBuffer(data); drop=:b) +file = CSV.File(IOBuffer(data); drop=(i, name) -> i == 2) ``` -`select` and `drop` take a list, one name or index, -or a `Regex`; the selected columns keep their file order. +`select` and `drop` take a list, one name or index, a `Regex`, or an +`(i, name) -> Bool` function; the selected columns keep their file order. ## [Limiting number of rows from data](@id limit_example) diff --git a/docs/src/migration.md b/docs/src/migration.md index 92d38885..8bdb18a7 100644 --- a/docs/src/migration.md +++ b/docs/src/migration.md @@ -57,11 +57,12 @@ throws), and extra trailing zeros are exact. The `PosLenString` output type is retired. Use `DataStrings.DataString`, `String`, or an InlineStrings.jl type. -`types`, `select`, `drop`, and `pool` no longer accept functions. Use type or -policy values, vectors, or dictionaries. `select` and `drop` take a list, one -name or index, or a `Regex`; they work on `CSV.File`, `CSV.lazy`, `CSV.Rows`, -and `CSV.Chunks` and return a unique file-ordered column set. Use -`Tables.Scan` for a serializable projection and filter expression. +`types` and `pool` no longer accept functions. Use type or policy values, +vectors, or dictionaries. `select` and `drop` take a list, one name or index, a +`Regex`, or the 0.10 `(i, name) -> Bool` function; they work on `CSV.File`, +`CSV.lazy`, `CSV.Rows`, and `CSV.Chunks` and return a unique file-ordered +column set. Use `Tables.Scan` for a serializable projection and filter +expression. ## Table access diff --git a/docs/src/reading.md b/docs/src/reading.md index b48451b0..88a9ab2d 100644 --- a/docs/src/reading.md +++ b/docs/src/reading.md @@ -223,8 +223,11 @@ numbers infer as `Float64`. name or index, or a `Regex` matched against the column names. They are mutually exclusive. `CSV.File`, `CSV.lazy`, `CSV.Rows`, and `CSV.Chunks` all return selected columns once, in file order, even when the list is repeated or -reordered. Function-valued selection is not supported. Use a `Tables.Scan` -request for expression-based projection and filtering. +reordered. A function of the form `(i, name) -> Bool` receives each column's +position and name (a `Symbol`); `select` keeps and `drop` removes the columns +for which it returns `true`. It sees only header names, so it runs before any +value is read, and CSV can call it more than once for a column. Use a +`Tables.Scan` request for expression-based projection and filtering. CSV.jl uses `DataStrings.DataString` for inferred text columns by default: diff --git a/docs/src/release-notes.md b/docs/src/release-notes.md index c83bb5f0..c7011412 100644 --- a/docs/src/release-notes.md +++ b/docs/src/release-notes.md @@ -1,3 +1,20 @@ +# CSV.jl 1.2.0 release notes + +Released October 15, 2026. Requires Julia 1.10 or later. + +## Changes + +- `select` and `drop` accept an `(i, name) -> Bool` function again, as in 0.10. + It receives each column's position and name before any value is read. 1.0 + and 1.1 rejected it. `types` and `pool` still do not accept functions. + +## Fixes + +- Delimiter detection only elects a delimiter that splits the first row. A + single-column file of times or timestamps, such as `event time` over + `10:30:00` rows, is read as one column again instead of being split on `:`, + as in 0.10. + # CSV.jl 1.1.0 release notes Released September 21, 2026. Requires Julia 1.10 or later. diff --git a/src/CSV.jl b/src/CSV.jl index 0882bbb2..97273034 100644 --- a/src/CSV.jl +++ b/src/CSV.jl @@ -79,7 +79,8 @@ become `missing` by default; `strict=true` or `on_error=:error` throws a [`CSV.ParseError`](@ref CSV.ParseError) when the cell is accessed. Rows do not retain parse diagnostics, so use [`CSV.File`](@ref CSV.File) when `CSV.problems` or a diagnostic cap is needed. `select` and `drop` (a list, one -name, or a `Regex`) project columns in stable file order. +name, a `Regex`, or an `(i, name) -> Bool` function) project columns in stable +file order. """ Rows @doc """ CSV.Chunks(source; ntasks=Threads.nthreads(), chunkbytes=nothing, keywords...) diff --git a/src/api.jl b/src/api.jl index b402e170..e9a26d64 100644 --- a/src/api.jl +++ b/src/api.jl @@ -18,8 +18,9 @@ # • problems are retained data; eager readers also warn once by default # (`strict=true` maps to `on_error=:error`, `maxwarnings` to `maxproblems`) # • empty unquoted cells are ALWAYS missing; `missingstring` ADDS spellings -# • `select`/`drop`/`types` take lists, names, regexes, or dictionaries -# (Tables.Scan is the expression channel) +# • `select`/`drop`/`types` take lists, names, regexes, or dictionaries; +# `select`/`drop` also take a `(i, name) -> Bool` function over header +# names (Tables.Scan is the expression channel) # • `stringtype` defaults to DataString; `stringtype=String` materializes; # InlineStrings are an extension # • Bool defaults accept lower, title, and upper case; user lists replace them diff --git a/src/core.jl b/src/core.jl index 314e3ef7..9ffc0513 100644 --- a/src/core.jl +++ b/src/core.jl @@ -3480,9 +3480,12 @@ function _selectpositions(select, drop, names::Vector{Symbol}; throw(ArgumentError("select and drop are mutually exclusive")) spec = select === nothing ? drop : select spec === nothing && return collect(eachindex(names)) - spec isa Base.Callable && - throw(ArgumentError("function-typed select/drop is retired; pass a list, " * - "a Regex, or use Tables.Scan for expressions")) + if spec isa Base.Callable + # a `(i, name) -> Bool` function sees header names only, so it becomes + # the Bool mask before any value is read + f = spec # the comprehension captures a single-assignment name + spec = Bool[f(i, nm)::Bool for (i, nm) in enumerate(names)] + end if spec isa Regex re = spec # the comprehension captures a single-assignment name matched = [nm for nm in names if occursin(re, String(nm))] diff --git a/test/api.jl b/test/api.jl index 8e442686..52f95540 100644 --- a/test/api.jl +++ b/test/api.jl @@ -9,7 +9,6 @@ # • empty unquoted cells are ALWAYS missing (custom missingstring ADDS) # • long rows do not widen the schema (extra fields ⇒ problem, not Column4) # • diagnostics are retained data, with one summary warning by default -# • function-typed select/drop retired # • wide integers that fit Int128 remain exact using Test, Dates, Tables, PooledArrays, CodecZlib, InlineStrings, FilePathsBase, Random, Mmap, Parsers @@ -496,7 +495,7 @@ end sourceparity(input; kw=(; drop=[2])) sourceparity(input; kw=(; drop=[false, true, false])) @test_throws ArgumentError A.File(IOBuffer(input); select=[:a], drop=[:b]) - @test_throws ArgumentError A.File(IOBuffer(input); select=(nm, i) -> i == 1) + @test_throws ArgumentError A.File(IOBuffer(input); select=(i, nm) -> i == 1, drop=[:b]) @test_throws ArgumentError A.File(IOBuffer(input); select=[:nope]) f = sourceparity("my col,b\n1,2\n"; kw=(; normalizenames=true, select=[:my_col])) @test Base.names(f) == [:my_col] @@ -2428,7 +2427,28 @@ end @test names(first(A.Chunks(IOBuffer(src); drop=r"^a"))) == [:b] @test_throws ArgumentError A.File(IOBuffer(src); select=r"^z") @test names(A.File(IOBuffer(src); drop=r"^z")) == [:ax, :ay, :b] - @test_throws ArgumentError A.File(IOBuffer(src); select=(i, nm) -> true) +end + +@testset "select and drop functions" begin + input = "a,b,c\n1,2,3\n4,5,6\n" + seen = Tuple{Int, Symbol}[] + f = sourceparity(input; kw=(; select=(i, nm) -> (push!(seen, (i, nm)); nm !== :b))) + @test Base.names(f) == [:a, :c] && f.c == [3, 6] + @test unique(seen) == [(1, :a), (2, :b), (3, :c)] + @test Base.names(sourceparity(input; kw=(; drop=(i, nm) -> i == 2))) == [:a, :c] + @test isempty(Base.names(A.File(IOBuffer(input); select=(i, nm) -> false))) + @test Base.names(A.File(IOBuffer(input); drop=(i, nm) -> false)) == [:a, :b, :c] + @test names(A.lazy(IOBuffer(input); drop=(i, nm) -> nm === :a)) == [:b, :c] + @test Tables.columnnames(A.Rows(IOBuffer(input); select=(i, nm) -> i > 1)) == [:b, :c] + @test all(ch -> names(ch) == [:a], A.Chunks(IOBuffer(input); select=(i, nm) -> i == 1)) + @test names(A.File([IOBuffer(input), IOBuffer(input)]; drop=(i, nm) -> i == 1)) == [:b, :c] + # the function sees the names after normalizenames + @test Base.names(A.File(IOBuffer("my col,b\n1,2\n"); normalizenames=true, + select=(i, nm) -> nm === :my_col)) == [:my_col] + # CSV.File(lazyfile) passes positions among the lazy file's columns + lf = A.lazy(IOBuffer(input); drop=[:a]) + @test Base.names(A.File(lf; select=(i, nm) -> i == 1)) == [:b] + @test_throws TypeError A.File(IOBuffer(input); select=(i, nm) -> missing) end @testset "types=String names the output type" begin