From 46c4bdec922c5adff8c894fc85f74626f5fd84ff Mon Sep 17 00:00:00 2001 From: cgarling Date: Wed, 2 Sep 2026 10:25:21 -0400 Subject: [PATCH 1/4] Add `to_tree` for extendable type writing Adds a write-conversion protocol for packages to extend allowing arbitrary Julia types to be embedded into ASDF documents. --- README.md | 5 ++ docs/make.jl | 1 + docs/src/api.md | 11 ++++ docs/src/custom-types.md | 89 +++++++++++++++++++++++++ docs/src/index.md | 5 +- src/ASDF.jl | 87 +++++++++++++++++++++++- test/test-write-converters.jl | 120 ++++++++++++++++++++++++++++++++++ 7 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 docs/src/custom-types.md create mode 100644 test/test-write-converters.jl diff --git a/README.md b/README.md index 1aeed6a..b57c79e 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs.io/en/latest/index.html) package, written in Julia. +Packages can extend `ASDF.to_tree` to embed their own Julia objects anywhere in +a larger ASDF document alongside ordinary metadata and binary arrays. See the +[custom type documentation](https://juliaastro.org/ASDF.jl/dev/custom-types/) +for the write-conversion interface. + ## Quickstart ```julia diff --git a/docs/make.jl b/docs/make.jl index f739fbb..283669d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,6 +20,7 @@ makedocs(; "JWST" => "examples/jwst.md", "Roman" => "examples/roman.md", ], + "Custom Julia types" => "custom-types.md", "Interoperability" => "interop.md", "API" => "api.md", ], diff --git a/docs/src/api.md b/docs/src/api.md index 93b75ca..47551a7 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -7,8 +7,19 @@ Modules = [ASDF] Private = false ``` +## Custom type writing + +```@docs +ASDF.WriteContext +ASDF.to_tree +ASDF.TaggedMapping +ASDF.TaggedSequence +ASDF.TaggedScalar +``` + ## Private ```@autodocs Modules = [ASDF] Public = false +Filter = value -> value ∉ (ASDF.WriteContext, ASDF.to_tree, ASDF.TaggedMapping, ASDF.TaggedSequence, ASDF.TaggedScalar) ``` diff --git a/docs/src/custom-types.md b/docs/src/custom-types.md new file mode 100644 index 0000000..4e644c8 --- /dev/null +++ b/docs/src/custom-types.md @@ -0,0 +1,89 @@ +# Custom Julia types + +ASDF documents often combine metadata, binary arrays, and objects owned by +domain packages. A package can make its types writable anywhere in an ASDF +document by extending [`ASDF.to_tree`](@ref): + +```@example custom_types +using ASDF + +struct Measurement + value::Float64 + unit::String +end + +function ASDF.to_tree(measurement::Measurement, context::ASDF.WriteContext) + properties = Dict("value" => measurement.value, "unit" => measurement.unit) + return ASDF.TaggedMapping("tag:example.org/measurement-1.0.0", properties) +end + +document = Dict( + "meta" => Dict("exposure" => Measurement(1200.0, "s")), + "data" => ASDF.NDArrayWrapper(reshape(collect(1.0:12.0), 3, 4)), +) + +save("custom-types.asdf", document) +``` + +`save` and [`ASDF.write_file`](@ref) recursively walk the complete document. +When they encounter a `Measurement`, Julia dispatch selects the method above. +ASDF then recursively converts custom objects contained in the returned node +before writing YAML and binary blocks. + +The original document is not modified. The same conversion can be inspected +without writing a file: + +```@example custom_types +node = ASDF.to_tree(Measurement(5.0, "m")) +node.tag +``` + +## Conversion contract + +The public interface has two forms: + +```julia +ASDF.to_tree(value) +ASDF.to_tree(value, context::ASDF.WriteContext) +``` + +The one-argument form recursively converts a value and is useful for inspecting +the ASDF representation. The two-argument form is the extension hook. Its +fallback returns the value unchanged. + +A package method should return one of: + +- a scalar, string, mapping, or array already supported by ASDF.jl; +- [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), or + [`ASDF.TaggedScalar`](@ref); +- [`ASDF.NDArrayWrapper`](@ref) for explicit inline or binary array storage. + +Converter methods are shallow. They may return mappings or sequences containing +other custom objects; ASDF.jl converts those children automatically. Methods +should accept the [`ASDF.WriteContext`](@ref) but treat it as opaque. + +ASDF.jl rejects cyclic mappings, arrays, or converter output because ASDF +reference serialization is not yet implemented. + +## Optional ASDF support + +When ASDF.jl is an optional dependency, define the method in a Julia package +extension that loads only when both packages are present: + +```julia +module MyPackageASDFExt + +using ASDF +using MyPackage + +function ASDF.to_tree(value::MyPackage.CustomType, context::ASDF.WriteContext) + return ASDF.TaggedMapping("tag:example.org/custom-1.0.0", Dict("value" => value.value)) +end + +end +``` + +This interface only controls writing. Loading an unknown tag with +`extensions = true` produces an `ASDF.TaggedMapping`, `ASDF.TaggedSequence`, or +`ASDF.TaggedScalar`; reconstructing package-owned objects and validating their +schemas require separate read-side support. diff --git a/docs/src/index.md b/docs/src/index.md index 3ad5c79..fbae546 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -125,4 +125,7 @@ af["data"][] == [1, 2, 3, 4] ## Tagged objects -Come back soon to see how custom Julia objects can be handled in ASDF.jl. +Packages can extend [`ASDF.to_tree`](@ref) to serialize their own Julia types +wherever they occur in a larger ASDF document. See [Custom Julia +types](@ref) for the conversion contract and an example combining a custom +metadata object with a binary array. diff --git a/src/ASDF.jl b/src/ASDF.jl index 8b68824..e292d0a 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -1153,6 +1153,86 @@ function YAML._print(io::IO, val::TaggedScalar, level::Int = 0, ignore_level::Bo return YAML._print(io, val.value, level, ignore_level) end +""" + WriteContext + +Opaque context for converting Julia objects to ASDF-compatible tree nodes. + +Packages that extend [`ASDF.to_tree`](@ref) should accept this context but must +not depend on its fields. Future versions may use it for schema selection, +array-storage policies, and extension provenance. +""" +struct WriteContext + _active::IdDict{Any, Nothing} +end +WriteContext() = WriteContext(IdDict{Any, Nothing}()) + +""" + ASDF.to_tree(value) + ASDF.to_tree(value, context::ASDF.WriteContext) + +Convert Julia objects to an ASDF-compatible tree. + +The one-argument form recursively converts `value` and any custom objects +nested in mappings, arrays, or tagged nodes. Packages add support for their own +types by defining a two-argument method that returns a scalar, mapping, +sequence, [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), +[`ASDF.TaggedScalar`](@ref), [`ASDF.NDArrayWrapper`](@ref), or another value +already supported by the ASDF writer. Returned mappings and sequences may +contain additional custom objects; ASDF converts those children recursively. + +The fallback two-argument method returns `value` unchanged. +""" +to_tree(value, ::WriteContext) = value +to_tree(value) = _convert_tree(value, WriteContext()) + +function _with_active(f, value, context::WriteContext) + haskey(context._active, value) && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) + context._active[value] = nothing + try + return f() + finally + delete!(context._active, value) + end +end + +function _convert_tree(value, context::WriteContext) + converted = to_tree(value, context) + converted === value && return _convert_tree_children(converted, context) + return _with_active(value, context) do + _convert_tree_children(converted, context) + end +end + +_convert_tree_children(value, context::WriteContext) = value +_convert_tree_children(value::TaggedScalar, context::WriteContext) = value + +function _convert_tree_children(value::TaggedMapping, context::WriteContext) + converted = _with_active(value.value, context) do + OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) + end + return TaggedMapping(value.tag, converted) +end + +function _convert_tree_children(value::TaggedSequence, context::WriteContext) + converted = _with_active(value.value, context) do + map(item -> _convert_tree(item, context), value.value) + end + return TaggedSequence(value.tag, converted) +end + +function _convert_tree_children(value::AbstractDict, context::WriteContext) + return _with_active(value, context) do + OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) + end +end + +function _convert_tree_children(value::AbstractArray, context::WriteContext) + return _with_active(value, context) do + map(item -> _convert_tree(item, context), value) + end +end + function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) # TODO: Get compression from underlying header block? return YAML._print(io, NDArrayWrapper(val[]; compression = C_None), level, ignore_level) @@ -1756,7 +1836,10 @@ end """ write_file(filename::AbstractString, document::AbstractDict) -Writes an ASDF file to disk. `document` is a plain `Dict` whose values may include [`NDArrayWrapper`](@ref) instances. These are serialized as binary blocks with appropriate compression. +Writes an ASDF file to disk. `document` may contain custom Julia objects with +two-argument [`ASDF.to_tree`](@ref) methods, including objects nested inside +mappings or arrays. Values may also include [`NDArrayWrapper`](@ref) instances, +which are serialized as binary blocks with appropriate compression. Layout of the output file: @@ -1783,6 +1866,8 @@ function write_file(filename::AbstractString, document::AbstractDict) # back to an unordered `Dict` and drop the order). The provenance entry is stamped last. full_document = OrderedDict{Any, Any}(document) full_document["asdf_library"] = library + # Convert package-owned objects before float normalization and block collection. + full_document = to_tree(full_document) # Rewrite floats so their exponents are YAML-1.1 compliant (see `yaml_compliant`). full_document = yaml_compliant(full_document) diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl new file mode 100644 index 0000000..9a4583c --- /dev/null +++ b/test/test-write-converters.jl @@ -0,0 +1,120 @@ +abstract type AbstractWriteValue end + +struct WriteValue <: AbstractWriteValue + value::Int +end + +struct SpecialWriteValue <: AbstractWriteValue + value::Int +end + +struct WriteParent + child +end + +struct WriteSequence + values::Vector{Any} +end + +struct WriteScalar + value::String +end + +struct WriteBlock + data::Matrix{Float64} +end + +struct CyclicWriteValue end + +ASDF.to_tree(value::AbstractWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::SpecialWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/special-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::WriteParent, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) +ASDF.to_tree(value::WriteSequence, context::ASDF.WriteContext) = ASDF.TaggedSequence("tag:example.org/write/sequence-1.0.0", value.values) +ASDF.to_tree(value::WriteScalar, context::ASDF.WriteContext) = ASDF.TaggedScalar("tag:example.org/write/scalar-1.0.0", value.value) +ASDF.to_tree(value::WriteBlock, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression = ASDF.C_Zlib))) +ASDF.to_tree(value::CyclicWriteValue, context::ASDF.WriteContext) = OrderedDict("self" => value) + +@testset "write conversion protocol" begin + shallow = ASDF.to_tree(WriteParent(WriteValue(3)), ASDF.WriteContext()) + @test shallow isa ASDF.TaggedMapping + @test shallow["child"] isa WriteValue + + parent = ASDF.to_tree(WriteParent(WriteValue(3))) + @test parent.tag == "tag:example.org/write/parent-1.0.0" + @test parent["child"] isa ASDF.TaggedMapping + @test parent["child"].tag == "tag:example.org/write/value-1.0.0" + @test parent["child"]["value"] == 3 + + sequence = ASDF.to_tree(WriteSequence(Any[WriteValue(4), "plain"])) + @test sequence isa ASDF.TaggedSequence + @test sequence.tag == "tag:example.org/write/sequence-1.0.0" + @test sequence[1] isa ASDF.TaggedMapping + @test sequence[2] == "plain" + + scalar = ASDF.to_tree(WriteScalar("converted")) + @test scalar isa ASDF.TaggedScalar + @test scalar.tag == "tag:example.org/write/scalar-1.0.0" + @test String(scalar) == "converted" + + @test ASDF.to_tree(WriteValue(5)).tag == "tag:example.org/write/value-1.0.0" + @test ASDF.to_tree(SpecialWriteValue(5)).tag == "tag:example.org/write/special-1.0.0" +end + +@testset "recursive write conversion" begin + tagged = ASDF.TaggedMapping("tag:example.org/write/existing-1.0.0", OrderedDict("child" => WriteValue(6))) + converted = ASDF.to_tree(tagged) + @test converted.tag == tagged.tag + @test converted["child"] isa ASDF.TaggedMapping + @test tagged["child"] isa WriteValue + + source = OrderedDict("zebra" => WriteValue(7), "apple" => Any[1, WriteValue(8)], "matrix" => [1 2; 3 4]) + plain = ASDF.to_tree(source) + @test collect(keys(plain)) == collect(keys(source)) + @test plain["zebra"] isa ASDF.TaggedMapping + @test plain["apple"][2] isa ASDF.TaggedMapping + @test plain["matrix"] == source["matrix"] + @test source["zebra"] isa WriteValue + @test source["apple"][2] isa WriteValue + + cyclic_mapping = OrderedDict{Any, Any}() + cyclic_mapping["self"] = cyclic_mapping + @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_mapping) + + cyclic_sequence = Any[] + push!(cyclic_sequence, cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF.to_tree(CyclicWriteValue()) +end + +@testset "heterogeneous document writing" begin + data = reshape(collect(1.0:12.0), 3, 4) + document = OrderedDict( + "roman" => OrderedDict( + "meta" => OrderedDict("model" => WriteParent(SpecialWriteValue(9)), "description" => "nested"), + "data" => WriteBlock(data), + ), + "name" => "product", + ) + + mktempdir() do directory + filename = joinpath(directory, "custom.asdf") + ASDF.write_file(filename, document) + loaded = ASDF.load_file(filename; extensions = true) + + @test collect(keys(loaded.metadata)) == ["roman", "name", "asdf_library"] + @test loaded["name"] == "product" + @test loaded["roman"]["meta"]["model"].tag == "tag:example.org/write/parent-1.0.0" + @test loaded["roman"]["meta"]["model"]["child"].tag == "tag:example.org/write/special-1.0.0" + @test loaded["roman"]["data"].tag == "tag:example.org/write/block-1.0.0" + @test loaded["roman"]["data"]["data"][] == data + @test !haskey(document, "asdf_library") + @test document["roman"]["meta"]["model"] isa WriteParent + @test document["roman"]["data"] isa WriteBlock + + save_filename = joinpath(directory, "custom-save.asdf") + save(save_filename, document) + saved = ASDF.load_file(save_filename; extensions = true) + @test saved["roman"]["data"]["data"][] == data + @test saved["roman"]["meta"]["model"]["child"]["value"] == 9 + end +end From 50985e66f1a7e9d58713b587c72a2c1888623f27 Mon Sep 17 00:00:00 2001 From: cgarling Date: Sun, 27 Sep 2026 15:40:28 -0400 Subject: [PATCH 2/4] Simplify extensible ASDF tree writing --- docs/src/api.md | 11 ---- docs/src/custom-types.md | 41 ++++++------ src/ASDF.jl | 115 +++++++++++++++------------------- test/test-write-converters.jl | 114 +++++++++++---------------------- 4 files changed, 103 insertions(+), 178 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 47551a7..93b75ca 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -7,19 +7,8 @@ Modules = [ASDF] Private = false ``` -## Custom type writing - -```@docs -ASDF.WriteContext -ASDF.to_tree -ASDF.TaggedMapping -ASDF.TaggedSequence -ASDF.TaggedScalar -``` - ## Private ```@autodocs Modules = [ASDF] Public = false -Filter = value -> value ∉ (ASDF.WriteContext, ASDF.to_tree, ASDF.TaggedMapping, ASDF.TaggedSequence, ASDF.TaggedScalar) ``` diff --git a/docs/src/custom-types.md b/docs/src/custom-types.md index 4e644c8..38de4a0 100644 --- a/docs/src/custom-types.md +++ b/docs/src/custom-types.md @@ -6,19 +6,20 @@ document by extending [`ASDF.to_tree`](@ref): ```@example custom_types using ASDF +using OrderedCollections struct Measurement value::Float64 unit::String end -function ASDF.to_tree(measurement::Measurement, context::ASDF.WriteContext) - properties = Dict("value" => measurement.value, "unit" => measurement.unit) +function ASDF.to_tree(measurement::Measurement) + properties = OrderedDict("value" => measurement.value, "unit" => measurement.unit) return ASDF.TaggedMapping("tag:example.org/measurement-1.0.0", properties) end -document = Dict( - "meta" => Dict("exposure" => Measurement(1200.0, "s")), +document = OrderedDict( + "meta" => OrderedDict("exposure" => Measurement(1200.0, "s")), "data" => ASDF.NDArrayWrapper(reshape(collect(1.0:12.0), 3, 4)), ) @@ -30,8 +31,8 @@ When they encounter a `Measurement`, Julia dispatch selects the method above. ASDF then recursively converts custom objects contained in the returned node before writing YAML and binary blocks. -The original document is not modified. The same conversion can be inspected -without writing a file: +The original document is not modified. Calling the hook directly inspects its +shallow representation: ```@example custom_types node = ASDF.to_tree(Measurement(5.0, "m")) @@ -40,30 +41,26 @@ node.tag ## Conversion contract -The public interface has two forms: - -```julia -ASDF.to_tree(value) -ASDF.to_tree(value, context::ASDF.WriteContext) -``` - -The one-argument form recursively converts a value and is useful for inspecting -the ASDF representation. The two-argument form is the extension hook. Its -fallback returns the value unchanged. +Packages extend the one-argument `ASDF.to_tree(value)` hook. The fallback +returns `value` unchanged. A package method should return one of: -- a scalar, string, mapping, or array already supported by ASDF.jl; +- `nothing`, a boolean, integer, float, or string; +- a mapping with boolean, integer, or string keys, a vector, tuple, or named tuple; - [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), or [`ASDF.TaggedScalar`](@ref); - [`ASDF.NDArrayWrapper`](@ref) for explicit inline or binary array storage. Converter methods are shallow. They may return mappings or sequences containing -other custom objects; ASDF.jl converts those children automatically. Methods -should accept the [`ASDF.WriteContext`](@ref) but treat it as opaque. +other custom objects; the writer converts those children automatically and +redispatches when a converter delegates to another custom type. Converters +should not call `to_tree` recursively themselves. -ASDF.jl rejects cyclic mappings, arrays, or converter output because ASDF -reference serialization is not yet implemented. +Unsupported leaves and mapping keys produce an error instead of being silently +stringified. Multidimensional arrays must be wrapped in `NDArrayWrapper`; +metadata sequences are vectors. ASDF.jl rejects cyclic mappings, sequences, or +converter output because ASDF reference serialization is not yet implemented. ## Optional ASDF support @@ -76,7 +73,7 @@ module MyPackageASDFExt using ASDF using MyPackage -function ASDF.to_tree(value::MyPackage.CustomType, context::ASDF.WriteContext) +function ASDF.to_tree(value::MyPackage.CustomType) return ASDF.TaggedMapping("tag:example.org/custom-1.0.0", Dict("value" => value.value)) end diff --git a/src/ASDF.jl b/src/ASDF.jl index 5fd10f3..9435f39 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -1067,6 +1067,7 @@ but otherwise behave as plain mappings; and, when loading with `extensions = tru extension tags that would otherwise raise an error. Each behaves exactly like its wrapped `value`. A mapping indexes and iterates as a dict, a sequence as a vector, a scalar as a string, while retaining the original `tag` so the node round-trips unchanged through [`ASDF.write_file`](@ref). +The `tag` field must contain the full tag URI; the writer emits namespace shorthand where applicable. """ struct TaggedMapping{D <: AbstractDict} <: AbstractDict{Any, Any} tag::String @@ -1153,84 +1154,66 @@ function YAML._print(io::IO, val::TaggedScalar, level::Int = 0, ignore_level::Bo return YAML._print(io, val.value, level, ignore_level) end -""" - WriteContext - -Opaque context for converting Julia objects to ASDF-compatible tree nodes. - -Packages that extend [`ASDF.to_tree`](@ref) should accept this context but must -not depend on its fields. Future versions may use it for schema selection, -array-storage policies, and extension provenance. -""" -struct WriteContext - _active::IdDict{Any, Nothing} -end -WriteContext() = WriteContext(IdDict{Any, Nothing}()) - """ ASDF.to_tree(value) - ASDF.to_tree(value, context::ASDF.WriteContext) -Convert Julia objects to an ASDF-compatible tree. +Convert one package-owned value to an ASDF-compatible tree node. -The one-argument form recursively converts `value` and any custom objects -nested in mappings, arrays, or tagged nodes. Packages add support for their own -types by defining a two-argument method that returns a scalar, mapping, -sequence, [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), -[`ASDF.TaggedScalar`](@ref), [`ASDF.NDArrayWrapper`](@ref), or another value -already supported by the ASDF writer. Returned mappings and sequences may -contain additional custom objects; ASDF converts those children recursively. - -The fallback two-argument method returns `value` unchanged. +Packages extend this shallow hook for their own types. A method must return a +supported scalar, mapping, vector, [`ASDF.TaggedMapping`](@ref), +[`ASDF.TaggedSequence`](@ref), [`ASDF.TaggedScalar`](@ref), or +[`ASDF.NDArrayWrapper`](@ref). ASDF recursively converts values nested in the +returned node while writing a file. The fallback returns `value` unchanged. """ -to_tree(value, ::WriteContext) = value -to_tree(value) = _convert_tree(value, WriteContext()) +to_tree(value) = value -function _with_active(f, value, context::WriteContext) - haskey(context._active, value) && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) - context._active[value] = nothing +function _convert_tree(value, active = Base.IdSet{Any}()) + value in active && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) + push!(active, value) try - return f() + converted = to_tree(value) + if converted === value + which(to_tree, (typeof(value),)) === which(to_tree, (Any,)) && return _convert_tree_children(value, active) + throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) + end + typeof(converted) === typeof(value) && throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) + return _convert_tree(converted, active) finally - delete!(context._active, value) + delete!(active, value) end end -function _convert_tree(value, context::WriteContext) - converted = to_tree(value, context) - converted === value && return _convert_tree_children(converted, context) - return _with_active(value, context) do - _convert_tree_children(converted, context) - end -end +_convert_tree_children(value::Nothing, active) = value +_convert_tree_children(value::Union{Bool,Integer,AbstractString}, active) = value +_convert_tree_children(value::AbstractFloat, active) = YAMLScalar(yaml_float_string(value)) +_convert_tree_children(value::NDArray, active) = value +_convert_tree_children(value::TaggedScalar, active) = TaggedScalar(value.tag, _convert_tree(value.value, active)) +_convert_tree_children(value::NamedTuple, active) = OrderedDict{Any,Any}(String(key) => _convert_tree(item, active) for (key, item) in pairs(value)) +_convert_tree_children(value::Tuple, active) = [_convert_tree(item, active) for item in value] -_convert_tree_children(value, context::WriteContext) = value -_convert_tree_children(value::TaggedScalar, context::WriteContext) = value - -function _convert_tree_children(value::TaggedMapping, context::WriteContext) - converted = _with_active(value.value, context) do - OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) - end - return TaggedMapping(value.tag, converted) +function _convert_tree_children(value::TaggedMapping, active) + return TaggedMapping(value.tag, _convert_tree_children(value.value, active)) end -function _convert_tree_children(value::TaggedSequence, context::WriteContext) - converted = _with_active(value.value, context) do - map(item -> _convert_tree(item, context), value.value) - end - return TaggedSequence(value.tag, converted) +function _convert_tree_children(value::TaggedSequence, active) + return TaggedSequence(value.tag, _convert_tree_children(value.value, active)) end -function _convert_tree_children(value::AbstractDict, context::WriteContext) - return _with_active(value, context) do - OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) +function _convert_tree_children(value::AbstractDict, active) + converted = OrderedDict{Any,Any}() + for (key, item) in value + key isa Union{Bool,Integer,AbstractString} || throw(ArgumentError("ASDF mapping key $(repr(key)) has unsupported type $(typeof(key)); keys must be booleans, integers, or strings")) + converted[key] = _convert_tree(item, active) end + return converted end -function _convert_tree_children(value::AbstractArray, context::WriteContext) - return _with_active(value, context) do - map(item -> _convert_tree(item, context), value) - end +_convert_tree_children(value::AbstractVector, active) = [_convert_tree(item, active) for item in value] +function _convert_tree_children(value::AbstractArray, active) + throw(ArgumentError("ASDF metadata arrays must be vectors; wrap $(typeof(value)) in ASDF.NDArrayWrapper to write an N-dimensional array")) +end +function _convert_tree_children(value, active) + throw(ArgumentError("value of type $(typeof(value)) is not supported by the ASDF writer; define ASDF.to_tree(::$(typeof(value)))")) end function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) @@ -1667,6 +1650,7 @@ function YAML._print(io::IO, val::ASDFLibrary, level::Int = 0, ignore_level::Boo library = OrderedDict(:name => val.name, :author => val.author, :homepage => val.homepage, :version => val.version) return YAML._print(io, library, level, ignore_level) end +_convert_tree_children(value::ASDFLibrary, active) = value """ NDArrayWrapper @@ -1692,6 +1676,7 @@ function NDArrayWrapper(array::AbstractArray; compression::Compression = C_Bzip2 return NDArrayWrapper(array, compression, inline, lz4_layout) end Base.getindex(val::NDArrayWrapper) = val.array +_convert_tree_children(value::NDArrayWrapper, active) = value """ Blocks @@ -1837,9 +1822,9 @@ end write_file(filename::AbstractString, document::AbstractDict) Writes an ASDF file to disk. `document` may contain custom Julia objects with -two-argument [`ASDF.to_tree`](@ref) methods, including objects nested inside -mappings or arrays. Values may also include [`NDArrayWrapper`](@ref) instances, -which are serialized as binary blocks with appropriate compression. +[`ASDF.to_tree`](@ref) methods, including objects nested inside mappings or +vectors. Values may also include [`NDArrayWrapper`](@ref) instances, which are +serialized as binary blocks with appropriate compression. Layout of the output file: @@ -1866,10 +1851,8 @@ function write_file(filename::AbstractString, document::AbstractDict) # back to an unordered `Dict` and drop the order). The provenance entry is stamped last. full_document = OrderedDict{Any, Any}(document) full_document["asdf_library"] = library - # Convert package-owned objects before float normalization and block collection. - full_document = to_tree(full_document) - # Rewrite floats so their exponents are YAML-1.1 compliant (see `yaml_compliant`). - full_document = yaml_compliant(full_document) + # Convert package-owned objects and normalize floats before block collection. + full_document = _convert_tree(full_document) # Write YAML part of file io = open(filename, "w") diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl index 9a4583c..4f6299c 100644 --- a/test/test-write-converters.jl +++ b/test/test-write-converters.jl @@ -1,96 +1,61 @@ -abstract type AbstractWriteValue end - -struct WriteValue <: AbstractWriteValue +struct WriteValue value::Int end - -struct SpecialWriteValue <: AbstractWriteValue - value::Int -end - struct WriteParent child end - -struct WriteSequence - values::Vector{Any} -end - -struct WriteScalar - value::String +struct WriteAlias + value::WriteValue end - struct WriteBlock data::Matrix{Float64} end +struct WriteLoop end +struct WriteSame + value::Int +end +struct UnsupportedWriteValue end -struct CyclicWriteValue end - -ASDF.to_tree(value::AbstractWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) -ASDF.to_tree(value::SpecialWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/special-1.0.0", OrderedDict("value" => value.value)) -ASDF.to_tree(value::WriteParent, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) -ASDF.to_tree(value::WriteSequence, context::ASDF.WriteContext) = ASDF.TaggedSequence("tag:example.org/write/sequence-1.0.0", value.values) -ASDF.to_tree(value::WriteScalar, context::ASDF.WriteContext) = ASDF.TaggedScalar("tag:example.org/write/scalar-1.0.0", value.value) -ASDF.to_tree(value::WriteBlock, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression = ASDF.C_Zlib))) -ASDF.to_tree(value::CyclicWriteValue, context::ASDF.WriteContext) = OrderedDict("self" => value) +ASDF.to_tree(value::WriteValue) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::WriteParent) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) +ASDF.to_tree(value::WriteAlias) = value.value +ASDF.to_tree(value::WriteBlock) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression=ASDF.C_Zlib))) +ASDF.to_tree(value::WriteLoop) = OrderedDict("self" => value) +ASDF.to_tree(value::WriteSame) = WriteSame(value.value) @testset "write conversion protocol" begin - shallow = ASDF.to_tree(WriteParent(WriteValue(3)), ASDF.WriteContext()) - @test shallow isa ASDF.TaggedMapping + shallow = ASDF.to_tree(WriteParent(WriteValue(3))) @test shallow["child"] isa WriteValue - parent = ASDF.to_tree(WriteParent(WriteValue(3))) - @test parent.tag == "tag:example.org/write/parent-1.0.0" - @test parent["child"] isa ASDF.TaggedMapping - @test parent["child"].tag == "tag:example.org/write/value-1.0.0" - @test parent["child"]["value"] == 3 - - sequence = ASDF.to_tree(WriteSequence(Any[WriteValue(4), "plain"])) - @test sequence isa ASDF.TaggedSequence - @test sequence.tag == "tag:example.org/write/sequence-1.0.0" - @test sequence[1] isa ASDF.TaggedMapping - @test sequence[2] == "plain" + converted = ASDF._convert_tree(WriteParent(WriteValue(3))) + @test converted.tag == "tag:example.org/write/parent-1.0.0" + @test converted["child"].tag == "tag:example.org/write/value-1.0.0" + @test ASDF._convert_tree(WriteAlias(WriteValue(4)))["value"] == 4 - scalar = ASDF.to_tree(WriteScalar("converted")) - @test scalar isa ASDF.TaggedScalar - @test scalar.tag == "tag:example.org/write/scalar-1.0.0" - @test String(scalar) == "converted" - - @test ASDF.to_tree(WriteValue(5)).tag == "tag:example.org/write/value-1.0.0" - @test ASDF.to_tree(SpecialWriteValue(5)).tag == "tag:example.org/write/special-1.0.0" -end + source = OrderedDict("tuple" => (WriteValue(5),), "named" => (child=WriteValue(6),)) + tree = ASDF._convert_tree(source) + @test tree["tuple"][1]["value"] == 5 + @test tree["named"]["child"]["value"] == 6 + @test source["tuple"][1] isa WriteValue -@testset "recursive write conversion" begin - tagged = ASDF.TaggedMapping("tag:example.org/write/existing-1.0.0", OrderedDict("child" => WriteValue(6))) - converted = ASDF.to_tree(tagged) - @test converted.tag == tagged.tag - @test converted["child"] isa ASDF.TaggedMapping - @test tagged["child"] isa WriteValue - - source = OrderedDict("zebra" => WriteValue(7), "apple" => Any[1, WriteValue(8)], "matrix" => [1 2; 3 4]) - plain = ASDF.to_tree(source) - @test collect(keys(plain)) == collect(keys(source)) - @test plain["zebra"] isa ASDF.TaggedMapping - @test plain["apple"][2] isa ASDF.TaggedMapping - @test plain["matrix"] == source["matrix"] - @test source["zebra"] isa WriteValue - @test source["apple"][2] isa WriteValue - - cyclic_mapping = OrderedDict{Any, Any}() + cyclic_mapping = OrderedDict{Any,Any}() cyclic_mapping["self"] = cyclic_mapping - @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_mapping) - cyclic_sequence = Any[] push!(cyclic_sequence, cyclic_sequence) - @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_sequence) - @test_throws "cyclic ASDF write conversion" ASDF.to_tree(CyclicWriteValue()) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_mapping) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(WriteLoop()) + @test_throws "must return a supported ASDF tree node" ASDF._convert_tree(WriteSame(1)) + @test_throws "not supported by the ASDF writer" ASDF._convert_tree(UnsupportedWriteValue()) + @test_throws "keys must be booleans, integers, or strings" ASDF._convert_tree(OrderedDict(WriteValue(1) => 2)) + @test_throws "wrap Matrix" ASDF._convert_tree([WriteValue(1) WriteValue(2)]) end @testset "heterogeneous document writing" begin data = reshape(collect(1.0:12.0), 3, 4) document = OrderedDict( "roman" => OrderedDict( - "meta" => OrderedDict("model" => WriteParent(SpecialWriteValue(9)), "description" => "nested"), + "meta" => OrderedDict("model" => WriteParent(WriteValue(9)), "description" => "nested"), "data" => WriteBlock(data), ), "name" => "product", @@ -99,22 +64,13 @@ end mktempdir() do directory filename = joinpath(directory, "custom.asdf") ASDF.write_file(filename, document) - loaded = ASDF.load_file(filename; extensions = true) + loaded = ASDF.load_file(filename; extensions=true) @test collect(keys(loaded.metadata)) == ["roman", "name", "asdf_library"] - @test loaded["name"] == "product" - @test loaded["roman"]["meta"]["model"].tag == "tag:example.org/write/parent-1.0.0" - @test loaded["roman"]["meta"]["model"]["child"].tag == "tag:example.org/write/special-1.0.0" - @test loaded["roman"]["data"].tag == "tag:example.org/write/block-1.0.0" + @test loaded["roman"]["meta"]["model"]["child"]["value"] == 9 @test loaded["roman"]["data"]["data"][] == data @test !haskey(document, "asdf_library") @test document["roman"]["meta"]["model"] isa WriteParent @test document["roman"]["data"] isa WriteBlock - - save_filename = joinpath(directory, "custom-save.asdf") - save(save_filename, document) - saved = ASDF.load_file(save_filename; extensions = true) - @test saved["roman"]["data"]["data"][] == data - @test saved["roman"]["meta"]["model"]["child"]["value"] == 9 end end From 563a977c13dee9fdb9b0204463bac03647a50ddd Mon Sep 17 00:00:00 2001 From: Ian Weaver Date: Wed, 30 Sep 2026 23:00:39 -0700 Subject: [PATCH 3/4] Next round of reviews --- Project.toml | 2 ++ docs/src/custom-types.md | 14 ++++++------ docs/src/index.md | 2 +- src/ASDF.jl | 35 +++++++++++++++--------------- test/Project.toml | 1 + test/test-read_chunked.jl | 7 ++++++ test/test-write-converters.jl | 40 ++++++++++++++++++++++++++++------- 7 files changed, 69 insertions(+), 32 deletions(-) diff --git a/Project.toml b/Project.toml index 3f12623..c1fe20e 100644 --- a/Project.toml +++ b/Project.toml @@ -14,6 +14,7 @@ ChunkCodecLibLz4 = "7e9cc85e-5614-42a3-ad86-b78f920b38a5" ChunkCodecLibLzma = "e95d29e5-19c5-4afd-ae0f-beb790efacdf" ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -29,6 +30,7 @@ ChunkCodecLibLz4 = "0.2.1, 1.0" ChunkCodecLibLzma = "1.0" ChunkCodecLibZlib = "0.2.0, 1.0" ChunkCodecLibZstd = "0.2.0, 1.0" +Dates = "1" FileIO = "1" MD5 = "0.2.1" OrderedCollections = "1.8.1, 2" diff --git a/docs/src/custom-types.md b/docs/src/custom-types.md index 38de4a0..9bdb7da 100644 --- a/docs/src/custom-types.md +++ b/docs/src/custom-types.md @@ -46,8 +46,9 @@ returns `value` unchanged. A package method should return one of: -- `nothing`, a boolean, integer, float, or string; -- a mapping with boolean, integer, or string keys, a vector, tuple, or named tuple; +- `nothing`, a boolean, integer, float, string, symbol, `Date`, or `DateTime`; +- a mapping with boolean, integer, string, or symbol keys, a vector, tuple, or + named tuple; - [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), or [`ASDF.TaggedScalar`](@ref); - [`ASDF.NDArrayWrapper`](@ref) for explicit inline or binary array storage. @@ -57,10 +58,11 @@ other custom objects; the writer converts those children automatically and redispatches when a converter delegates to another custom type. Converters should not call `to_tree` recursively themselves. -Unsupported leaves and mapping keys produce an error instead of being silently -stringified. Multidimensional arrays must be wrapped in `NDArrayWrapper`; -metadata sequences are vectors. ASDF.jl rejects cyclic mappings, sequences, or -converter output because ASDF reference serialization is not yet implemented. +Symbols are written as strings. Unsupported leaves and mapping keys produce an +error instead of being silently stringified. Multidimensional arrays must be +wrapped in `NDArrayWrapper`; metadata sequences are vectors. ASDF.jl rejects +cyclic mappings, sequences, or converter output because ASDF reference +serialization is not yet implemented. ## Optional ASDF support diff --git a/docs/src/index.md b/docs/src/index.md index b1b3791..1c0e3c8 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -4,7 +4,7 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs ## Introduction -The ASDF file format is based on the human-readable [YAML](http://yaml.org/) standard, extended with efficient binary blocks to store array data. Basic arithmetic types (`Bool`, `Int`, `Float`, `Complex`) and `String` types are supported out of the box. Other types (structures) need to be declared to be supported. +The ASDF file format is based on the human-readable [YAML](http://yaml.org/) standard, extended with efficient binary blocks to store array data. Basic arithmetic types (`Bool`, `Int`, `Float`) and `String` types are supported out of the box, as are arrays of real and complex numbers. Other types (structures) need to be declared to be supported. ASDF supports arbitrary array strides, both C (Python) and Fortran (Julia) memory layouts, as well as compression. The YAML metadata can contain arbitrary information corresponding to scalars, arrays, or dictionaries. diff --git a/src/ASDF.jl b/src/ASDF.jl index 9435f39..b19c8aa 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -13,6 +13,7 @@ using YAML: YAML using OrderedCollections: OrderedDict using FileIO: @format_str, File, load, save using AbstractTrees: AbstractTrees +using Dates: Date, DateTime export load, save @@ -1168,27 +1169,24 @@ returned node while writing a file. The fallback returns `value` unchanged. to_tree(value) = value function _convert_tree(value, active = Base.IdSet{Any}()) + # Plain scalars have neither hooks nor children, so they skip the cycle guard. + value isa Union{Nothing, Bool, Int, Float64, String} && return _convert_tree_children(value, active) value in active && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) + length(active) < 1000 || throw(ArgumentError("ASDF write conversion is nested more than 1000 levels deep at $(typeof(value)); check for an ASDF.to_tree method that never returns a supported node")) push!(active, value) try converted = to_tree(value) - if converted === value - which(to_tree, (typeof(value),)) === which(to_tree, (Any,)) && return _convert_tree_children(value, active) - throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) - end - typeof(converted) === typeof(value) && throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) - return _convert_tree(converted, active) + return converted === value ? _convert_tree_children(value, active) : _convert_tree(converted, active) finally delete!(active, value) end end -_convert_tree_children(value::Nothing, active) = value -_convert_tree_children(value::Union{Bool,Integer,AbstractString}, active) = value +_convert_tree_children(value::Union{Nothing, Integer, AbstractString, Date, DateTime}, active) = value _convert_tree_children(value::AbstractFloat, active) = YAMLScalar(yaml_float_string(value)) +_convert_tree_children(value::Symbol, active) = String(value) _convert_tree_children(value::NDArray, active) = value -_convert_tree_children(value::TaggedScalar, active) = TaggedScalar(value.tag, _convert_tree(value.value, active)) -_convert_tree_children(value::NamedTuple, active) = OrderedDict{Any,Any}(String(key) => _convert_tree(item, active) for (key, item) in pairs(value)) +_convert_tree_children(value::NamedTuple, active) = OrderedDict{Any, Any}(String(key) => _convert_tree(item, active) for (key, item) in pairs(value)) _convert_tree_children(value::Tuple, active) = [_convert_tree(item, active) for item in value] function _convert_tree_children(value::TaggedMapping, active) @@ -1200,9 +1198,10 @@ function _convert_tree_children(value::TaggedSequence, active) end function _convert_tree_children(value::AbstractDict, active) - converted = OrderedDict{Any,Any}() + converted = OrderedDict{Any, Any}() for (key, item) in value - key isa Union{Bool,Integer,AbstractString} || throw(ArgumentError("ASDF mapping key $(repr(key)) has unsupported type $(typeof(key)); keys must be booleans, integers, or strings")) + key isa Symbol && (key = String(key)) + key isa Union{Integer, AbstractString} || throw(ArgumentError("ASDF mapping key $(repr(key)) has unsupported type $(typeof(key)); keys must be booleans, integers, strings, or symbols")) converted[key] = _convert_tree(item, active) end return converted @@ -1213,7 +1212,7 @@ function _convert_tree_children(value::AbstractArray, active) throw(ArgumentError("ASDF metadata arrays must be vectors; wrap $(typeof(value)) in ASDF.NDArrayWrapper to write an N-dimensional array")) end function _convert_tree_children(value, active) - throw(ArgumentError("value of type $(typeof(value)) is not supported by the ASDF writer; define ASDF.to_tree(::$(typeof(value)))")) + throw(ArgumentError("value of type $(typeof(value)) is not supported by the ASDF writer; convert it to a supported value or, for a type you own, define ASDF.to_tree(::$(typeof(value)))")) end function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) @@ -1241,11 +1240,9 @@ struct YAMLScalar end YAML._print(io::IO, val::YAMLScalar, level::Int = 0, ignore_level::Bool = false) = println(io, val.text) -# Pre-write pass: recursively rewrite a document so every float is emitted in YAML-1.1-compliant form. +# Rewrite inline ndarray data so every float is emitted in YAML-1.1-compliant form. Document +# metadata gets the same treatment from `_convert_tree`. yaml_compliant(val::AbstractFloat) = YAMLScalar(yaml_float_string(val)) -yaml_compliant(val::TaggedMapping) = TaggedMapping(val.tag, yaml_compliant(val.value)) -yaml_compliant(val::TaggedSequence) = TaggedSequence(val.tag, yaml_compliant(val.value)) -yaml_compliant(val::AbstractDict) = OrderedDict{Any, Any}(k => yaml_compliant(v) for (k, v) in val) yaml_compliant(val::AbstractArray) = map(yaml_compliant, val) yaml_compliant(val) = val @@ -1372,6 +1369,10 @@ function Base.getindex(chunked_ndarray::ChunkedNDArray) return data::AbstractArray end +# Chunked output is not implemented, so a loaded chunked array is written back as one contiguous +# block, the same way `YAML._print(::NDArray)` writes a loaded plain array. +_convert_tree_children(value::ChunkedNDArray, active) = NDArrayWrapper(value[]; compression = C_None) + ################################################################################ """ diff --git a/test/Project.toml b/test/Project.toml index fc68153..b9d5654 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] ASDF = "686f71d1-807d-59a4-a860-28280ea06d7b" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" diff --git a/test/test-read_chunked.jl b/test/test-read_chunked.jl index 5818533..370af08 100644 --- a/test/test-read_chunked.jl +++ b/test/test-read_chunked.jl @@ -13,4 +13,11 @@ 13 23 33 43 14 24 34 44 ] + + # Chunked output is not implemented; a loaded chunked array is re-saved as one contiguous block. + mktempdir() do directory + filename = joinpath(directory, "chunking.asdf") + save(filename, asdf) + @test load(filename)["chunky"][] == chunky + end end diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl index 4f6299c..22a398c 100644 --- a/test/test-write-converters.jl +++ b/test/test-write-converters.jl @@ -1,3 +1,5 @@ +using Dates: Date, DateTime + struct WriteValue value::Int end @@ -11,17 +13,24 @@ struct WriteBlock data::Matrix{Float64} end struct WriteLoop end -struct WriteSame +mutable struct WriteSame value::Int end +struct WriteMapping <: AbstractDict{String, Any} + entries::OrderedDict{String, Any} +end +Base.iterate(m::WriteMapping, state...) = iterate(m.entries, state...) +Base.length(m::WriteMapping) = length(m.entries) +Base.getindex(m::WriteMapping, key) = m.entries[key] struct UnsupportedWriteValue end ASDF.to_tree(value::WriteValue) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) ASDF.to_tree(value::WriteParent) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) ASDF.to_tree(value::WriteAlias) = value.value -ASDF.to_tree(value::WriteBlock) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression=ASDF.C_Zlib))) +ASDF.to_tree(value::WriteBlock) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression = ASDF.C_Zlib))) ASDF.to_tree(value::WriteLoop) = OrderedDict("self" => value) ASDF.to_tree(value::WriteSame) = WriteSame(value.value) +ASDF.to_tree(value::WriteMapping) = ASDF.TaggedMapping("tag:example.org/write/mapping-1.0.0", value) @testset "write conversion protocol" begin shallow = ASDF.to_tree(WriteParent(WriteValue(3))) @@ -32,22 +41,30 @@ ASDF.to_tree(value::WriteSame) = WriteSame(value.value) @test converted["child"].tag == "tag:example.org/write/value-1.0.0" @test ASDF._convert_tree(WriteAlias(WriteValue(4)))["value"] == 4 - source = OrderedDict("tuple" => (WriteValue(5),), "named" => (child=WriteValue(6),)) + source = OrderedDict("tuple" => (WriteValue(5),), "named" => (child = WriteValue(6),)) tree = ASDF._convert_tree(source) @test tree["tuple"][1]["value"] == 5 @test tree["named"]["child"]["value"] == 6 @test source["tuple"][1] isa WriteValue - cyclic_mapping = OrderedDict{Any,Any}() + # A hook may tag its own mapping input without tripping the cycle guard. + tagged = ASDF._convert_tree(WriteMapping(OrderedDict{String, Any}("child" => WriteValue(7)))) + @test tagged.tag == "tag:example.org/write/mapping-1.0.0" + @test tagged["child"]["value"] == 7 + + # Symbols are written as strings, both as keys and as values. + @test ASDF._convert_tree(OrderedDict(:unit => :s)) == OrderedDict("unit" => "s") + + cyclic_mapping = OrderedDict{Any, Any}() cyclic_mapping["self"] = cyclic_mapping cyclic_sequence = Any[] push!(cyclic_sequence, cyclic_sequence) @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_mapping) @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_sequence) @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(WriteLoop()) - @test_throws "must return a supported ASDF tree node" ASDF._convert_tree(WriteSame(1)) + @test_throws "nested more than 1000 levels deep" ASDF._convert_tree(WriteSame(1)) @test_throws "not supported by the ASDF writer" ASDF._convert_tree(UnsupportedWriteValue()) - @test_throws "keys must be booleans, integers, or strings" ASDF._convert_tree(OrderedDict(WriteValue(1) => 2)) + @test_throws "keys must be booleans, integers, strings, or symbols" ASDF._convert_tree(OrderedDict(WriteValue(1) => 2)) @test_throws "wrap Matrix" ASDF._convert_tree([WriteValue(1) WriteValue(2)]) end @@ -59,14 +76,21 @@ end "data" => WriteBlock(data), ), "name" => "product", + :band => :F150W, + "observed" => Date(2024, 3, 5), + "history" => Any[OrderedDict("time" => DateTime(2024, 3, 5, 6, 7, 8), "note" => nothing)], ) mktempdir() do directory filename = joinpath(directory, "custom.asdf") ASDF.write_file(filename, document) - loaded = ASDF.load_file(filename; extensions=true) + loaded = ASDF.load_file(filename; extensions = true) - @test collect(keys(loaded.metadata)) == ["roman", "name", "asdf_library"] + @test collect(keys(loaded.metadata)) == ["roman", "name", "band", "observed", "history", "asdf_library"] + @test loaded["band"] == "F150W" + @test loaded["observed"] == Date(2024, 3, 5) + @test loaded["history"][1]["time"] == DateTime(2024, 3, 5, 6, 7, 8) + @test loaded["history"][1]["note"] === nothing @test loaded["roman"]["meta"]["model"]["child"]["value"] == 9 @test loaded["roman"]["data"]["data"][] == data @test !haskey(document, "asdf_library") From 9f32305355127e7b5aeaa94cddd71baac45b146e Mon Sep 17 00:00:00 2001 From: cgarling Date: Fri, 23 Oct 2026 09:54:42 -0400 Subject: [PATCH 4/4] Reject colliding keys after symbol normalization Symbol keys are written as strings, so a mapping containing both `:a` and `"a"` produced one YAML key and silently dropped one of the values. Raise an ArgumentError instead. --- src/ASDF.jl | 2 ++ test/test-write-converters.jl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/ASDF.jl b/src/ASDF.jl index b19c8aa..d30ac60 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -1202,6 +1202,8 @@ function _convert_tree_children(value::AbstractDict, active) for (key, item) in value key isa Symbol && (key = String(key)) key isa Union{Integer, AbstractString} || throw(ArgumentError("ASDF mapping key $(repr(key)) has unsupported type $(typeof(key)); keys must be booleans, integers, strings, or symbols")) + # `:a` and `"a"` are distinct Julia keys but the same YAML key. + haskey(converted, key) && throw(ArgumentError("ASDF mapping key $(repr(key)) occurs more than once after key normalization")) converted[key] = _convert_tree(item, active) end return converted diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl index 22a398c..2857a15 100644 --- a/test/test-write-converters.jl +++ b/test/test-write-converters.jl @@ -54,6 +54,8 @@ ASDF.to_tree(value::WriteMapping) = ASDF.TaggedMapping("tag:example.org/write/ma # Symbols are written as strings, both as keys and as values. @test ASDF._convert_tree(OrderedDict(:unit => :s)) == OrderedDict("unit" => "s") + # Keys that collide after normalization must not silently drop a value. + @test_throws "occurs more than once" ASDF._convert_tree(OrderedDict{Any, Any}(:a => 1, "a" => 2)) cyclic_mapping = OrderedDict{Any, Any}() cyclic_mapping["self"] = cyclic_mapping