diff --git a/Project.toml b/Project.toml index 3f12623..c1fe20e 100644 --- a/Project.toml +++ b/Project.toml @@ -14,6 +14,7 @@ ChunkCodecLibLz4 = "7e9cc85e-5614-42a3-ad86-b78f920b38a5" ChunkCodecLibLzma = "e95d29e5-19c5-4afd-ae0f-beb790efacdf" ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -29,6 +30,7 @@ ChunkCodecLibLz4 = "0.2.1, 1.0" ChunkCodecLibLzma = "1.0" ChunkCodecLibZlib = "0.2.0, 1.0" ChunkCodecLibZstd = "0.2.0, 1.0" +Dates = "1" FileIO = "1" MD5 = "0.2.1" OrderedCollections = "1.8.1, 2" diff --git a/README.md b/README.md index 1aeed6a..b57c79e 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs.io/en/latest/index.html) package, written in Julia. +Packages can extend `ASDF.to_tree` to embed their own Julia objects anywhere in +a larger ASDF document alongside ordinary metadata and binary arrays. See the +[custom type documentation](https://juliaastro.org/ASDF.jl/dev/custom-types/) +for the write-conversion interface. + ## Quickstart ```julia diff --git a/docs/make.jl b/docs/make.jl index f739fbb..283669d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,6 +20,7 @@ makedocs(; "JWST" => "examples/jwst.md", "Roman" => "examples/roman.md", ], + "Custom Julia types" => "custom-types.md", "Interoperability" => "interop.md", "API" => "api.md", ], diff --git a/docs/src/custom-types.md b/docs/src/custom-types.md new file mode 100644 index 0000000..9bdb7da --- /dev/null +++ b/docs/src/custom-types.md @@ -0,0 +1,88 @@ +# Custom Julia types + +ASDF documents often combine metadata, binary arrays, and objects owned by +domain packages. A package can make its types writable anywhere in an ASDF +document by extending [`ASDF.to_tree`](@ref): + +```@example custom_types +using ASDF +using OrderedCollections + +struct Measurement + value::Float64 + unit::String +end + +function ASDF.to_tree(measurement::Measurement) + properties = OrderedDict("value" => measurement.value, "unit" => measurement.unit) + return ASDF.TaggedMapping("tag:example.org/measurement-1.0.0", properties) +end + +document = OrderedDict( + "meta" => OrderedDict("exposure" => Measurement(1200.0, "s")), + "data" => ASDF.NDArrayWrapper(reshape(collect(1.0:12.0), 3, 4)), +) + +save("custom-types.asdf", document) +``` + +`save` and [`ASDF.write_file`](@ref) recursively walk the complete document. +When they encounter a `Measurement`, Julia dispatch selects the method above. +ASDF then recursively converts custom objects contained in the returned node +before writing YAML and binary blocks. + +The original document is not modified. Calling the hook directly inspects its +shallow representation: + +```@example custom_types +node = ASDF.to_tree(Measurement(5.0, "m")) +node.tag +``` + +## Conversion contract + +Packages extend the one-argument `ASDF.to_tree(value)` hook. The fallback +returns `value` unchanged. + +A package method should return one of: + +- `nothing`, a boolean, integer, float, string, symbol, `Date`, or `DateTime`; +- a mapping with boolean, integer, string, or symbol keys, a vector, tuple, or + named tuple; +- [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), or + [`ASDF.TaggedScalar`](@ref); +- [`ASDF.NDArrayWrapper`](@ref) for explicit inline or binary array storage. + +Converter methods are shallow. They may return mappings or sequences containing +other custom objects; the writer converts those children automatically and +redispatches when a converter delegates to another custom type. Converters +should not call `to_tree` recursively themselves. + +Symbols are written as strings. Unsupported leaves and mapping keys produce an +error instead of being silently stringified. Multidimensional arrays must be +wrapped in `NDArrayWrapper`; metadata sequences are vectors. ASDF.jl rejects +cyclic mappings, sequences, or converter output because ASDF reference +serialization is not yet implemented. + +## Optional ASDF support + +When ASDF.jl is an optional dependency, define the method in a Julia package +extension that loads only when both packages are present: + +```julia +module MyPackageASDFExt + +using ASDF +using MyPackage + +function ASDF.to_tree(value::MyPackage.CustomType) + return ASDF.TaggedMapping("tag:example.org/custom-1.0.0", Dict("value" => value.value)) +end + +end +``` + +This interface only controls writing. Loading an unknown tag with +`extensions = true` produces an `ASDF.TaggedMapping`, `ASDF.TaggedSequence`, or +`ASDF.TaggedScalar`; reconstructing package-owned objects and validating their +schemas require separate read-side support. diff --git a/docs/src/index.md b/docs/src/index.md index cc5a031..1c0e3c8 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -4,7 +4,7 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs ## Introduction -The ASDF file format is based on the human-readable [YAML](http://yaml.org/) standard, extended with efficient binary blocks to store array data. Basic arithmetic types (`Bool`, `Int`, `Float`, `Complex`) and `String` types are supported out of the box. Other types (structures) need to be declared to be supported. +The ASDF file format is based on the human-readable [YAML](http://yaml.org/) standard, extended with efficient binary blocks to store array data. Basic arithmetic types (`Bool`, `Int`, `Float`) and `String` types are supported out of the box, as are arrays of real and complex numbers. Other types (structures) need to be declared to be supported. ASDF supports arbitrary array strides, both C (Python) and Fortran (Julia) memory layouts, as well as compression. The YAML metadata can contain arbitrary information corresponding to scalars, arrays, or dictionaries. @@ -125,4 +125,7 @@ af["data"][] == [1, 2, 3, 4] ## Tagged objects -Come back soon to see how custom Julia objects can be handled in ASDF.jl. +Packages can extend [`ASDF.to_tree`](@ref) to serialize their own Julia types +wherever they occur in a larger ASDF document. See [Custom Julia +types](@ref) for the conversion contract and an example combining a custom +metadata object with a binary array. diff --git a/src/ASDF.jl b/src/ASDF.jl index 0f37cdb..d30ac60 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -13,6 +13,7 @@ using YAML: YAML using OrderedCollections: OrderedDict using FileIO: @format_str, File, load, save using AbstractTrees: AbstractTrees +using Dates: Date, DateTime export load, save @@ -1067,6 +1068,7 @@ but otherwise behave as plain mappings; and, when loading with `extensions = tru extension tags that would otherwise raise an error. Each behaves exactly like its wrapped `value`. A mapping indexes and iterates as a dict, a sequence as a vector, a scalar as a string, while retaining the original `tag` so the node round-trips unchanged through [`ASDF.write_file`](@ref). +The `tag` field must contain the full tag URI; the writer emits namespace shorthand where applicable. """ struct TaggedMapping{D <: AbstractDict} <: AbstractDict{Any, Any} tag::String @@ -1153,6 +1155,68 @@ function YAML._print(io::IO, val::TaggedScalar, level::Int = 0, ignore_level::Bo return YAML._print(io, val.value, level, ignore_level) end +""" + ASDF.to_tree(value) + +Convert one package-owned value to an ASDF-compatible tree node. + +Packages extend this shallow hook for their own types. A method must return a +supported scalar, mapping, vector, [`ASDF.TaggedMapping`](@ref), +[`ASDF.TaggedSequence`](@ref), [`ASDF.TaggedScalar`](@ref), or +[`ASDF.NDArrayWrapper`](@ref). ASDF recursively converts values nested in the +returned node while writing a file. The fallback returns `value` unchanged. +""" +to_tree(value) = value + +function _convert_tree(value, active = Base.IdSet{Any}()) + # Plain scalars have neither hooks nor children, so they skip the cycle guard. + value isa Union{Nothing, Bool, Int, Float64, String} && return _convert_tree_children(value, active) + value in active && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) + length(active) < 1000 || throw(ArgumentError("ASDF write conversion is nested more than 1000 levels deep at $(typeof(value)); check for an ASDF.to_tree method that never returns a supported node")) + push!(active, value) + try + converted = to_tree(value) + return converted === value ? _convert_tree_children(value, active) : _convert_tree(converted, active) + finally + delete!(active, value) + end +end + +_convert_tree_children(value::Union{Nothing, Integer, AbstractString, Date, DateTime}, active) = value +_convert_tree_children(value::AbstractFloat, active) = YAMLScalar(yaml_float_string(value)) +_convert_tree_children(value::Symbol, active) = String(value) +_convert_tree_children(value::NDArray, active) = value +_convert_tree_children(value::NamedTuple, active) = OrderedDict{Any, Any}(String(key) => _convert_tree(item, active) for (key, item) in pairs(value)) +_convert_tree_children(value::Tuple, active) = [_convert_tree(item, active) for item in value] + +function _convert_tree_children(value::TaggedMapping, active) + return TaggedMapping(value.tag, _convert_tree_children(value.value, active)) +end + +function _convert_tree_children(value::TaggedSequence, active) + return TaggedSequence(value.tag, _convert_tree_children(value.value, active)) +end + +function _convert_tree_children(value::AbstractDict, active) + converted = OrderedDict{Any, Any}() + for (key, item) in value + key isa Symbol && (key = String(key)) + key isa Union{Integer, AbstractString} || throw(ArgumentError("ASDF mapping key $(repr(key)) has unsupported type $(typeof(key)); keys must be booleans, integers, strings, or symbols")) + # `:a` and `"a"` are distinct Julia keys but the same YAML key. + haskey(converted, key) && throw(ArgumentError("ASDF mapping key $(repr(key)) occurs more than once after key normalization")) + converted[key] = _convert_tree(item, active) + end + return converted +end + +_convert_tree_children(value::AbstractVector, active) = [_convert_tree(item, active) for item in value] +function _convert_tree_children(value::AbstractArray, active) + throw(ArgumentError("ASDF metadata arrays must be vectors; wrap $(typeof(value)) in ASDF.NDArrayWrapper to write an N-dimensional array")) +end +function _convert_tree_children(value, active) + throw(ArgumentError("value of type $(typeof(value)) is not supported by the ASDF writer; convert it to a supported value or, for a type you own, define ASDF.to_tree(::$(typeof(value)))")) +end + function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) # TODO: Get compression from underlying header block? return YAML._print(io, NDArrayWrapper(val[]; compression = C_None), level, ignore_level) @@ -1178,11 +1242,9 @@ struct YAMLScalar end YAML._print(io::IO, val::YAMLScalar, level::Int = 0, ignore_level::Bool = false) = println(io, val.text) -# Pre-write pass: recursively rewrite a document so every float is emitted in YAML-1.1-compliant form. +# Rewrite inline ndarray data so every float is emitted in YAML-1.1-compliant form. Document +# metadata gets the same treatment from `_convert_tree`. yaml_compliant(val::AbstractFloat) = YAMLScalar(yaml_float_string(val)) -yaml_compliant(val::TaggedMapping) = TaggedMapping(val.tag, yaml_compliant(val.value)) -yaml_compliant(val::TaggedSequence) = TaggedSequence(val.tag, yaml_compliant(val.value)) -yaml_compliant(val::AbstractDict) = OrderedDict{Any, Any}(k => yaml_compliant(v) for (k, v) in val) yaml_compliant(val::AbstractArray) = map(yaml_compliant, val) yaml_compliant(val) = val @@ -1309,6 +1371,10 @@ function Base.getindex(chunked_ndarray::ChunkedNDArray) return data::AbstractArray end +# Chunked output is not implemented, so a loaded chunked array is written back as one contiguous +# block, the same way `YAML._print(::NDArray)` writes a loaded plain array. +_convert_tree_children(value::ChunkedNDArray, active) = NDArrayWrapper(value[]; compression = C_None) + ################################################################################ """ @@ -1587,6 +1653,7 @@ function YAML._print(io::IO, val::ASDFLibrary, level::Int = 0, ignore_level::Boo library = OrderedDict(:name => val.name, :author => val.author, :homepage => val.homepage, :version => val.version) return YAML._print(io, library, level, ignore_level) end +_convert_tree_children(value::ASDFLibrary, active) = value """ NDArrayWrapper @@ -1612,6 +1679,7 @@ function NDArrayWrapper(array::AbstractArray; compression::Compression = C_Bzip2 return NDArrayWrapper(array, compression, inline, lz4_layout) end Base.getindex(val::NDArrayWrapper) = val.array +_convert_tree_children(value::NDArrayWrapper, active) = value """ Blocks @@ -1756,7 +1824,10 @@ end """ write_file(filename::AbstractString, document::AbstractDict) -Writes an ASDF file to disk. `document` is a plain `Dict` whose values may include [`NDArrayWrapper`](@ref) instances. These are serialized as binary blocks with appropriate compression. +Writes an ASDF file to disk. `document` may contain custom Julia objects with +[`ASDF.to_tree`](@ref) methods, including objects nested inside mappings or +vectors. Values may also include [`NDArrayWrapper`](@ref) instances, which are +serialized as binary blocks with appropriate compression. Layout of the output file: @@ -1783,8 +1854,8 @@ function write_file(filename::AbstractString, document::AbstractDict) # back to an unordered `Dict` and drop the order). The provenance entry is stamped last. full_document = OrderedDict{Any, Any}(document) full_document["asdf_library"] = library - # Rewrite floats so their exponents are YAML-1.1 compliant (see `yaml_compliant`). - full_document = yaml_compliant(full_document) + # Convert package-owned objects and normalize floats before block collection. + full_document = _convert_tree(full_document) # Write YAML part of file io = open(filename, "w") diff --git a/test/Project.toml b/test/Project.toml index fc68153..b9d5654 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] ASDF = "686f71d1-807d-59a4-a860-28280ea06d7b" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" diff --git a/test/test-read_chunked.jl b/test/test-read_chunked.jl index 5818533..370af08 100644 --- a/test/test-read_chunked.jl +++ b/test/test-read_chunked.jl @@ -13,4 +13,11 @@ 13 23 33 43 14 24 34 44 ] + + # Chunked output is not implemented; a loaded chunked array is re-saved as one contiguous block. + mktempdir() do directory + filename = joinpath(directory, "chunking.asdf") + save(filename, asdf) + @test load(filename)["chunky"][] == chunky + end end diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl new file mode 100644 index 0000000..2857a15 --- /dev/null +++ b/test/test-write-converters.jl @@ -0,0 +1,102 @@ +using Dates: Date, DateTime + +struct WriteValue + value::Int +end +struct WriteParent + child +end +struct WriteAlias + value::WriteValue +end +struct WriteBlock + data::Matrix{Float64} +end +struct WriteLoop end +mutable struct WriteSame + value::Int +end +struct WriteMapping <: AbstractDict{String, Any} + entries::OrderedDict{String, Any} +end +Base.iterate(m::WriteMapping, state...) = iterate(m.entries, state...) +Base.length(m::WriteMapping) = length(m.entries) +Base.getindex(m::WriteMapping, key) = m.entries[key] +struct UnsupportedWriteValue end + +ASDF.to_tree(value::WriteValue) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::WriteParent) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) +ASDF.to_tree(value::WriteAlias) = value.value +ASDF.to_tree(value::WriteBlock) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression = ASDF.C_Zlib))) +ASDF.to_tree(value::WriteLoop) = OrderedDict("self" => value) +ASDF.to_tree(value::WriteSame) = WriteSame(value.value) +ASDF.to_tree(value::WriteMapping) = ASDF.TaggedMapping("tag:example.org/write/mapping-1.0.0", value) + +@testset "write conversion protocol" begin + shallow = ASDF.to_tree(WriteParent(WriteValue(3))) + @test shallow["child"] isa WriteValue + + converted = ASDF._convert_tree(WriteParent(WriteValue(3))) + @test converted.tag == "tag:example.org/write/parent-1.0.0" + @test converted["child"].tag == "tag:example.org/write/value-1.0.0" + @test ASDF._convert_tree(WriteAlias(WriteValue(4)))["value"] == 4 + + source = OrderedDict("tuple" => (WriteValue(5),), "named" => (child = WriteValue(6),)) + tree = ASDF._convert_tree(source) + @test tree["tuple"][1]["value"] == 5 + @test tree["named"]["child"]["value"] == 6 + @test source["tuple"][1] isa WriteValue + + # A hook may tag its own mapping input without tripping the cycle guard. + tagged = ASDF._convert_tree(WriteMapping(OrderedDict{String, Any}("child" => WriteValue(7)))) + @test tagged.tag == "tag:example.org/write/mapping-1.0.0" + @test tagged["child"]["value"] == 7 + + # Symbols are written as strings, both as keys and as values. + @test ASDF._convert_tree(OrderedDict(:unit => :s)) == OrderedDict("unit" => "s") + # Keys that collide after normalization must not silently drop a value. + @test_throws "occurs more than once" ASDF._convert_tree(OrderedDict{Any, Any}(:a => 1, "a" => 2)) + + cyclic_mapping = OrderedDict{Any, Any}() + cyclic_mapping["self"] = cyclic_mapping + cyclic_sequence = Any[] + push!(cyclic_sequence, cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_mapping) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(WriteLoop()) + @test_throws "nested more than 1000 levels deep" ASDF._convert_tree(WriteSame(1)) + @test_throws "not supported by the ASDF writer" ASDF._convert_tree(UnsupportedWriteValue()) + @test_throws "keys must be booleans, integers, strings, or symbols" ASDF._convert_tree(OrderedDict(WriteValue(1) => 2)) + @test_throws "wrap Matrix" ASDF._convert_tree([WriteValue(1) WriteValue(2)]) +end + +@testset "heterogeneous document writing" begin + data = reshape(collect(1.0:12.0), 3, 4) + document = OrderedDict( + "roman" => OrderedDict( + "meta" => OrderedDict("model" => WriteParent(WriteValue(9)), "description" => "nested"), + "data" => WriteBlock(data), + ), + "name" => "product", + :band => :F150W, + "observed" => Date(2024, 3, 5), + "history" => Any[OrderedDict("time" => DateTime(2024, 3, 5, 6, 7, 8), "note" => nothing)], + ) + + mktempdir() do directory + filename = joinpath(directory, "custom.asdf") + ASDF.write_file(filename, document) + loaded = ASDF.load_file(filename; extensions = true) + + @test collect(keys(loaded.metadata)) == ["roman", "name", "band", "observed", "history", "asdf_library"] + @test loaded["band"] == "F150W" + @test loaded["observed"] == Date(2024, 3, 5) + @test loaded["history"][1]["time"] == DateTime(2024, 3, 5, 6, 7, 8) + @test loaded["history"][1]["note"] === nothing + @test loaded["roman"]["meta"]["model"]["child"]["value"] == 9 + @test loaded["roman"]["data"]["data"][] == data + @test !haskey(document, "asdf_library") + @test document["roman"]["meta"]["model"] isa WriteParent + @test document["roman"]["data"] isa WriteBlock + end +end