From 46c4bdec922c5adff8c894fc85f74626f5fd84ff Mon Sep 17 00:00:00 2001 From: cgarling Date: Wed, 2 Sep 2026 10:25:21 -0400 Subject: [PATCH 1/2] Add `to_tree` for extendable type writing Adds a write-conversion protocol for packages to extend allowing arbitrary Julia types to be embedded into ASDF documents. --- README.md | 5 ++ docs/make.jl | 1 + docs/src/api.md | 11 ++++ docs/src/custom-types.md | 89 +++++++++++++++++++++++++ docs/src/index.md | 5 +- src/ASDF.jl | 87 +++++++++++++++++++++++- test/test-write-converters.jl | 120 ++++++++++++++++++++++++++++++++++ 7 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 docs/src/custom-types.md create mode 100644 test/test-write-converters.jl diff --git a/README.md b/README.md index 1aeed6a..b57c79e 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs.io/en/latest/index.html) package, written in Julia. +Packages can extend `ASDF.to_tree` to embed their own Julia objects anywhere in +a larger ASDF document alongside ordinary metadata and binary arrays. See the +[custom type documentation](https://juliaastro.org/ASDF.jl/dev/custom-types/) +for the write-conversion interface. + ## Quickstart ```julia diff --git a/docs/make.jl b/docs/make.jl index f739fbb..283669d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,6 +20,7 @@ makedocs(; "JWST" => "examples/jwst.md", "Roman" => "examples/roman.md", ], + "Custom Julia types" => "custom-types.md", "Interoperability" => "interop.md", "API" => "api.md", ], diff --git a/docs/src/api.md b/docs/src/api.md index 93b75ca..47551a7 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -7,8 +7,19 @@ Modules = [ASDF] Private = false ``` +## Custom type writing + +```@docs +ASDF.WriteContext +ASDF.to_tree +ASDF.TaggedMapping +ASDF.TaggedSequence +ASDF.TaggedScalar +``` + ## Private ```@autodocs Modules = [ASDF] Public = false +Filter = value -> value ∉ (ASDF.WriteContext, ASDF.to_tree, ASDF.TaggedMapping, ASDF.TaggedSequence, ASDF.TaggedScalar) ``` diff --git a/docs/src/custom-types.md b/docs/src/custom-types.md new file mode 100644 index 0000000..4e644c8 --- /dev/null +++ b/docs/src/custom-types.md @@ -0,0 +1,89 @@ +# Custom Julia types + +ASDF documents often combine metadata, binary arrays, and objects owned by +domain packages. A package can make its types writable anywhere in an ASDF +document by extending [`ASDF.to_tree`](@ref): + +```@example custom_types +using ASDF + +struct Measurement + value::Float64 + unit::String +end + +function ASDF.to_tree(measurement::Measurement, context::ASDF.WriteContext) + properties = Dict("value" => measurement.value, "unit" => measurement.unit) + return ASDF.TaggedMapping("tag:example.org/measurement-1.0.0", properties) +end + +document = Dict( + "meta" => Dict("exposure" => Measurement(1200.0, "s")), + "data" => ASDF.NDArrayWrapper(reshape(collect(1.0:12.0), 3, 4)), +) + +save("custom-types.asdf", document) +``` + +`save` and [`ASDF.write_file`](@ref) recursively walk the complete document. +When they encounter a `Measurement`, Julia dispatch selects the method above. +ASDF then recursively converts custom objects contained in the returned node +before writing YAML and binary blocks. + +The original document is not modified. The same conversion can be inspected +without writing a file: + +```@example custom_types +node = ASDF.to_tree(Measurement(5.0, "m")) +node.tag +``` + +## Conversion contract + +The public interface has two forms: + +```julia +ASDF.to_tree(value) +ASDF.to_tree(value, context::ASDF.WriteContext) +``` + +The one-argument form recursively converts a value and is useful for inspecting +the ASDF representation. The two-argument form is the extension hook. Its +fallback returns the value unchanged. + +A package method should return one of: + +- a scalar, string, mapping, or array already supported by ASDF.jl; +- [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), or + [`ASDF.TaggedScalar`](@ref); +- [`ASDF.NDArrayWrapper`](@ref) for explicit inline or binary array storage. + +Converter methods are shallow. They may return mappings or sequences containing +other custom objects; ASDF.jl converts those children automatically. Methods +should accept the [`ASDF.WriteContext`](@ref) but treat it as opaque. + +ASDF.jl rejects cyclic mappings, arrays, or converter output because ASDF +reference serialization is not yet implemented. + +## Optional ASDF support + +When ASDF.jl is an optional dependency, define the method in a Julia package +extension that loads only when both packages are present: + +```julia +module MyPackageASDFExt + +using ASDF +using MyPackage + +function ASDF.to_tree(value::MyPackage.CustomType, context::ASDF.WriteContext) + return ASDF.TaggedMapping("tag:example.org/custom-1.0.0", Dict("value" => value.value)) +end + +end +``` + +This interface only controls writing. Loading an unknown tag with +`extensions = true` produces an `ASDF.TaggedMapping`, `ASDF.TaggedSequence`, or +`ASDF.TaggedScalar`; reconstructing package-owned objects and validating their +schemas require separate read-side support. diff --git a/docs/src/index.md b/docs/src/index.md index 3ad5c79..fbae546 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -125,4 +125,7 @@ af["data"][] == [1, 2, 3, 4] ## Tagged objects -Come back soon to see how custom Julia objects can be handled in ASDF.jl. +Packages can extend [`ASDF.to_tree`](@ref) to serialize their own Julia types +wherever they occur in a larger ASDF document. See [Custom Julia +types](@ref) for the conversion contract and an example combining a custom +metadata object with a binary array. diff --git a/src/ASDF.jl b/src/ASDF.jl index 8b68824..e292d0a 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -1153,6 +1153,86 @@ function YAML._print(io::IO, val::TaggedScalar, level::Int = 0, ignore_level::Bo return YAML._print(io, val.value, level, ignore_level) end +""" + WriteContext + +Opaque context for converting Julia objects to ASDF-compatible tree nodes. + +Packages that extend [`ASDF.to_tree`](@ref) should accept this context but must +not depend on its fields. Future versions may use it for schema selection, +array-storage policies, and extension provenance. +""" +struct WriteContext + _active::IdDict{Any, Nothing} +end +WriteContext() = WriteContext(IdDict{Any, Nothing}()) + +""" + ASDF.to_tree(value) + ASDF.to_tree(value, context::ASDF.WriteContext) + +Convert Julia objects to an ASDF-compatible tree. + +The one-argument form recursively converts `value` and any custom objects +nested in mappings, arrays, or tagged nodes. Packages add support for their own +types by defining a two-argument method that returns a scalar, mapping, +sequence, [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), +[`ASDF.TaggedScalar`](@ref), [`ASDF.NDArrayWrapper`](@ref), or another value +already supported by the ASDF writer. Returned mappings and sequences may +contain additional custom objects; ASDF converts those children recursively. + +The fallback two-argument method returns `value` unchanged. +""" +to_tree(value, ::WriteContext) = value +to_tree(value) = _convert_tree(value, WriteContext()) + +function _with_active(f, value, context::WriteContext) + haskey(context._active, value) && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) + context._active[value] = nothing + try + return f() + finally + delete!(context._active, value) + end +end + +function _convert_tree(value, context::WriteContext) + converted = to_tree(value, context) + converted === value && return _convert_tree_children(converted, context) + return _with_active(value, context) do + _convert_tree_children(converted, context) + end +end + +_convert_tree_children(value, context::WriteContext) = value +_convert_tree_children(value::TaggedScalar, context::WriteContext) = value + +function _convert_tree_children(value::TaggedMapping, context::WriteContext) + converted = _with_active(value.value, context) do + OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) + end + return TaggedMapping(value.tag, converted) +end + +function _convert_tree_children(value::TaggedSequence, context::WriteContext) + converted = _with_active(value.value, context) do + map(item -> _convert_tree(item, context), value.value) + end + return TaggedSequence(value.tag, converted) +end + +function _convert_tree_children(value::AbstractDict, context::WriteContext) + return _with_active(value, context) do + OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) + end +end + +function _convert_tree_children(value::AbstractArray, context::WriteContext) + return _with_active(value, context) do + map(item -> _convert_tree(item, context), value) + end +end + function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) # TODO: Get compression from underlying header block? return YAML._print(io, NDArrayWrapper(val[]; compression = C_None), level, ignore_level) @@ -1756,7 +1836,10 @@ end """ write_file(filename::AbstractString, document::AbstractDict) -Writes an ASDF file to disk. `document` is a plain `Dict` whose values may include [`NDArrayWrapper`](@ref) instances. These are serialized as binary blocks with appropriate compression. +Writes an ASDF file to disk. `document` may contain custom Julia objects with +two-argument [`ASDF.to_tree`](@ref) methods, including objects nested inside +mappings or arrays. Values may also include [`NDArrayWrapper`](@ref) instances, +which are serialized as binary blocks with appropriate compression. Layout of the output file: @@ -1783,6 +1866,8 @@ function write_file(filename::AbstractString, document::AbstractDict) # back to an unordered `Dict` and drop the order). The provenance entry is stamped last. full_document = OrderedDict{Any, Any}(document) full_document["asdf_library"] = library + # Convert package-owned objects before float normalization and block collection. + full_document = to_tree(full_document) # Rewrite floats so their exponents are YAML-1.1 compliant (see `yaml_compliant`). full_document = yaml_compliant(full_document) diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl new file mode 100644 index 0000000..9a4583c --- /dev/null +++ b/test/test-write-converters.jl @@ -0,0 +1,120 @@ +abstract type AbstractWriteValue end + +struct WriteValue <: AbstractWriteValue + value::Int +end + +struct SpecialWriteValue <: AbstractWriteValue + value::Int +end + +struct WriteParent + child +end + +struct WriteSequence + values::Vector{Any} +end + +struct WriteScalar + value::String +end + +struct WriteBlock + data::Matrix{Float64} +end + +struct CyclicWriteValue end + +ASDF.to_tree(value::AbstractWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::SpecialWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/special-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::WriteParent, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) +ASDF.to_tree(value::WriteSequence, context::ASDF.WriteContext) = ASDF.TaggedSequence("tag:example.org/write/sequence-1.0.0", value.values) +ASDF.to_tree(value::WriteScalar, context::ASDF.WriteContext) = ASDF.TaggedScalar("tag:example.org/write/scalar-1.0.0", value.value) +ASDF.to_tree(value::WriteBlock, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression = ASDF.C_Zlib))) +ASDF.to_tree(value::CyclicWriteValue, context::ASDF.WriteContext) = OrderedDict("self" => value) + +@testset "write conversion protocol" begin + shallow = ASDF.to_tree(WriteParent(WriteValue(3)), ASDF.WriteContext()) + @test shallow isa ASDF.TaggedMapping + @test shallow["child"] isa WriteValue + + parent = ASDF.to_tree(WriteParent(WriteValue(3))) + @test parent.tag == "tag:example.org/write/parent-1.0.0" + @test parent["child"] isa ASDF.TaggedMapping + @test parent["child"].tag == "tag:example.org/write/value-1.0.0" + @test parent["child"]["value"] == 3 + + sequence = ASDF.to_tree(WriteSequence(Any[WriteValue(4), "plain"])) + @test sequence isa ASDF.TaggedSequence + @test sequence.tag == "tag:example.org/write/sequence-1.0.0" + @test sequence[1] isa ASDF.TaggedMapping + @test sequence[2] == "plain" + + scalar = ASDF.to_tree(WriteScalar("converted")) + @test scalar isa ASDF.TaggedScalar + @test scalar.tag == "tag:example.org/write/scalar-1.0.0" + @test String(scalar) == "converted" + + @test ASDF.to_tree(WriteValue(5)).tag == "tag:example.org/write/value-1.0.0" + @test ASDF.to_tree(SpecialWriteValue(5)).tag == "tag:example.org/write/special-1.0.0" +end + +@testset "recursive write conversion" begin + tagged = ASDF.TaggedMapping("tag:example.org/write/existing-1.0.0", OrderedDict("child" => WriteValue(6))) + converted = ASDF.to_tree(tagged) + @test converted.tag == tagged.tag + @test converted["child"] isa ASDF.TaggedMapping + @test tagged["child"] isa WriteValue + + source = OrderedDict("zebra" => WriteValue(7), "apple" => Any[1, WriteValue(8)], "matrix" => [1 2; 3 4]) + plain = ASDF.to_tree(source) + @test collect(keys(plain)) == collect(keys(source)) + @test plain["zebra"] isa ASDF.TaggedMapping + @test plain["apple"][2] isa ASDF.TaggedMapping + @test plain["matrix"] == source["matrix"] + @test source["zebra"] isa WriteValue + @test source["apple"][2] isa WriteValue + + cyclic_mapping = OrderedDict{Any, Any}() + cyclic_mapping["self"] = cyclic_mapping + @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_mapping) + + cyclic_sequence = Any[] + push!(cyclic_sequence, cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF.to_tree(CyclicWriteValue()) +end + +@testset "heterogeneous document writing" begin + data = reshape(collect(1.0:12.0), 3, 4) + document = OrderedDict( + "roman" => OrderedDict( + "meta" => OrderedDict("model" => WriteParent(SpecialWriteValue(9)), "description" => "nested"), + "data" => WriteBlock(data), + ), + "name" => "product", + ) + + mktempdir() do directory + filename = joinpath(directory, "custom.asdf") + ASDF.write_file(filename, document) + loaded = ASDF.load_file(filename; extensions = true) + + @test collect(keys(loaded.metadata)) == ["roman", "name", "asdf_library"] + @test loaded["name"] == "product" + @test loaded["roman"]["meta"]["model"].tag == "tag:example.org/write/parent-1.0.0" + @test loaded["roman"]["meta"]["model"]["child"].tag == "tag:example.org/write/special-1.0.0" + @test loaded["roman"]["data"].tag == "tag:example.org/write/block-1.0.0" + @test loaded["roman"]["data"]["data"][] == data + @test !haskey(document, "asdf_library") + @test document["roman"]["meta"]["model"] isa WriteParent + @test document["roman"]["data"] isa WriteBlock + + save_filename = joinpath(directory, "custom-save.asdf") + save(save_filename, document) + saved = ASDF.load_file(save_filename; extensions = true) + @test saved["roman"]["data"]["data"][] == data + @test saved["roman"]["meta"]["model"]["child"]["value"] == 9 + end +end From 50985e66f1a7e9d58713b587c72a2c1888623f27 Mon Sep 17 00:00:00 2001 From: cgarling Date: Sun, 27 Sep 2026 15:40:28 -0400 Subject: [PATCH 2/2] Simplify extensible ASDF tree writing --- docs/src/api.md | 11 ---- docs/src/custom-types.md | 41 ++++++------ src/ASDF.jl | 115 +++++++++++++++------------------- test/test-write-converters.jl | 114 +++++++++++---------------------- 4 files changed, 103 insertions(+), 178 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 47551a7..93b75ca 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -7,19 +7,8 @@ Modules = [ASDF] Private = false ``` -## Custom type writing - -```@docs -ASDF.WriteContext -ASDF.to_tree -ASDF.TaggedMapping -ASDF.TaggedSequence -ASDF.TaggedScalar -``` - ## Private ```@autodocs Modules = [ASDF] Public = false -Filter = value -> value ∉ (ASDF.WriteContext, ASDF.to_tree, ASDF.TaggedMapping, ASDF.TaggedSequence, ASDF.TaggedScalar) ``` diff --git a/docs/src/custom-types.md b/docs/src/custom-types.md index 4e644c8..38de4a0 100644 --- a/docs/src/custom-types.md +++ b/docs/src/custom-types.md @@ -6,19 +6,20 @@ document by extending [`ASDF.to_tree`](@ref): ```@example custom_types using ASDF +using OrderedCollections struct Measurement value::Float64 unit::String end -function ASDF.to_tree(measurement::Measurement, context::ASDF.WriteContext) - properties = Dict("value" => measurement.value, "unit" => measurement.unit) +function ASDF.to_tree(measurement::Measurement) + properties = OrderedDict("value" => measurement.value, "unit" => measurement.unit) return ASDF.TaggedMapping("tag:example.org/measurement-1.0.0", properties) end -document = Dict( - "meta" => Dict("exposure" => Measurement(1200.0, "s")), +document = OrderedDict( + "meta" => OrderedDict("exposure" => Measurement(1200.0, "s")), "data" => ASDF.NDArrayWrapper(reshape(collect(1.0:12.0), 3, 4)), ) @@ -30,8 +31,8 @@ When they encounter a `Measurement`, Julia dispatch selects the method above. ASDF then recursively converts custom objects contained in the returned node before writing YAML and binary blocks. -The original document is not modified. The same conversion can be inspected -without writing a file: +The original document is not modified. Calling the hook directly inspects its +shallow representation: ```@example custom_types node = ASDF.to_tree(Measurement(5.0, "m")) @@ -40,30 +41,26 @@ node.tag ## Conversion contract -The public interface has two forms: - -```julia -ASDF.to_tree(value) -ASDF.to_tree(value, context::ASDF.WriteContext) -``` - -The one-argument form recursively converts a value and is useful for inspecting -the ASDF representation. The two-argument form is the extension hook. Its -fallback returns the value unchanged. +Packages extend the one-argument `ASDF.to_tree(value)` hook. The fallback +returns `value` unchanged. A package method should return one of: -- a scalar, string, mapping, or array already supported by ASDF.jl; +- `nothing`, a boolean, integer, float, or string; +- a mapping with boolean, integer, or string keys, a vector, tuple, or named tuple; - [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), or [`ASDF.TaggedScalar`](@ref); - [`ASDF.NDArrayWrapper`](@ref) for explicit inline or binary array storage. Converter methods are shallow. They may return mappings or sequences containing -other custom objects; ASDF.jl converts those children automatically. Methods -should accept the [`ASDF.WriteContext`](@ref) but treat it as opaque. +other custom objects; the writer converts those children automatically and +redispatches when a converter delegates to another custom type. Converters +should not call `to_tree` recursively themselves. -ASDF.jl rejects cyclic mappings, arrays, or converter output because ASDF -reference serialization is not yet implemented. +Unsupported leaves and mapping keys produce an error instead of being silently +stringified. Multidimensional arrays must be wrapped in `NDArrayWrapper`; +metadata sequences are vectors. ASDF.jl rejects cyclic mappings, sequences, or +converter output because ASDF reference serialization is not yet implemented. ## Optional ASDF support @@ -76,7 +73,7 @@ module MyPackageASDFExt using ASDF using MyPackage -function ASDF.to_tree(value::MyPackage.CustomType, context::ASDF.WriteContext) +function ASDF.to_tree(value::MyPackage.CustomType) return ASDF.TaggedMapping("tag:example.org/custom-1.0.0", Dict("value" => value.value)) end diff --git a/src/ASDF.jl b/src/ASDF.jl index 5fd10f3..9435f39 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -1067,6 +1067,7 @@ but otherwise behave as plain mappings; and, when loading with `extensions = tru extension tags that would otherwise raise an error. Each behaves exactly like its wrapped `value`. A mapping indexes and iterates as a dict, a sequence as a vector, a scalar as a string, while retaining the original `tag` so the node round-trips unchanged through [`ASDF.write_file`](@ref). +The `tag` field must contain the full tag URI; the writer emits namespace shorthand where applicable. """ struct TaggedMapping{D <: AbstractDict} <: AbstractDict{Any, Any} tag::String @@ -1153,84 +1154,66 @@ function YAML._print(io::IO, val::TaggedScalar, level::Int = 0, ignore_level::Bo return YAML._print(io, val.value, level, ignore_level) end -""" - WriteContext - -Opaque context for converting Julia objects to ASDF-compatible tree nodes. - -Packages that extend [`ASDF.to_tree`](@ref) should accept this context but must -not depend on its fields. Future versions may use it for schema selection, -array-storage policies, and extension provenance. -""" -struct WriteContext - _active::IdDict{Any, Nothing} -end -WriteContext() = WriteContext(IdDict{Any, Nothing}()) - """ ASDF.to_tree(value) - ASDF.to_tree(value, context::ASDF.WriteContext) -Convert Julia objects to an ASDF-compatible tree. +Convert one package-owned value to an ASDF-compatible tree node. -The one-argument form recursively converts `value` and any custom objects -nested in mappings, arrays, or tagged nodes. Packages add support for their own -types by defining a two-argument method that returns a scalar, mapping, -sequence, [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), -[`ASDF.TaggedScalar`](@ref), [`ASDF.NDArrayWrapper`](@ref), or another value -already supported by the ASDF writer. Returned mappings and sequences may -contain additional custom objects; ASDF converts those children recursively. - -The fallback two-argument method returns `value` unchanged. +Packages extend this shallow hook for their own types. A method must return a +supported scalar, mapping, vector, [`ASDF.TaggedMapping`](@ref), +[`ASDF.TaggedSequence`](@ref), [`ASDF.TaggedScalar`](@ref), or +[`ASDF.NDArrayWrapper`](@ref). ASDF recursively converts values nested in the +returned node while writing a file. The fallback returns `value` unchanged. """ -to_tree(value, ::WriteContext) = value -to_tree(value) = _convert_tree(value, WriteContext()) +to_tree(value) = value -function _with_active(f, value, context::WriteContext) - haskey(context._active, value) && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) - context._active[value] = nothing +function _convert_tree(value, active = Base.IdSet{Any}()) + value in active && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) + push!(active, value) try - return f() + converted = to_tree(value) + if converted === value + which(to_tree, (typeof(value),)) === which(to_tree, (Any,)) && return _convert_tree_children(value, active) + throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) + end + typeof(converted) === typeof(value) && throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) + return _convert_tree(converted, active) finally - delete!(context._active, value) + delete!(active, value) end end -function _convert_tree(value, context::WriteContext) - converted = to_tree(value, context) - converted === value && return _convert_tree_children(converted, context) - return _with_active(value, context) do - _convert_tree_children(converted, context) - end -end +_convert_tree_children(value::Nothing, active) = value +_convert_tree_children(value::Union{Bool,Integer,AbstractString}, active) = value +_convert_tree_children(value::AbstractFloat, active) = YAMLScalar(yaml_float_string(value)) +_convert_tree_children(value::NDArray, active) = value +_convert_tree_children(value::TaggedScalar, active) = TaggedScalar(value.tag, _convert_tree(value.value, active)) +_convert_tree_children(value::NamedTuple, active) = OrderedDict{Any,Any}(String(key) => _convert_tree(item, active) for (key, item) in pairs(value)) +_convert_tree_children(value::Tuple, active) = [_convert_tree(item, active) for item in value] -_convert_tree_children(value, context::WriteContext) = value -_convert_tree_children(value::TaggedScalar, context::WriteContext) = value - -function _convert_tree_children(value::TaggedMapping, context::WriteContext) - converted = _with_active(value.value, context) do - OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) - end - return TaggedMapping(value.tag, converted) +function _convert_tree_children(value::TaggedMapping, active) + return TaggedMapping(value.tag, _convert_tree_children(value.value, active)) end -function _convert_tree_children(value::TaggedSequence, context::WriteContext) - converted = _with_active(value.value, context) do - map(item -> _convert_tree(item, context), value.value) - end - return TaggedSequence(value.tag, converted) +function _convert_tree_children(value::TaggedSequence, active) + return TaggedSequence(value.tag, _convert_tree_children(value.value, active)) end -function _convert_tree_children(value::AbstractDict, context::WriteContext) - return _with_active(value, context) do - OrderedDict{Any, Any}(key => _convert_tree(item, context) for (key, item) in value) +function _convert_tree_children(value::AbstractDict, active) + converted = OrderedDict{Any,Any}() + for (key, item) in value + key isa Union{Bool,Integer,AbstractString} || throw(ArgumentError("ASDF mapping key $(repr(key)) has unsupported type $(typeof(key)); keys must be booleans, integers, or strings")) + converted[key] = _convert_tree(item, active) end + return converted end -function _convert_tree_children(value::AbstractArray, context::WriteContext) - return _with_active(value, context) do - map(item -> _convert_tree(item, context), value) - end +_convert_tree_children(value::AbstractVector, active) = [_convert_tree(item, active) for item in value] +function _convert_tree_children(value::AbstractArray, active) + throw(ArgumentError("ASDF metadata arrays must be vectors; wrap $(typeof(value)) in ASDF.NDArrayWrapper to write an N-dimensional array")) +end +function _convert_tree_children(value, active) + throw(ArgumentError("value of type $(typeof(value)) is not supported by the ASDF writer; define ASDF.to_tree(::$(typeof(value)))")) end function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) @@ -1667,6 +1650,7 @@ function YAML._print(io::IO, val::ASDFLibrary, level::Int = 0, ignore_level::Boo library = OrderedDict(:name => val.name, :author => val.author, :homepage => val.homepage, :version => val.version) return YAML._print(io, library, level, ignore_level) end +_convert_tree_children(value::ASDFLibrary, active) = value """ NDArrayWrapper @@ -1692,6 +1676,7 @@ function NDArrayWrapper(array::AbstractArray; compression::Compression = C_Bzip2 return NDArrayWrapper(array, compression, inline, lz4_layout) end Base.getindex(val::NDArrayWrapper) = val.array +_convert_tree_children(value::NDArrayWrapper, active) = value """ Blocks @@ -1837,9 +1822,9 @@ end write_file(filename::AbstractString, document::AbstractDict) Writes an ASDF file to disk. `document` may contain custom Julia objects with -two-argument [`ASDF.to_tree`](@ref) methods, including objects nested inside -mappings or arrays. Values may also include [`NDArrayWrapper`](@ref) instances, -which are serialized as binary blocks with appropriate compression. +[`ASDF.to_tree`](@ref) methods, including objects nested inside mappings or +vectors. Values may also include [`NDArrayWrapper`](@ref) instances, which are +serialized as binary blocks with appropriate compression. Layout of the output file: @@ -1866,10 +1851,8 @@ function write_file(filename::AbstractString, document::AbstractDict) # back to an unordered `Dict` and drop the order). The provenance entry is stamped last. full_document = OrderedDict{Any, Any}(document) full_document["asdf_library"] = library - # Convert package-owned objects before float normalization and block collection. - full_document = to_tree(full_document) - # Rewrite floats so their exponents are YAML-1.1 compliant (see `yaml_compliant`). - full_document = yaml_compliant(full_document) + # Convert package-owned objects and normalize floats before block collection. + full_document = _convert_tree(full_document) # Write YAML part of file io = open(filename, "w") diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl index 9a4583c..4f6299c 100644 --- a/test/test-write-converters.jl +++ b/test/test-write-converters.jl @@ -1,96 +1,61 @@ -abstract type AbstractWriteValue end - -struct WriteValue <: AbstractWriteValue +struct WriteValue value::Int end - -struct SpecialWriteValue <: AbstractWriteValue - value::Int -end - struct WriteParent child end - -struct WriteSequence - values::Vector{Any} -end - -struct WriteScalar - value::String +struct WriteAlias + value::WriteValue end - struct WriteBlock data::Matrix{Float64} end +struct WriteLoop end +struct WriteSame + value::Int +end +struct UnsupportedWriteValue end -struct CyclicWriteValue end - -ASDF.to_tree(value::AbstractWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) -ASDF.to_tree(value::SpecialWriteValue, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/special-1.0.0", OrderedDict("value" => value.value)) -ASDF.to_tree(value::WriteParent, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) -ASDF.to_tree(value::WriteSequence, context::ASDF.WriteContext) = ASDF.TaggedSequence("tag:example.org/write/sequence-1.0.0", value.values) -ASDF.to_tree(value::WriteScalar, context::ASDF.WriteContext) = ASDF.TaggedScalar("tag:example.org/write/scalar-1.0.0", value.value) -ASDF.to_tree(value::WriteBlock, context::ASDF.WriteContext) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression = ASDF.C_Zlib))) -ASDF.to_tree(value::CyclicWriteValue, context::ASDF.WriteContext) = OrderedDict("self" => value) +ASDF.to_tree(value::WriteValue) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::WriteParent) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) +ASDF.to_tree(value::WriteAlias) = value.value +ASDF.to_tree(value::WriteBlock) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression=ASDF.C_Zlib))) +ASDF.to_tree(value::WriteLoop) = OrderedDict("self" => value) +ASDF.to_tree(value::WriteSame) = WriteSame(value.value) @testset "write conversion protocol" begin - shallow = ASDF.to_tree(WriteParent(WriteValue(3)), ASDF.WriteContext()) - @test shallow isa ASDF.TaggedMapping + shallow = ASDF.to_tree(WriteParent(WriteValue(3))) @test shallow["child"] isa WriteValue - parent = ASDF.to_tree(WriteParent(WriteValue(3))) - @test parent.tag == "tag:example.org/write/parent-1.0.0" - @test parent["child"] isa ASDF.TaggedMapping - @test parent["child"].tag == "tag:example.org/write/value-1.0.0" - @test parent["child"]["value"] == 3 - - sequence = ASDF.to_tree(WriteSequence(Any[WriteValue(4), "plain"])) - @test sequence isa ASDF.TaggedSequence - @test sequence.tag == "tag:example.org/write/sequence-1.0.0" - @test sequence[1] isa ASDF.TaggedMapping - @test sequence[2] == "plain" + converted = ASDF._convert_tree(WriteParent(WriteValue(3))) + @test converted.tag == "tag:example.org/write/parent-1.0.0" + @test converted["child"].tag == "tag:example.org/write/value-1.0.0" + @test ASDF._convert_tree(WriteAlias(WriteValue(4)))["value"] == 4 - scalar = ASDF.to_tree(WriteScalar("converted")) - @test scalar isa ASDF.TaggedScalar - @test scalar.tag == "tag:example.org/write/scalar-1.0.0" - @test String(scalar) == "converted" - - @test ASDF.to_tree(WriteValue(5)).tag == "tag:example.org/write/value-1.0.0" - @test ASDF.to_tree(SpecialWriteValue(5)).tag == "tag:example.org/write/special-1.0.0" -end + source = OrderedDict("tuple" => (WriteValue(5),), "named" => (child=WriteValue(6),)) + tree = ASDF._convert_tree(source) + @test tree["tuple"][1]["value"] == 5 + @test tree["named"]["child"]["value"] == 6 + @test source["tuple"][1] isa WriteValue -@testset "recursive write conversion" begin - tagged = ASDF.TaggedMapping("tag:example.org/write/existing-1.0.0", OrderedDict("child" => WriteValue(6))) - converted = ASDF.to_tree(tagged) - @test converted.tag == tagged.tag - @test converted["child"] isa ASDF.TaggedMapping - @test tagged["child"] isa WriteValue - - source = OrderedDict("zebra" => WriteValue(7), "apple" => Any[1, WriteValue(8)], "matrix" => [1 2; 3 4]) - plain = ASDF.to_tree(source) - @test collect(keys(plain)) == collect(keys(source)) - @test plain["zebra"] isa ASDF.TaggedMapping - @test plain["apple"][2] isa ASDF.TaggedMapping - @test plain["matrix"] == source["matrix"] - @test source["zebra"] isa WriteValue - @test source["apple"][2] isa WriteValue - - cyclic_mapping = OrderedDict{Any, Any}() + cyclic_mapping = OrderedDict{Any,Any}() cyclic_mapping["self"] = cyclic_mapping - @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_mapping) - cyclic_sequence = Any[] push!(cyclic_sequence, cyclic_sequence) - @test_throws "cyclic ASDF write conversion" ASDF.to_tree(cyclic_sequence) - @test_throws "cyclic ASDF write conversion" ASDF.to_tree(CyclicWriteValue()) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_mapping) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(WriteLoop()) + @test_throws "must return a supported ASDF tree node" ASDF._convert_tree(WriteSame(1)) + @test_throws "not supported by the ASDF writer" ASDF._convert_tree(UnsupportedWriteValue()) + @test_throws "keys must be booleans, integers, or strings" ASDF._convert_tree(OrderedDict(WriteValue(1) => 2)) + @test_throws "wrap Matrix" ASDF._convert_tree([WriteValue(1) WriteValue(2)]) end @testset "heterogeneous document writing" begin data = reshape(collect(1.0:12.0), 3, 4) document = OrderedDict( "roman" => OrderedDict( - "meta" => OrderedDict("model" => WriteParent(SpecialWriteValue(9)), "description" => "nested"), + "meta" => OrderedDict("model" => WriteParent(WriteValue(9)), "description" => "nested"), "data" => WriteBlock(data), ), "name" => "product", @@ -99,22 +64,13 @@ end mktempdir() do directory filename = joinpath(directory, "custom.asdf") ASDF.write_file(filename, document) - loaded = ASDF.load_file(filename; extensions = true) + loaded = ASDF.load_file(filename; extensions=true) @test collect(keys(loaded.metadata)) == ["roman", "name", "asdf_library"] - @test loaded["name"] == "product" - @test loaded["roman"]["meta"]["model"].tag == "tag:example.org/write/parent-1.0.0" - @test loaded["roman"]["meta"]["model"]["child"].tag == "tag:example.org/write/special-1.0.0" - @test loaded["roman"]["data"].tag == "tag:example.org/write/block-1.0.0" + @test loaded["roman"]["meta"]["model"]["child"]["value"] == 9 @test loaded["roman"]["data"]["data"][] == data @test !haskey(document, "asdf_library") @test document["roman"]["meta"]["model"] isa WriteParent @test document["roman"]["data"] isa WriteBlock - - save_filename = joinpath(directory, "custom-save.asdf") - save(save_filename, document) - saved = ASDF.load_file(save_filename; extensions = true) - @test saved["roman"]["data"]["data"][] == data - @test saved["roman"]["meta"]["model"]["child"]["value"] == 9 end end