diff --git a/README.md b/README.md index 1aeed6a..b57c79e 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ A new [Advanced Scientific Data Format (ASDF)](https://asdf-standard.readthedocs.io/en/latest/index.html) package, written in Julia. +Packages can extend `ASDF.to_tree` to embed their own Julia objects anywhere in +a larger ASDF document alongside ordinary metadata and binary arrays. See the +[custom type documentation](https://juliaastro.org/ASDF.jl/dev/custom-types/) +for the write-conversion interface. + ## Quickstart ```julia diff --git a/docs/make.jl b/docs/make.jl index f739fbb..283669d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,6 +20,7 @@ makedocs(; "JWST" => "examples/jwst.md", "Roman" => "examples/roman.md", ], + "Custom Julia types" => "custom-types.md", "Interoperability" => "interop.md", "API" => "api.md", ], diff --git a/docs/src/custom-types.md b/docs/src/custom-types.md new file mode 100644 index 0000000..38de4a0 --- /dev/null +++ b/docs/src/custom-types.md @@ -0,0 +1,86 @@ +# Custom Julia types + +ASDF documents often combine metadata, binary arrays, and objects owned by +domain packages. A package can make its types writable anywhere in an ASDF +document by extending [`ASDF.to_tree`](@ref): + +```@example custom_types +using ASDF +using OrderedCollections + +struct Measurement + value::Float64 + unit::String +end + +function ASDF.to_tree(measurement::Measurement) + properties = OrderedDict("value" => measurement.value, "unit" => measurement.unit) + return ASDF.TaggedMapping("tag:example.org/measurement-1.0.0", properties) +end + +document = OrderedDict( + "meta" => OrderedDict("exposure" => Measurement(1200.0, "s")), + "data" => ASDF.NDArrayWrapper(reshape(collect(1.0:12.0), 3, 4)), +) + +save("custom-types.asdf", document) +``` + +`save` and [`ASDF.write_file`](@ref) recursively walk the complete document. +When they encounter a `Measurement`, Julia dispatch selects the method above. +ASDF then recursively converts custom objects contained in the returned node +before writing YAML and binary blocks. + +The original document is not modified. Calling the hook directly inspects its +shallow representation: + +```@example custom_types +node = ASDF.to_tree(Measurement(5.0, "m")) +node.tag +``` + +## Conversion contract + +Packages extend the one-argument `ASDF.to_tree(value)` hook. The fallback +returns `value` unchanged. + +A package method should return one of: + +- `nothing`, a boolean, integer, float, or string; +- a mapping with boolean, integer, or string keys, a vector, tuple, or named tuple; +- [`ASDF.TaggedMapping`](@ref), [`ASDF.TaggedSequence`](@ref), or + [`ASDF.TaggedScalar`](@ref); +- [`ASDF.NDArrayWrapper`](@ref) for explicit inline or binary array storage. + +Converter methods are shallow. They may return mappings or sequences containing +other custom objects; the writer converts those children automatically and +redispatches when a converter delegates to another custom type. Converters +should not call `to_tree` recursively themselves. + +Unsupported leaves and mapping keys produce an error instead of being silently +stringified. Multidimensional arrays must be wrapped in `NDArrayWrapper`; +metadata sequences are vectors. ASDF.jl rejects cyclic mappings, sequences, or +converter output because ASDF reference serialization is not yet implemented. + +## Optional ASDF support + +When ASDF.jl is an optional dependency, define the method in a Julia package +extension that loads only when both packages are present: + +```julia +module MyPackageASDFExt + +using ASDF +using MyPackage + +function ASDF.to_tree(value::MyPackage.CustomType) + return ASDF.TaggedMapping("tag:example.org/custom-1.0.0", Dict("value" => value.value)) +end + +end +``` + +This interface only controls writing. Loading an unknown tag with +`extensions = true` produces an `ASDF.TaggedMapping`, `ASDF.TaggedSequence`, or +`ASDF.TaggedScalar`; reconstructing package-owned objects and validating their +schemas require separate read-side support. diff --git a/docs/src/index.md b/docs/src/index.md index cc5a031..b1b3791 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -125,4 +125,7 @@ af["data"][] == [1, 2, 3, 4] ## Tagged objects -Come back soon to see how custom Julia objects can be handled in ASDF.jl. +Packages can extend [`ASDF.to_tree`](@ref) to serialize their own Julia types +wherever they occur in a larger ASDF document. See [Custom Julia +types](@ref) for the conversion contract and an example combining a custom +metadata object with a binary array. diff --git a/src/ASDF.jl b/src/ASDF.jl index 0f37cdb..9435f39 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -1067,6 +1067,7 @@ but otherwise behave as plain mappings; and, when loading with `extensions = tru extension tags that would otherwise raise an error. Each behaves exactly like its wrapped `value`. A mapping indexes and iterates as a dict, a sequence as a vector, a scalar as a string, while retaining the original `tag` so the node round-trips unchanged through [`ASDF.write_file`](@ref). +The `tag` field must contain the full tag URI; the writer emits namespace shorthand where applicable. """ struct TaggedMapping{D <: AbstractDict} <: AbstractDict{Any, Any} tag::String @@ -1153,6 +1154,68 @@ function YAML._print(io::IO, val::TaggedScalar, level::Int = 0, ignore_level::Bo return YAML._print(io, val.value, level, ignore_level) end +""" + ASDF.to_tree(value) + +Convert one package-owned value to an ASDF-compatible tree node. + +Packages extend this shallow hook for their own types. A method must return a +supported scalar, mapping, vector, [`ASDF.TaggedMapping`](@ref), +[`ASDF.TaggedSequence`](@ref), [`ASDF.TaggedScalar`](@ref), or +[`ASDF.NDArrayWrapper`](@ref). ASDF recursively converts values nested in the +returned node while writing a file. The fallback returns `value` unchanged. +""" +to_tree(value) = value + +function _convert_tree(value, active = Base.IdSet{Any}()) + value in active && throw(ArgumentError("cyclic ASDF write conversion involving $(typeof(value)) is not supported")) + push!(active, value) + try + converted = to_tree(value) + if converted === value + which(to_tree, (typeof(value),)) === which(to_tree, (Any,)) && return _convert_tree_children(value, active) + throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) + end + typeof(converted) === typeof(value) && throw(ArgumentError("ASDF.to_tree(::$(typeof(value))) must return a supported ASDF tree node, not another $(typeof(value))")) + return _convert_tree(converted, active) + finally + delete!(active, value) + end +end + +_convert_tree_children(value::Nothing, active) = value +_convert_tree_children(value::Union{Bool,Integer,AbstractString}, active) = value +_convert_tree_children(value::AbstractFloat, active) = YAMLScalar(yaml_float_string(value)) +_convert_tree_children(value::NDArray, active) = value +_convert_tree_children(value::TaggedScalar, active) = TaggedScalar(value.tag, _convert_tree(value.value, active)) +_convert_tree_children(value::NamedTuple, active) = OrderedDict{Any,Any}(String(key) => _convert_tree(item, active) for (key, item) in pairs(value)) +_convert_tree_children(value::Tuple, active) = [_convert_tree(item, active) for item in value] + +function _convert_tree_children(value::TaggedMapping, active) + return TaggedMapping(value.tag, _convert_tree_children(value.value, active)) +end + +function _convert_tree_children(value::TaggedSequence, active) + return TaggedSequence(value.tag, _convert_tree_children(value.value, active)) +end + +function _convert_tree_children(value::AbstractDict, active) + converted = OrderedDict{Any,Any}() + for (key, item) in value + key isa Union{Bool,Integer,AbstractString} || throw(ArgumentError("ASDF mapping key $(repr(key)) has unsupported type $(typeof(key)); keys must be booleans, integers, or strings")) + converted[key] = _convert_tree(item, active) + end + return converted +end + +_convert_tree_children(value::AbstractVector, active) = [_convert_tree(item, active) for item in value] +function _convert_tree_children(value::AbstractArray, active) + throw(ArgumentError("ASDF metadata arrays must be vectors; wrap $(typeof(value)) in ASDF.NDArrayWrapper to write an N-dimensional array")) +end +function _convert_tree_children(value, active) + throw(ArgumentError("value of type $(typeof(value)) is not supported by the ASDF writer; define ASDF.to_tree(::$(typeof(value)))")) +end + function YAML._print(io::IO, val::NDArray, level::Int = 0, ignore_level::Bool = false) # TODO: Get compression from underlying header block? return YAML._print(io, NDArrayWrapper(val[]; compression = C_None), level, ignore_level) @@ -1587,6 +1650,7 @@ function YAML._print(io::IO, val::ASDFLibrary, level::Int = 0, ignore_level::Boo library = OrderedDict(:name => val.name, :author => val.author, :homepage => val.homepage, :version => val.version) return YAML._print(io, library, level, ignore_level) end +_convert_tree_children(value::ASDFLibrary, active) = value """ NDArrayWrapper @@ -1612,6 +1676,7 @@ function NDArrayWrapper(array::AbstractArray; compression::Compression = C_Bzip2 return NDArrayWrapper(array, compression, inline, lz4_layout) end Base.getindex(val::NDArrayWrapper) = val.array +_convert_tree_children(value::NDArrayWrapper, active) = value """ Blocks @@ -1756,7 +1821,10 @@ end """ write_file(filename::AbstractString, document::AbstractDict) -Writes an ASDF file to disk. `document` is a plain `Dict` whose values may include [`NDArrayWrapper`](@ref) instances. These are serialized as binary blocks with appropriate compression. +Writes an ASDF file to disk. `document` may contain custom Julia objects with +[`ASDF.to_tree`](@ref) methods, including objects nested inside mappings or +vectors. Values may also include [`NDArrayWrapper`](@ref) instances, which are +serialized as binary blocks with appropriate compression. Layout of the output file: @@ -1783,8 +1851,8 @@ function write_file(filename::AbstractString, document::AbstractDict) # back to an unordered `Dict` and drop the order). The provenance entry is stamped last. full_document = OrderedDict{Any, Any}(document) full_document["asdf_library"] = library - # Rewrite floats so their exponents are YAML-1.1 compliant (see `yaml_compliant`). - full_document = yaml_compliant(full_document) + # Convert package-owned objects and normalize floats before block collection. + full_document = _convert_tree(full_document) # Write YAML part of file io = open(filename, "w") diff --git a/test/test-write-converters.jl b/test/test-write-converters.jl new file mode 100644 index 0000000..4f6299c --- /dev/null +++ b/test/test-write-converters.jl @@ -0,0 +1,76 @@ +struct WriteValue + value::Int +end +struct WriteParent + child +end +struct WriteAlias + value::WriteValue +end +struct WriteBlock + data::Matrix{Float64} +end +struct WriteLoop end +struct WriteSame + value::Int +end +struct UnsupportedWriteValue end + +ASDF.to_tree(value::WriteValue) = ASDF.TaggedMapping("tag:example.org/write/value-1.0.0", OrderedDict("value" => value.value)) +ASDF.to_tree(value::WriteParent) = ASDF.TaggedMapping("tag:example.org/write/parent-1.0.0", OrderedDict("child" => value.child)) +ASDF.to_tree(value::WriteAlias) = value.value +ASDF.to_tree(value::WriteBlock) = ASDF.TaggedMapping("tag:example.org/write/block-1.0.0", OrderedDict("data" => ASDF.NDArrayWrapper(value.data; compression=ASDF.C_Zlib))) +ASDF.to_tree(value::WriteLoop) = OrderedDict("self" => value) +ASDF.to_tree(value::WriteSame) = WriteSame(value.value) + +@testset "write conversion protocol" begin + shallow = ASDF.to_tree(WriteParent(WriteValue(3))) + @test shallow["child"] isa WriteValue + + converted = ASDF._convert_tree(WriteParent(WriteValue(3))) + @test converted.tag == "tag:example.org/write/parent-1.0.0" + @test converted["child"].tag == "tag:example.org/write/value-1.0.0" + @test ASDF._convert_tree(WriteAlias(WriteValue(4)))["value"] == 4 + + source = OrderedDict("tuple" => (WriteValue(5),), "named" => (child=WriteValue(6),)) + tree = ASDF._convert_tree(source) + @test tree["tuple"][1]["value"] == 5 + @test tree["named"]["child"]["value"] == 6 + @test source["tuple"][1] isa WriteValue + + cyclic_mapping = OrderedDict{Any,Any}() + cyclic_mapping["self"] = cyclic_mapping + cyclic_sequence = Any[] + push!(cyclic_sequence, cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_mapping) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(cyclic_sequence) + @test_throws "cyclic ASDF write conversion" ASDF._convert_tree(WriteLoop()) + @test_throws "must return a supported ASDF tree node" ASDF._convert_tree(WriteSame(1)) + @test_throws "not supported by the ASDF writer" ASDF._convert_tree(UnsupportedWriteValue()) + @test_throws "keys must be booleans, integers, or strings" ASDF._convert_tree(OrderedDict(WriteValue(1) => 2)) + @test_throws "wrap Matrix" ASDF._convert_tree([WriteValue(1) WriteValue(2)]) +end + +@testset "heterogeneous document writing" begin + data = reshape(collect(1.0:12.0), 3, 4) + document = OrderedDict( + "roman" => OrderedDict( + "meta" => OrderedDict("model" => WriteParent(WriteValue(9)), "description" => "nested"), + "data" => WriteBlock(data), + ), + "name" => "product", + ) + + mktempdir() do directory + filename = joinpath(directory, "custom.asdf") + ASDF.write_file(filename, document) + loaded = ASDF.load_file(filename; extensions=true) + + @test collect(keys(loaded.metadata)) == ["roman", "name", "asdf_library"] + @test loaded["roman"]["meta"]["model"]["child"]["value"] == 9 + @test loaded["roman"]["data"]["data"][] == data + @test !haskey(document, "asdf_library") + @test document["roman"]["meta"]["model"] isa WriteParent + @test document["roman"]["data"] isa WriteBlock + end +end