Towards DFG v1 deprecated/renamed fields and serialization changes (#1892)

* Update some deprecated/renamed fields

* getVariableType -> getStateKind, getFactorType -> getObservation

* fix searialization

Author: Affie

IncrementalInference/ext/FluxModelsSerialization.jl

@@ -33,7 +33,7 @@ function _deserializeFluxDataBase64(sdata::AbstractString)
   return data
 end
 
-function DFG.packDistribution(obj::FluxModelsDistribution)
+function DFG.pack(obj::FluxModelsDistribution)
   #
 
   # and the specialSampler function -- likely to be deprecated
@@ -75,7 +75,7 @@ function DFG.packDistribution(obj::FluxModelsDistribution)
   #
 end
 
-function DFG.unpackDistribution(obj::PackedFluxModelsDistribution)
+function DFG.unpack(obj::PackedFluxModelsDistribution)
   #
   obj.serializeHollow && @warn(
     "Deserialization of FluxModelsDistribution.serializationHollow=true is not yet well developed, please open issues at IncrementalInference.jl accordingly."
@@ -104,15 +104,15 @@ function Base.convert(
 )
   #
   # convert to packed type first
-  return DFG.packDistribution(obj)
+  return DFG.pack(obj)
 end
 
 function convert(
   ::Union{Type{<:SamplableBelief}, Type{<:FluxModelsDistribution}},
   obj::PackedFluxModelsDistribution,
 )
   #
-  return DFG.unpackDistribution(obj)
+  return DFG.unpack(obj)
 end
 
 #

IncrementalInference/ext/IncrInfrDiffEqFactorExt.jl

@@ -174,7 +174,7 @@ end
 #     # backward direction
 #     prob = oder.backwardProblem
 #     addOp, diffOp, _, _ = AMP.buildHybridManifoldCallbacks(
-#       convert(Tuple, getManifold(getVariableType(cf.fullvariables[1]))),
+#       convert(Tuple, getManifold(getStateKind(cf.fullvariables[1]))),
 #     )
 #     # FIXME use ccw.varValsAll containter?
 #     (getBelief(cf.fullvariables[2]) |> getPoints)[cf._sampleIdx]
@@ -183,7 +183,7 @@ end
 #     prob = oder.forwardProblem
 #     # buffer manifold operations for use during factor evaluation
 #     addOp, diffOp, _, _ = AMP.buildHybridManifoldCallbacks(
-#       convert(Tuple, getManifold(getVariableType(cf.fullvariables[2]))),
+#       convert(Tuple, getManifold(getStateKind(cf.fullvariables[2]))),
 #     )
 #     # FIXME use ccw.varValsAll containter?
 #     (getBelief(cf.fullvariables[1]) |> getPoints)[cf._sampleIdx]
@@ -254,15 +254,15 @@ end
 #     # backward direction
 #     prob = oder.backwardProblem
 #     addOp, diffOp, _, _ = AMP.buildHybridManifoldCallbacks(
-#       convert(Tuple, getManifold(getVariableType(cf.fullvariables[1]))),
+#       convert(Tuple, getManifold(getStateKind(cf.fullvariables[1]))),
 #     )
 #     cf._legacyParams[2]
 #   else
 #     # forward backward
 #     prob = oder.forwardProblem
 #     # buffer manifold operations for use during factor evaluation
 #     addOp, diffOp, _, _ = AMP.buildHybridManifoldCallbacks(
-#       convert(Tuple, getManifold(getVariableType(cf.fullvariables[2]))),
+#       convert(Tuple, getManifold(getStateKind(cf.fullvariables[2]))),
 #     )
 #     cf._legacyParams[1]
 #   end
@@ -290,7 +290,7 @@ function IncrementalInference.sampleFactor(cf::CalcFactor{<:DERelative}, N::Int
 
   # how many trajectories to propagate?
   # 
-  v2T = getVariableType(cf.fullvariables[2])
+  v2T = getStateKind(cf.fullvariables[2])
   meas = [allocate(getPointIdentity(v2T)) for _ = 1:N]
   # meas = [zeros(getDimension(cf.fullvariables[2])) for _ = 1:N]
 
@@ -299,7 +299,7 @@ function IncrementalInference.sampleFactor(cf::CalcFactor{<:DERelative}, N::Int
   u0pts, M = if cf.solvefor == 1
     # backward direction
     prob = oder.backwardProblem
-    M_ = getManifold(getVariableType(cf.fullvariables[1]))
+    M_ = getManifold(getStateKind(cf.fullvariables[1]))
     addOp, diffOp, _, _ = AMP.buildHybridManifoldCallbacks(
       AMP._manifoldtuple(M_),
     )
@@ -308,7 +308,7 @@ function IncrementalInference.sampleFactor(cf::CalcFactor{<:DERelative}, N::Int
   else
     # forward backward
     prob = oder.forwardProblem
-    M_ = getManifold(getVariableType(cf.fullvariables[2]))
+    M_ = getManifold(getStateKind(cf.fullvariables[2]))
     # buffer manifold operations for use during factor evaluation
     addOp, diffOp, _, _ = AMP.buildHybridManifoldCallbacks(
       AMP._manifoldtuple(M_),

IncrementalInference/src/CliqueStateMachine/services/CliqueStateMachine.jl

@@ -175,17 +175,16 @@ function presolveChecklist_StateMachine(csmc::CliqStateMachineContainer)
         csmc,
         "CSM-0b create empty data for $(getLabel(var)) on solveKey=$(csmc.solveKey)",
       )
-      varType = getVariableType(var)
+      varType = getStateKind(var)
       # FIXME check the marginalization requirements
       setDefaultNodeData!(
         var,
         0,
-        getSolverParams(csmc.cliqSubFg).N,
-        getDimension(varType);
+        getSolverParams(csmc.cliqSubFg).N;
         solveKey = csmc.solveKey,
         initialized = false,
         varType = varType,
-        dontmargin = false,
+        # dontmargin = false,
       )
       #
       @info "create vnd solveKey" csmc.solveKey N

IncrementalInference/src/Deprecated.jl

@@ -1,3 +1,51 @@
+
+# moved here from DistributedFactorGraphs.jl, replace with new way.
+function typeModuleName(variableType::StateType)
+    Base.depwarn("typeModuleName is obsolete", :typeModuleName)
+    io = IOBuffer()
+    ioc = IOContext(io, :module => DistributedFactorGraphs)
+    show(ioc, typeof(variableType))
+    return String(take!(io))
+end
+
+"""
+    $(SIGNATURES)
+Get a type from the serialization module.
+"""
+function getTypeFromSerializationModule(_typeString::AbstractString)
+    @debug "DFG converting type string to Julia type" _typeString
+    try
+        # split the type at last `.`
+        split_st = split(_typeString, r"\.(?!.*\.)")
+        #if module is specified look for the module in main, otherwise use Main        
+        if length(split_st) == 2
+            m = getfield(Main, Symbol(split_st[1]))
+        else
+            m = Main
+        end
+        noparams = split(split_st[end], r"{")
+        ret = if 1 < length(noparams)
+            # fix #671, but does not work with specific module yet
+            bidx = findfirst(r"{", split_st[end])[1]
+            @error("getTypeFromSerializationModule eval obsolete")
+            Core.eval(m, Base.Meta.parse("$(noparams[1])$(split_st[end][bidx:end])"))
+        else
+            getfield(m, Symbol(split_st[end]))
+        end
+
+        return ret
+
+    catch ex
+        @error "Unable to deserialize type $(_typeString)"
+        io = IOBuffer()
+        showerror(io, ex, catch_backtrace())
+        err = String(take!(io))
+        @error(err)
+    end
+    return nothing
+end
+
+
 ## ================================================================================================
 ## Deprecated in v0.36
 ## ================================================================================================
@@ -161,7 +209,7 @@ setVariablePosteriorEstimates!(args...; kw...) = error("PPEs are obsolete (use `
 
 @deprecate calcPPE(
   var::VariableCompute,
-  varType::StateType = getVariableType(var);
+  varType::StateType = getStateKind(var);
   solveKey::Symbol = :default,
   kwargs...,
 ) calcMeanMaxSuggested(var, solveKey)
@@ -246,7 +294,7 @@ end
 # function ManifoldsVector(fg::AbstractDFG, varIds::Vector{Symbol})
 #   manis = Bool[]
 #   for k = varIds
-#     push!(manis, getVariableType(fg, k) |> getManifold)
+#     push!(manis, getStateKind(fg, k) |> getManifold)
 #   end
 #   ManifoldsVector(manis)
 # end
@@ -309,7 +357,7 @@ function solveGraphParametric2(
 
   for vId in varIds
     p = getState(fg, vId, solvekey).val[1]
-    flatvar[vId] = getCoordinates(getVariableType(fg, vId), p)
+    flatvar[vId] = getCoordinates(getStateKind(fg, vId), p)
   end
 
   initValues = flatvar.X
@@ -379,7 +427,7 @@ end
 # ) where {N_, F <: AbstractRelativeRoots, S, T}
 #   #
 #   # error("<:AbstractRelativeRoots is obsolete, use one of the other <:AbstractRelative types instead.")
-#   # TODO get xDim = getDimension(getVariableType(Xi[sfidx])) but without having Xi
+#   # TODO get xDim = getDimension(getStateKind(Xi[sfidx])) but without having Xi
 #   if testshuffle || ccwl.partial
 #     error(
 #       "<:AbstractRelativeRoots factors with less or more measurement dimensions than variable dimensions have been discontinued, rather use <:AbstractManifoldMinimize.",
@@ -442,8 +490,8 @@ function Base.getproperty(ccw::CommonConvWrapper, f::Symbol)
     error("CommonConvWrapper.params is deprecated, use .varValsAll instead")
     return ccw.varValsAll[]
   elseif f == :vartypes
-    @warn "CommonConvWrapper.vartypes is deprecated, use typeof.(getVariableType.(ccw.fullvariables) instead" maxlog=3
-    return typeof.(getVariableType.(ccw.fullvariables))
+    @warn "CommonConvWrapper.vartypes is deprecated, use typeof.(getStateKind.(ccw.fullvariables) instead" maxlog=3
+    return typeof.(getStateKind.(ccw.fullvariables))
   elseif f == :hypotheses
     @warn "CommonConvWrapper.hypotheses is now under ccw.hyporecipe.hypotheses" maxlog=5
     return ccw.hyporecipe.hypotheses

IncrementalInference/src/ExportAPI.jl

@@ -95,9 +95,9 @@ export CSMHistory,
   getVariableOrder,
   isVariable,
   isFactor,
-  getFactorType,
+  getObservation,
   getSofttype,
-  getVariableType,
+  getStateKind,
   getLogPath,
   joinLogPath,
   lsfPriors,
@@ -299,7 +299,7 @@ export ContinuousScalar, ContinuousEuclid # TODO figure out if this will be depr
 export Circular, Circle
 
 # serializing distributions
-export packDistribution, unpackDistribution
+export pack, unpack
 export PackedCategorical #, PackedDiscreteNonParametric
 export PackedUniform, PackedNormal
 export PackedZeroMeanDiagNormal,

IncrementalInference/src/Factors/GenericFunctions.jl

@@ -238,7 +238,7 @@ function convert(
 )
   #
 
-  varT = DFG.typeModuleName(getVariableType(obj.M))
+  varT = typeModuleName(getStateKind(obj.M))
 
   c = AMP.makeCoordsFromPoint(obj.M, obj.p)
 
@@ -255,7 +255,7 @@ function convert(
   #
 
   # piggy back on serialization of StateType rather than try serialize anything Manifolds.jl
-  M = DFG.getTypeFromSerializationModule(obj.varType) |> getManifold
+  M = getTypeFromSerializationModule(obj.varType) |> getManifold
 
   # TODO this is too excessive
   e0 = getPointIdentity(M)

IncrementalInference/src/Factors/LinearRelative.jl

@@ -1,5 +1,5 @@
 
-export LinearRelative, PackedLinearRelative
+export LinearRelative
 
 """
 $(TYPEDEF)
@@ -10,8 +10,8 @@ Default linear offset between two scalar variables.
 X_2 = X_1 + η_Z
 ```
 """
-struct LinearRelative{N, T <: SamplableBelief} <: RelativeObservation 
-  Z::T
+DFG.@tags struct LinearRelative{N, T <: SamplableBelief} <: RelativeObservation 
+  Z::T & (lower = DFG.Packed, choosetype = DFG.resolvePackedType)
 end
 
 # need several helper constructors since the dimension over which LinearRelative will be used is unknown at this point
@@ -56,24 +56,4 @@ function Base.convert(
   return LieGroups.TranslationGroup(N)
 end
 
-"""
-$(TYPEDEF)
-Serialization type for `LinearRelative` binary factor.
-"""
-Base.@kwdef mutable struct PackedLinearRelative <: AbstractPackedObservation
-  Z::PackedBelief
-end
-function convert(::Type{PackedLinearRelative}, d::LinearRelative)
-  return PackedLinearRelative(convert(PackedBelief, d.Z))
-end
-function convert(::Type{LinearRelative}, d::PackedLinearRelative)
-  return LinearRelative(convert(SamplableBelief, d.Z))
-end
-function DFG.pack(d::LinearRelative)
-  return PackedLinearRelative(DFG.packDistribution(d.Z))
-end
-function DFG.unpack(d::PackedLinearRelative)
-  return LinearRelative(DFG.unpackDistribution(d.Z))
-end
-
 #

IncrementalInference/src/Factors/Mixture.jl

@@ -54,32 +54,18 @@ function Mixture(f::Type{<:AbstractObservation}, args...)
     return f(Mixture(args...))
 end
 
-Base.@kwdef struct PackedMixture{C} <: PackedBelief
-  _type::String = "IncrementalInference.PackedMixture"
-  components::C
-  prior::PackedCategorical
+DFG.@tags struct PackedMixture
+  components::NamedTuple & (choosetype = obj->NamedTuple{Tuple(Symbol.(keys(obj))), Tuple{DFG.resolveType.(values(obj))...}}, )
+  prior & (lower = DFG.Packed, choosetype = DFG.resolvePackedType)
 end
 
-# FIXME 🦨 use JSON properly (in JSONv1 upgrade)
-function PackedMixture(_type::String, comp_obj::JSON3.Object, prior_obj::JSON3.Object) 
-  PackedMixture(
-    _type,
-    NamedTuple(map(c -> c[1]=>convert(PackedBelief, c[2]), collect(pairs(comp_obj)))),
-    PackedCategorical(; prior_obj...),
-  )
-end
-
-
-function DFG.packDistribution(m::Mixture)
-  PackedMixture(;
-    components = map(packDistribution, m.components),
-    prior = PackedCategorical(; p = m.prior.p)
-  )
+function DFG.pack(m::Mixture)
+  PackedMixture(map(DFG.Packed, m.components), m.prior)
 end
 
-function DFG.unpackDistribution(pm::PackedMixture)
+function DFG.unpack(pm::PackedMixture)
   Mixture(
-    map(unpackDistribution, pm.components),
-    unpackDistribution(pm.prior)
+    map(unpack, pm.components),
+    pm.prior
   )
 end

IncrementalInference/src/Factors/PartialPrior.jl

@@ -35,10 +35,10 @@ Base.@kwdef struct PackedPartialPrior <: AbstractPackedObservation
 end
 
 function convert(::Type{PackedPartialPrior}, d::PartialPrior)
-  return PackedPartialPrior(DFG.typeModuleName(d.varType), convert(PackedBelief, d.Z), [d.partial...;])
+  return PackedPartialPrior(typeModuleName(d.varType), convert(PackedBelief, d.Z), [d.partial...;])
   # return PackedPartialPrior(convert(PackedBelief, d.Z), [d.partial...;])
 end
 function convert(::Type{PartialPrior}, d::PackedPartialPrior)
-  return PartialPrior(DFG.getTypeFromSerializationModule(d.varType), convert(SamplableBelief, d.Z),(d.partials...,))
+  return PartialPrior(getTypeFromSerializationModule(d.varType), convert(SamplableBelief, d.Z),(d.partials...,))
   # return PartialPrior(convert(SamplableBelief, d.Z), (d.partials...,))
 end