From 5af1afac7c4b47db997c492947628dfee979b37f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 31 Oct 2023 23:10:13 +0000 Subject: [PATCH 001/182] initial implementation of VarNameVector --- src/DynamicPPL.jl | 2 + src/varnamevector.jl | 113 ++++++++++++++++++++++++++++++++++++++++++ test/varnamevector.jl | 27 ++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/varnamevector.jl create mode 100644 test/varnamevector.jl diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index b90381dea..1306b123b 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -46,6 +46,7 @@ export AbstractVarInfo, UntypedVarInfo, TypedVarInfo, SimpleVarInfo, + VarNameVector, push!!, empty!!, subset, @@ -168,6 +169,7 @@ include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") include("simple_varinfo.jl") +include("varnamevector.jl") include("context_implementations.jl") include("compiler.jl") include("prob_macro.jl") diff --git a/src/varnamevector.jl b/src/varnamevector.jl new file mode 100644 index 000000000..8be220972 --- /dev/null +++ b/src/varnamevector.jl @@ -0,0 +1,113 @@ +# Similar to `Metadata` but representing a `Vector` and simpler interface. +# TODO: Should we subtype `AbstractVector`? +struct VarNameVector{ + TIdcs<:OrderedDict{<:VarName,Int}, + TVN<:AbstractVector{<:VarName}, + TVal<:AbstractVector, + TTrans<:AbstractVector +} + "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" + idcs::TIdcs # Dict{<:VarName,Int} + + "vector of identifiers for the random variables, where `vns[idcs[vn]] == vn`" + vns::TVN # AbstractVector{<:VarName} + + "vector of index ranges in `vals` corresponding to `vns`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" + ranges::Vector{UnitRange{Int}} + + "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[idcs[vn]]]`" + vals::TVal # AbstractVector{<:Real} + + "vector of transformations whose inverse takes us back to the original space" + transforms::TTrans +end + +# Useful transformation going from the flattened representation. +struct FromVec{Sz} + sz::Sz +end + +FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) + +# TODO: Should we materialize the `reshape`? +(f::FromVec)(x) = reshape(x, f.sz) +(f::FromVec{Tuple{}})(x) = only(x) + +Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) + +tovec(x::Real) = [x] +tovec(x::AbstractArray) = vec(x) + +Bijectors.inverse(f::FromVec) = tovec +Bijectors.inverse(f::FromVec{Tuple{}}) = tovec + +VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) +VarNameVector(vns, vals) = VarNameVector(collect(vns), collect(vals)) +function VarNameVector( + vns::AbstractVector, + vals::AbstractVector, + transforms = map(FromVec, vals) +) + # TODO: Check uniqueness of `vns`? + + # Convert `vals` into a vector of vectors. + vals_vecs = map(tovec, vals) + + # TODO: Is this really the way to do this? + if !(eltype(vns) <: VarName) + vns = convert(Vector{VarName}, vns) + end + idcs = OrderedDict{eltype(vns),Int}() + ranges = Vector{UnitRange{Int}}() + offset = 0 + for (i, (vn, x)) in enumerate(zip(vns, vals_vecs)) + # Add the varname index. + push!(idcs, vn => length(idcs) + 1) + # Add the range. + r = (offset + 1):(offset + length(x)) + push!(ranges, r) + # Update the offset. + offset = r[end] + end + + return VarNameVector(idcs, vns, ranges, reduce(vcat, vals_vecs), transforms) +end + +# Basic array interface. +Base.eltype(vmd::VarNameVector) = eltype(vmd.vals) +Base.length(vmd::VarNameVector) = length(vmd.vals) +Base.size(vmd::VarNameVector) = size(vmd.vals) + +Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() + +# `getindex` & `setindex!` +getidc(vmd::VarNameVector, vn::VarName) = vmd.idcs[vn] + +getrange(vmd::VarNameVector, i::Int) = vmd.ranges[i] +getrange(vmd::VarNameVector, vn::VarName) = getrange(vmd, getidc(vmd, vn)) + +gettransform(vmd::VarNameVector, vn::VarName) = vmd.transforms[getidc(vmd, vn)] + +Base.getindex(vmd::VarNameVector, i::Int) = vmd.vals[i] +function Base.getindex(vmd::VarNameVector, vn::VarName) + x = vmd.vals[getrange(vmd, vn)] + f = gettransform(vmd, vn) + return f(x) +end + +Base.setindex!(vmd::VarNameVector, val, i::Int) = vmd.vals[i] = val +function Base.setindex!(vmd::VarNameVector, val, vn::VarName) + f = inverse(gettransform(vmd, vn)) + vmd.vals[getrange(vmd, vn)] = f(val) +end + +# TODO: Re-use some of the show functionality from Base? +function Base.show(io::IO, vmd::VarNameVector) + print(io, "[") + for (i, vn) in enumerate(vmd.vns) + if i > 1 + print(io, ", ") + end + print(io, vn, " = ", vmd[vn]) + end +end diff --git a/test/varnamevector.jl b/test/varnamevector.jl new file mode 100644 index 000000000..22b8c817b --- /dev/null +++ b/test/varnamevector.jl @@ -0,0 +1,27 @@ +@testset "VarNameVector" begin + vns = [ + @varname(x[1]), + @varname(x[2]), + @varname(x[3]), + ] + vals = [ + 1, + 2:3, + reshape(4:9, 2, 3), + ] + vnv = VarNameVector(vns, vals) + + # `getindex` + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val + end + + # `setindex!` + for (vn, val) in zip(vns, vals) + vnv[vn] = val .+ 100 + end + + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val .+ 100 + end +end From 8ce53f7b6ceccf0ae82a24539a391c10d0ebf927 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 7 Nov 2023 10:55:18 +0000 Subject: [PATCH 002/182] added some hacky getval and getdist get things to work for VarInfo --- src/DynamicPPL.jl | 2 +- src/varinfo.jl | 31 +++++++++++++++--- src/varnamevector.jl | 77 +++++++++++++++++++++++++++++++++----------- 3 files changed, 87 insertions(+), 23 deletions(-) diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 1306b123b..d74c09446 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -165,11 +165,11 @@ include("sampler.jl") include("varname.jl") include("distribution_wrappers.jl") include("contexts.jl") +include("varnamevector.jl") include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") include("simple_varinfo.jl") -include("varnamevector.jl") include("context_implementations.jl") include("compiler.jl") include("prob_macro.jl") diff --git a/src/varinfo.jl b/src/varinfo.jl index 590626df3..b0c20b87d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -101,6 +101,7 @@ struct VarInfo{Tmeta,Tlogp} <: AbstractVarInfo logp::Base.RefValue{Tlogp} num_produce::Base.RefValue{Int} end +const VectorVarInfo = VarInfo{<:VarNameVector} const UntypedVarInfo = VarInfo{<:Metadata} const TypedVarInfo = VarInfo{<:NamedTuple} const VarInfoOrThreadSafeVarInfo{Tmeta} = Union{ @@ -520,6 +521,8 @@ Return the distribution from which `vn` was sampled in `vi`. """ getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] +# HACK: we shouldn't need this +getdist(::VarNameVector, ::VarName) = nothing """ getval(vi::VarInfo, vn::VarName) @@ -530,6 +533,8 @@ The values may or may not be transformed to Euclidean space. """ getval(vi::VarInfo, vn::VarName) = getval(getmetadata(vi, vn), vn) getval(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +# HACK: We shouldn't need this +getval(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) """ setval!(vi::VarInfo, val, vn::VarName) @@ -562,13 +567,14 @@ Return the values of all the variables in `vi`. The values may or may not be transformed to Euclidean space. """ -getall(vi::UntypedVarInfo) = getall(vi.metadata) +getall(vi::VarInfo) = getall(vi.metadata) # NOTE: `mapreduce` over `NamedTuple` results in worse type-inference. # See for example https://github.com/JuliaLang/julia/pull/46381. getall(vi::TypedVarInfo) = reduce(vcat, map(getall, vi.metadata)) function getall(md::Metadata) return mapreduce(Base.Fix1(getval, md), vcat, md.vns; init=similar(md.vals, 0)) end +getall(vnv::VarNameVector) = vnv.vals """ setall!(vi::VarInfo, val) @@ -743,7 +749,7 @@ end @inline function _getranges(vi::VarInfo, s::Selector, space) return _getranges(vi, _getidcs(vi, s, space)) end -@inline function _getranges(vi::UntypedVarInfo, idcs::Vector{Int}) +@inline function _getranges(vi::VarInfo, idcs::Vector{Int}) return mapreduce(i -> vi.metadata.ranges[i], vcat, idcs; init=Int[]) end @inline _getranges(vi::TypedVarInfo, idcs::NamedTuple) = _getranges(vi.metadata, idcs) @@ -848,6 +854,12 @@ function TypedVarInfo(vi::UntypedVarInfo) return VarInfo(nt, Ref(logp), Ref(num_produce)) end TypedVarInfo(vi::TypedVarInfo) = vi +function TypedVarInfo(vi::VectorVarInfo) + logp = getlogp(vi) + num_produce = get_num_produce(vi) + nt = group_by_symbol(vi.metadata) + return VarInfo(nt, Ref(logp), Ref(num_produce)) +end function BangBang.empty!!(vi::VarInfo) _empty!(vi.metadata) @@ -867,6 +879,8 @@ end # Functions defined only for UntypedVarInfo Base.keys(vi::UntypedVarInfo) = keys(vi.metadata.idcs) +Base.keys(vi::VectorVarInfo) = keys(vi.metadata) + # HACK: Necessary to avoid returning `Any[]` which won't dispatch correctly # on other methods in the codebase which requires `Vector{<:VarName}`. Base.keys(vi::TypedVarInfo{<:NamedTuple{()}}) = VarName[] @@ -890,7 +904,10 @@ function setgid!(vi::VarInfo, gid::Selector, vn::VarName) return push!(getmetadata(vi, vn).gids[getidx(vi, vn)], gid) end -istrans(vi::VarInfo, vn::VarName) = is_flagged(vi, vn, "trans") +istrans(vi::VarInfo, vn::VarName) = istrans(getmetadata(vi, vn), vn) +istrans(md::Metadata, vn::VarName) = is_flagged(md, vn, "trans") +istrans(vnv::VarNameVector, vn::VarName) = !(gettransform(vnv, vn) isa FromVec) + getlogp(vi::VarInfo) = vi.logp[] @@ -1406,6 +1423,12 @@ function getindex(vi::VarInfo, vn::VarName, dist::Distribution) val = getval(vi, vn) return maybe_invlink_and_reconstruct(vi, vn, dist, val) end +function getindex(vi::VectorVarInfo, vn::VarName, ::Nothing) + if !haskey(vi, vn) + throw(KeyError(vn)) + end + return getmetadata(vi, vn)[vn] +end function getindex(vi::VarInfo, vns::Vector{<:VarName}) # FIXME(torfjelde): Using `getdist(vi, first(vns))` won't be correct in cases # such as `x .~ [Normal(), Exponential()]`. @@ -1440,7 +1463,7 @@ Return the current value(s) of the random variables sampled by `spl` in `vi`. The value(s) may or may not be transformed to Euclidean space. """ -getindex(vi::UntypedVarInfo, spl::Sampler) = copy(getval(vi, _getranges(vi, spl))) +getindex(vi::VarInfo, spl::Sampler) = copy(getval(vi, _getranges(vi, spl))) function getindex(vi::TypedVarInfo, spl::Sampler) # Gets the ranges as a NamedTuple ranges = _getranges(vi, spl) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 8be220972..7407b3068 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -74,40 +74,81 @@ function VarNameVector( end # Basic array interface. -Base.eltype(vmd::VarNameVector) = eltype(vmd.vals) -Base.length(vmd::VarNameVector) = length(vmd.vals) -Base.size(vmd::VarNameVector) = size(vmd.vals) +Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) +Base.length(vnv::VarNameVector) = length(vnv.vals) +Base.size(vnv::VarNameVector) = size(vnv.vals) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() +# Dictionary interface. +Base.keys(vnv::VarNameVector) = vnv.vns + +Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.idcs, vn) + # `getindex` & `setindex!` -getidc(vmd::VarNameVector, vn::VarName) = vmd.idcs[vn] +getidx(vnv::VarNameVector, vn::VarName) = vnv.idcs[vn] -getrange(vmd::VarNameVector, i::Int) = vmd.ranges[i] -getrange(vmd::VarNameVector, vn::VarName) = getrange(vmd, getidc(vmd, vn)) +getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] +getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) -gettransform(vmd::VarNameVector, vn::VarName) = vmd.transforms[getidc(vmd, vn)] +gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] -Base.getindex(vmd::VarNameVector, i::Int) = vmd.vals[i] -function Base.getindex(vmd::VarNameVector, vn::VarName) - x = vmd.vals[getrange(vmd, vn)] - f = gettransform(vmd, vn) +Base.getindex(vnv::VarNameVector, ::Colon) = vnv.vals +Base.getindex(vnv::VarNameVector, i::Int) = vnv.vals[i] +function Base.getindex(vnv::VarNameVector, vn::VarName) + x = vnv.vals[getrange(vnv, vn)] + f = gettransform(vnv, vn) return f(x) end -Base.setindex!(vmd::VarNameVector, val, i::Int) = vmd.vals[i] = val -function Base.setindex!(vmd::VarNameVector, val, vn::VarName) - f = inverse(gettransform(vmd, vn)) - vmd.vals[getrange(vmd, vn)] = f(val) +# HACK: remove this as soon as possible. +Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] + +Base.setindex!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +function Base.setindex!(vnv::VarNameVector, val, vn::VarName) + f = inverse(gettransform(vnv, vn)) + vnv.vals[getrange(vnv, vn)] = f(val) +end + +function Base.empty!(vnv::VarNameVector) + # TODO: Or should the semantics be different, e.g. keeping `vns`? + empty!(vnv.idcs) + empty!(vnv.vns) + empty!(vnv.ranges) + empty!(vnv.vals) + empty!(vnv.transforms) + return nothing end +BangBang.empty!!(vnv::VarNameVector) = empty!(vnv) # TODO: Re-use some of the show functionality from Base? -function Base.show(io::IO, vmd::VarNameVector) +function Base.show(io::IO, vnv::VarNameVector) print(io, "[") - for (i, vn) in enumerate(vmd.vns) + for (i, vn) in enumerate(vnv.vns) if i > 1 print(io, ", ") end - print(io, vn, " = ", vmd[vn]) + print(io, vn, " = ", vnv[vn]) end end + +# Typed version. +function group_by_symbol(vnv::VarNameVector) + # Group varnames in `vnv` by the symbol. + d = OrderedDict{Symbol,Vector{VarName}}() + for vn in vnv.vns + push!(get!(d, getsym(vn), Vector{VarName}()), vn) + end + + # Create a `NamedTuple` from the grouped varnames. + nt_vals = map(values(d)) do vns + # TODO: Do we need to specialize the inputs here? + VarNameVector( + map(identity, vns), + map(Base.Fix1(getindex, vnv), vns), + map(Base.Fix1(gettransform, vnv), vns) + ) + end + + return NamedTuple{Tuple(keys(d))}(nt_vals) +end From fc6a051188f2bd3111caaed0ec9697e147c65c75 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 7 Nov 2023 11:13:54 +0000 Subject: [PATCH 003/182] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/varinfo.jl | 1 - src/varnamevector.jl | 10 ++++------ test/varnamevector.jl | 12 ++---------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index b0c20b87d..df1be3d21 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -908,7 +908,6 @@ istrans(vi::VarInfo, vn::VarName) = istrans(getmetadata(vi, vn), vn) istrans(md::Metadata, vn::VarName) = is_flagged(md, vn, "trans") istrans(vnv::VarNameVector, vn::VarName) = !(gettransform(vnv, vn) isa FromVec) - getlogp(vi::VarInfo) = vi.logp[] function setlogp!!(vi::VarInfo, logp) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 7407b3068..dd55fb7c0 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -4,7 +4,7 @@ struct VarNameVector{ TIdcs<:OrderedDict{<:VarName,Int}, TVN<:AbstractVector{<:VarName}, TVal<:AbstractVector, - TTrans<:AbstractVector + TTrans<:AbstractVector, } "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" idcs::TIdcs # Dict{<:VarName,Int} @@ -44,9 +44,7 @@ Bijectors.inverse(f::FromVec{Tuple{}}) = tovec VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) VarNameVector(vns, vals) = VarNameVector(collect(vns), collect(vals)) function VarNameVector( - vns::AbstractVector, - vals::AbstractVector, - transforms = map(FromVec, vals) + vns::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) ) # TODO: Check uniqueness of `vns`? @@ -107,7 +105,7 @@ Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] Base.setindex!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val function Base.setindex!(vnv::VarNameVector, val, vn::VarName) f = inverse(gettransform(vnv, vn)) - vnv.vals[getrange(vnv, vn)] = f(val) + return vnv.vals[getrange(vnv, vn)] = f(val) end function Base.empty!(vnv::VarNameVector) @@ -146,7 +144,7 @@ function group_by_symbol(vnv::VarNameVector) VarNameVector( map(identity, vns), map(Base.Fix1(getindex, vnv), vns), - map(Base.Fix1(gettransform, vnv), vns) + map(Base.Fix1(gettransform, vnv), vns), ) end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 22b8c817b..158eca99d 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,14 +1,6 @@ @testset "VarNameVector" begin - vns = [ - @varname(x[1]), - @varname(x[2]), - @varname(x[3]), - ] - vals = [ - 1, - 2:3, - reshape(4:9, 2, 3), - ] + vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] + vals = [1, 2:3, reshape(4:9, 2, 3)] vnv = VarNameVector(vns, vals) # `getindex` From 7cd599d908b2a444461f024feb1ddf15e6743bd8 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:46:59 +0000 Subject: [PATCH 004/182] added arbitrary metadata field as discussed --- src/varnamevector.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index dd55fb7c0..b1949a17c 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -5,6 +5,7 @@ struct VarNameVector{ TVN<:AbstractVector{<:VarName}, TVal<:AbstractVector, TTrans<:AbstractVector, + MData } "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" idcs::TIdcs # Dict{<:VarName,Int} @@ -20,6 +21,13 @@ struct VarNameVector{ "vector of transformations whose inverse takes us back to the original space" transforms::TTrans + + "metadata associated with the varnames" + metadata::MData +end + +function VarNameVector(idcs, vns, ranges, vals, transforms) + return VarNameVector(idcs, vns, ranges, vals, transforms, nothing) end # Useful transformation going from the flattened representation. @@ -117,7 +125,7 @@ function Base.empty!(vnv::VarNameVector) empty!(vnv.transforms) return nothing end -BangBang.empty!!(vnv::VarNameVector) = empty!(vnv) +BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) # TODO: Re-use some of the show functionality from Base? function Base.show(io::IO, vnv::VarNameVector) From ed0a757921590c2d90120ada02f4cfd34aef826d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:47:45 +0000 Subject: [PATCH 005/182] renamed idcs to varname_to_index --- src/varnamevector.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index b1949a17c..f71300f33 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -8,15 +8,15 @@ struct VarNameVector{ MData } "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" - idcs::TIdcs # Dict{<:VarName,Int} + varname_to_index::TIdcs # Dict{<:VarName,Int} - "vector of identifiers for the random variables, where `vns[idcs[vn]] == vn`" + "vector of identifiers for the random variables, where `vns[varname_to_index[vn]] == vn`" vns::TVN # AbstractVector{<:VarName} "vector of index ranges in `vals` corresponding to `vns`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" ranges::Vector{UnitRange{Int}} - "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[idcs[vn]]]`" + "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" vals::TVal # AbstractVector{<:Real} "vector of transformations whose inverse takes us back to the original space" @@ -26,8 +26,8 @@ struct VarNameVector{ metadata::MData end -function VarNameVector(idcs, vns, ranges, vals, transforms) - return VarNameVector(idcs, vns, ranges, vals, transforms, nothing) +function VarNameVector(varname_to_index, vns, ranges, vals, transforms) + return VarNameVector(varname_to_index, vns, ranges, vals, transforms, nothing) end # Useful transformation going from the flattened representation. @@ -63,12 +63,12 @@ function VarNameVector( if !(eltype(vns) <: VarName) vns = convert(Vector{VarName}, vns) end - idcs = OrderedDict{eltype(vns),Int}() + varname_to_index = OrderedDict{eltype(vns),Int}() ranges = Vector{UnitRange{Int}}() offset = 0 for (i, (vn, x)) in enumerate(zip(vns, vals_vecs)) # Add the varname index. - push!(idcs, vn => length(idcs) + 1) + push!(varname_to_index, vn => length(varname_to_index) + 1) # Add the range. r = (offset + 1):(offset + length(x)) push!(ranges, r) @@ -76,7 +76,7 @@ function VarNameVector( offset = r[end] end - return VarNameVector(idcs, vns, ranges, reduce(vcat, vals_vecs), transforms) + return VarNameVector(varname_to_index, vns, ranges, reduce(vcat, vals_vecs), transforms) end # Basic array interface. @@ -89,10 +89,10 @@ Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. Base.keys(vnv::VarNameVector) = vnv.vns -Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.idcs, vn) +Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) # `getindex` & `setindex!` -getidx(vnv::VarNameVector, vn::VarName) = vnv.idcs[vn] +getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) @@ -118,7 +118,7 @@ end function Base.empty!(vnv::VarNameVector) # TODO: Or should the semantics be different, e.g. keeping `vns`? - empty!(vnv.idcs) + empty!(vnv.varname_to_index) empty!(vnv.vns) empty!(vnv.ranges) empty!(vnv.vals) From 4ebd252ce46b40aa2c8a0daaf39da9f154b15c8b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:48:31 +0000 Subject: [PATCH 006/182] renamed vns to varnames for VarNameVector --- src/varnamevector.jl | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f71300f33..2c111feb2 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -7,13 +7,13 @@ struct VarNameVector{ TTrans<:AbstractVector, MData } - "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" + "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" varname_to_index::TIdcs # Dict{<:VarName,Int} - "vector of identifiers for the random variables, where `vns[varname_to_index[vn]] == vn`" - vns::TVN # AbstractVector{<:VarName} + "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" + varnames::TVN # AbstractVector{<:VarName} - "vector of index ranges in `vals` corresponding to `vns`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" + "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" ranges::Vector{UnitRange{Int}} "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" @@ -26,8 +26,8 @@ struct VarNameVector{ metadata::MData end -function VarNameVector(varname_to_index, vns, ranges, vals, transforms) - return VarNameVector(varname_to_index, vns, ranges, vals, transforms, nothing) +function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) + return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, nothing) end # Useful transformation going from the flattened representation. @@ -50,23 +50,23 @@ Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) -VarNameVector(vns, vals) = VarNameVector(collect(vns), collect(vals)) +VarNameVector(varnames, vals) = VarNameVector(collect(varnames), collect(vals)) function VarNameVector( - vns::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) + varnames::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) ) - # TODO: Check uniqueness of `vns`? + # TODO: Check uniqueness of `varnames`? # Convert `vals` into a vector of vectors. vals_vecs = map(tovec, vals) # TODO: Is this really the way to do this? - if !(eltype(vns) <: VarName) - vns = convert(Vector{VarName}, vns) + if !(eltype(varnames) <: VarName) + varnames = convert(Vector{VarName}, varnames) end - varname_to_index = OrderedDict{eltype(vns),Int}() + varname_to_index = OrderedDict{eltype(varnames),Int}() ranges = Vector{UnitRange{Int}}() offset = 0 - for (i, (vn, x)) in enumerate(zip(vns, vals_vecs)) + for (i, (vn, x)) in enumerate(zip(varnames, vals_vecs)) # Add the varname index. push!(varname_to_index, vn => length(varname_to_index) + 1) # Add the range. @@ -76,7 +76,7 @@ function VarNameVector( offset = r[end] end - return VarNameVector(varname_to_index, vns, ranges, reduce(vcat, vals_vecs), transforms) + return VarNameVector(varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms) end # Basic array interface. @@ -87,7 +87,7 @@ Base.size(vnv::VarNameVector) = size(vnv.vals) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. -Base.keys(vnv::VarNameVector) = vnv.vns +Base.keys(vnv::VarNameVector) = vnv.varnames Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) @@ -117,9 +117,9 @@ function Base.setindex!(vnv::VarNameVector, val, vn::VarName) end function Base.empty!(vnv::VarNameVector) - # TODO: Or should the semantics be different, e.g. keeping `vns`? + # TODO: Or should the semantics be different, e.g. keeping `varnames`? empty!(vnv.varname_to_index) - empty!(vnv.vns) + empty!(vnv.varnames) empty!(vnv.ranges) empty!(vnv.vals) empty!(vnv.transforms) @@ -130,7 +130,7 @@ BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) # TODO: Re-use some of the show functionality from Base? function Base.show(io::IO, vnv::VarNameVector) print(io, "[") - for (i, vn) in enumerate(vnv.vns) + for (i, vn) in enumerate(vnv.varnames) if i > 1 print(io, ", ") end @@ -142,17 +142,17 @@ end function group_by_symbol(vnv::VarNameVector) # Group varnames in `vnv` by the symbol. d = OrderedDict{Symbol,Vector{VarName}}() - for vn in vnv.vns + for vn in vnv.varnames push!(get!(d, getsym(vn), Vector{VarName}()), vn) end # Create a `NamedTuple` from the grouped varnames. - nt_vals = map(values(d)) do vns + nt_vals = map(values(d)) do varnames # TODO: Do we need to specialize the inputs here? VarNameVector( - map(identity, vns), - map(Base.Fix1(getindex, vnv), vns), - map(Base.Fix1(gettransform, vnv), vns), + map(identity, varnames), + map(Base.Fix1(getindex, vnv), varnames), + map(Base.Fix1(gettransform, vnv), varnames), ) end From 9f12c9ac4ee38b3742bb7c807d4957294fa8ce5f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:58:02 +0000 Subject: [PATCH 007/182] added keys impl for Metadata --- src/varinfo.jl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index df1be3d21..7551dc3fc 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -876,10 +876,9 @@ end return expr end -# Functions defined only for UntypedVarInfo -Base.keys(vi::UntypedVarInfo) = keys(vi.metadata.idcs) - -Base.keys(vi::VectorVarInfo) = keys(vi.metadata) +# `keys` +Base.keys(md::Metadata) = keys(md.idcs) +Base.keys(vi::VarInfo) = keys(vi.metadata) # HACK: Necessary to avoid returning `Any[]` which won't dispatch correctly # on other methods in the codebase which requires `Vector{<:VarName}`. @@ -889,7 +888,7 @@ Base.keys(vi::TypedVarInfo{<:NamedTuple{()}}) = VarName[] push!(expr.args, :vcat) for n in names - push!(expr.args, :(vi.metadata.$n.vns)) + push!(expr.args, :(keys(vi.metadata.$n))) end return expr @@ -1569,7 +1568,7 @@ const _MAX_VARS_SHOWN = 4 function _show_varnames(io::IO, vi) md = vi.metadata - vns = md.vns + vns = keys(md) vns_by_name = Dict{Symbol,Vector{VarName}}() for vn in vns From 5a151219f77339c5b6aa939f805077f0bb0605ee Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:07:20 +0000 Subject: [PATCH 008/182] added push! and update! for VarNameVector --- src/varnamevector.jl | 111 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 2c111feb2..a7acba79f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -22,12 +22,15 @@ struct VarNameVector{ "vector of transformations whose inverse takes us back to the original space" transforms::TTrans + "inactive ranges" + inactive_ranges::Vector{UnitRange{Int}} + "metadata associated with the varnames" metadata::MData end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) - return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, nothing) + return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing) end # Useful transformation going from the flattened representation. @@ -127,15 +130,107 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) -# TODO: Re-use some of the show functionality from Base? -function Base.show(io::IO, vnv::VarNameVector) - print(io, "[") - for (i, vn) in enumerate(vnv.varnames) - if i > 1 - print(io, ", ") +function nextrange(vnd::VarNameVector, x) + n = length(vnd) + return n + 1:n + length(x) +end + +# `push!` and `push!!`: add a variable to the varname vector. +function push!( + vnv::VarNameVector, + vn::VarName, + val, + transform=FromVec(val), +) + # Error if we already have the variable. + haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) + return update!(vnv, vn, val, transform) +end + +# `update!` and `update!!`: update a variable in the varname vector. +function update!( + vnv::VarNameVector, + vn::VarName, + val, + transform=FromVec(val), +) + val_vec = tovec(val) + if !haskey(vnv, vn) + # Here we just add a new entry. + vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 + push!(vnv.varnames, vn) + push!(vnv.ranges, nextrange(vnv, val_vec)) + append!(vnv.vals, val_vec) + push!(vnv.transforms, transform) + else + # Here we update the existing entry. + idx = getidx(vnv, vn) + r_old = getrange(vnv, idx) + n_old = length(r_old) + n_new = length(val_vec) + # Existing keys needs to be handled differently depending on + # whether the size of the value is increasing or decreasing. + if n_new > n_old + # Remove the old range. + delete!(vnv.ranges, vn) + # Add the new range. + r_new = nextrange(vnv, val_vec) + vnv.varname_to_ranges[vn] = r_new + # Grow the underlying vector to accomodate the new value. + resize!(vnv.vals, r_new[end]) + # Keep track of the deleted ranges. + push!(vnv.inactive_ranges, r_old) + else + # `n_new <= n_old` + # Just decrease the current range. + r_new = r_old[1]:(r_old[1] + n_new - 1) + vnv.ranges[idx] = r_new + # And mark the rest as inactive if needed. + if n_new < n_old + push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) + end end - print(io, vn, " = ", vnv[vn]) + + # Update the value. + vnv.vals[r_new] = val_vec + # Update the transform. + vnv.transforms[idx] = transform + + # TODO: Should we maybe sweep over inactive ranges and re-contiguify + # if we the total number of inactive elements is "large" in some sense? + end + + return vnv +end + +function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) + offset = 0 + # NOTE: assumes `ranges` are ordered. + for i = 1:length(ranges) + r_old = ranges[i] + ranges[i] = offset + 1:offset + length(r_old) + offset += length(r_old) + end + + return ranges +end + +function inactive_ranges_sweep!(vnv::VarNameVector) + # Extract the re-contiguified values. + # NOTE: We need to do this before we update the ranges. + vals = vnv[:] + # And then we re-contiguify the ranges. + recontiguify_ranges!(vnv.ranges) + # Clear the inactive ranges. + empty!(vnv.inactive_ranges) + # Now we update the values. + for (i, r) in enumerate(vnv.ranges) + vnv.vals[r] = vals[r] end + # And (potentially) shrink the underlying vector. + resize!(vnv.vals, vnv.ranges[end][end]) + # The rest should be left as is. + return vnv end # Typed version. From edde2c1704ba54de4f54483a357b824db828cd34 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:08:03 +0000 Subject: [PATCH 009/182] added getindex_raw! and setindex_raw! for VarNameVector --- src/varnamevector.jl | 53 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a7acba79f..f4abf6e4a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -82,6 +82,16 @@ function VarNameVector( return VarNameVector(varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms) end +# Some `VarNameVector` specific functions. +getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] + +getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] +getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) + +gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] + +has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) + # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) Base.length(vnv::VarNameVector) = length(vnv.vals) @@ -95,28 +105,47 @@ Base.keys(vnv::VarNameVector) = vnv.varnames Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) # `getindex` & `setindex!` -getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] - -getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] -getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) - -gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] - -Base.getindex(vnv::VarNameVector, ::Colon) = vnv.vals -Base.getindex(vnv::VarNameVector, i::Int) = vnv.vals[i] +Base.getindex(vnv::VarNameVector, i::Int) = getindex_raw(vnv, i) function Base.getindex(vnv::VarNameVector, vn::VarName) - x = vnv.vals[getrange(vnv, vn)] + x = getindex_raw(vnv, vn) f = gettransform(vnv, vn) return f(x) end +getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[i] +function getindex_raw(vnv::VarNameVector, vn::VarName) + return vnv.vals[getrange(vnv, vn)] +end + +# `getindex` for `Colon` +function Base.getindex(vnv::VarNameVector, ::Colon) + return if has_inactive_ranges(vnv) + mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) + else + vnv.vals + end +end + +function getindex_raw(vnv::VarNameVector, ::Colon) + return if has_inactive_ranges(vnv) + mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) + else + vnv.vals + end +end + # HACK: remove this as soon as possible. Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] -Base.setindex!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +Base.setindex!(vnv::VarNameVector, val, i::Int) = setindex_raw!(vnv, val, i) function Base.setindex!(vnv::VarNameVector, val, vn::VarName) f = inverse(gettransform(vnv, vn)) - return vnv.vals[getrange(vnv, vn)] = f(val) + return setindex_raw!(vnv, f(val), vn) +end + +setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) + return vnv.vals[getrange(vnv, vn)] = val end function Base.empty!(vnv::VarNameVector) From ed460023a8d002e27eb0a14077ba34d83932a614 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:08:22 +0000 Subject: [PATCH 010/182] added `iterate` and `convert` (for `AbstractDict) impls for `VarNameVector` --- src/varnamevector.jl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f4abf6e4a..279e26b2a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -282,3 +282,19 @@ function group_by_symbol(vnv::VarNameVector) return NamedTuple{Tuple(keys(d))}(nt_vals) end + +# `iterate` +function Base.iterate(vnv::VarNameVector, state=nothing) + res = state === nothing ? iterate(vnv.varname_to_index) : iterate(vnv.varname_to_index, state) + res === nothing && return nothing + (vn, idx), state_new = res + return vn => vnv.vals[getrange(vnv, idx)], state_new +end + +# `convert` +function Base.convert(::Type{D}, vnv::VarNameVector) where {D<:AbstractDict} + return ConstructionBase.constructorof(D)( + keys(vnv.varname_to_index), + map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)) + ) +end From 5b00059ee605499ce8861b0c70e31391c3f55255 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:09:34 +0000 Subject: [PATCH 011/182] make the key and eltype part of the `VarNameVector` type --- src/varnamevector.jl | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 279e26b2a..30bcccd82 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,14 +1,15 @@ # Similar to `Metadata` but representing a `Vector` and simpler interface. -# TODO: Should we subtype `AbstractVector`? +# TODO: Should we subtype `AbstractVector` or `AbstractDict`? struct VarNameVector{ - TIdcs<:OrderedDict{<:VarName,Int}, - TVN<:AbstractVector{<:VarName}, - TVal<:AbstractVector, + K, + V, + TVN<:AbstractVector{K}, + TVal<:AbstractVector{V}, TTrans<:AbstractVector, MData } "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" - varname_to_index::TIdcs # Dict{<:VarName,Int} + varname_to_index::OrderedDict{K,Int} "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" varnames::TVN # AbstractVector{<:VarName} @@ -32,6 +33,8 @@ end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing) end +# TODO: Do we need this? +VarNameVector{K,V}() where {K,V} = VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) # Useful transformation going from the flattened representation. struct FromVec{Sz} @@ -52,8 +55,12 @@ tovec(x::AbstractArray) = vec(x) Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec +collect_maybe(x) = collect(x) +collect_maybe(x::AbstractArray) = x + +VarNameVector() = VarNameVector{VarName,Real}() VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) -VarNameVector(varnames, vals) = VarNameVector(collect(varnames), collect(vals)) +VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) function VarNameVector( varnames::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) ) @@ -79,7 +86,13 @@ function VarNameVector( offset = r[end] end - return VarNameVector(varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms) + return VarNameVector( + varname_to_index, + varnames, + ranges, + reduce(vcat, vals_vecs), + transforms + ) end # Some `VarNameVector` specific functions. From bef7e0a74ba270bba713f94833597ae4dd61c07a Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:25:38 +0000 Subject: [PATCH 012/182] added more tests for VarNameVector --- test/varnamevector.jl | 108 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 12 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 158eca99d..ed225c7bb 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,19 +1,103 @@ +replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) + @testset "VarNameVector" begin - vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] - vals = [1, 2:3, reshape(4:9, 2, 3)] - vnv = VarNameVector(vns, vals) + # Need to test element-related operations: + # - `getindex` + # - `setindex!` + # - `push!` + # - `update!` + # + # And these should all be tested for different types of values: + # - scalar + # - vector + # - matrix - # `getindex` - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val - end + # Need to test operations on `VarNameVector`: + # - `empty!` + # - `iterate` + # - `convert` to + # - `AbstractDict` + + test_pairs = OrderedDict( + @varname(x[1]) => rand(), + @varname(x[2]) => rand(2), + @varname(x[3]) => rand(2, 3), + @varname(x[4]) => rand(2, 3, 4) + ) - # `setindex!` - for (vn, val) in zip(vns, vals) - vnv[vn] = val .+ 100 + @testset "constructor" begin + @testset "no args" begin + # Empty. + vnv = VarNameVector() + @test isempty(vnv) + @test eltype(vnv) == Real + + # Empty with types. + vnv = VarNameVector{VarName,Float64}() + @test isempty(vnv) + @test eltype(vnv) == Float64 + end + + # Should be able to handle different types of values. + @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in Iterators.product( + keys(test_pairs), keys(test_pairs) + ) + val_left = test_pairs[vn_left] + val_right = test_pairs[vn_right] + vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) + @test length(vnv) == length(val_left) + length(val_right) + @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + end + + # Should also work when mixing varnames with different symbols. + @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in Iterators.product( + keys(test_pairs), keys(test_pairs) + ) + val_left = test_pairs[vn_left] + val_right = test_pairs[vn_right] + vnv = VarNameVector([vn_left, replace_sym(vn_right, :y)], [val_left, val_right]) + @test length(vnv) == length(val_left) + length(val_right) + @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + end end - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val .+ 100 + @testset "basics" begin + vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] + vals = [1, 2:3, reshape(4:9, 2, 3)] + vnv = VarNameVector(vns, vals) + + # `getindex` + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val + end + + # `setindex!` + for (vn, val) in zip(vns, vals) + vnv[vn] = val .+ 100 + end + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val .+ 100 + end + + # `push!` + vn = @varname(x[4]) + val = 10:12 + push!(vnv, vn, val) + @test vnv[vn] == val + + # `push!` existing varname is not allowed + @test_throws ArgumentError push!(vnv, vn, val) + + # `update!` works with both existing and new varname + # existing + val = 20:22 + DynamicPPL.update!(vnv, vn, val) + @test vnv[vn] == val + + # new + vn = @varname(x[5]) + val = 30:32 + DynamicPPL.update!(vnv, vn, val) + @test vnv[vn] == val end end From 006ee8dfb815299da0948db8634b8dfa016fd3be Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:26:38 +0000 Subject: [PATCH 013/182] formatting --- src/varnamevector.jl | 49 +++++++++++++++++-------------------------- test/varnamevector.jl | 5 +++-- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 30bcccd82..bb553835f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,12 +1,7 @@ # Similar to `Metadata` but representing a `Vector` and simpler interface. # TODO: Should we subtype `AbstractVector` or `AbstractDict`? struct VarNameVector{ - K, - V, - TVN<:AbstractVector{K}, - TVal<:AbstractVector{V}, - TTrans<:AbstractVector, - MData + K,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData } "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" varname_to_index::OrderedDict{K,Int} @@ -31,10 +26,14 @@ struct VarNameVector{ end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) - return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing) + return VarNameVector( + varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing + ) end # TODO: Do we need this? -VarNameVector{K,V}() where {K,V} = VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) +function VarNameVector{K,V}() where {K,V} + return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) +end # Useful transformation going from the flattened representation. struct FromVec{Sz} @@ -87,11 +86,7 @@ function VarNameVector( end return VarNameVector( - varname_to_index, - varnames, - ranges, - reduce(vcat, vals_vecs), - transforms + varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms ) end @@ -174,28 +169,18 @@ BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) function nextrange(vnd::VarNameVector, x) n = length(vnd) - return n + 1:n + length(x) + return (n + 1):(n + length(x)) end # `push!` and `push!!`: add a variable to the varname vector. -function push!( - vnv::VarNameVector, - vn::VarName, - val, - transform=FromVec(val), -) +function push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) return update!(vnv, vn, val, transform) end # `update!` and `update!!`: update a variable in the varname vector. -function update!( - vnv::VarNameVector, - vn::VarName, - val, - transform=FromVec(val), -) +function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) val_vec = tovec(val) if !haskey(vnv, vn) # Here we just add a new entry. @@ -248,9 +233,9 @@ end function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) offset = 0 # NOTE: assumes `ranges` are ordered. - for i = 1:length(ranges) + for i in 1:length(ranges) r_old = ranges[i] - ranges[i] = offset + 1:offset + length(r_old) + ranges[i] = (offset + 1):(offset + length(r_old)) offset += length(r_old) end @@ -298,7 +283,11 @@ end # `iterate` function Base.iterate(vnv::VarNameVector, state=nothing) - res = state === nothing ? iterate(vnv.varname_to_index) : iterate(vnv.varname_to_index, state) + res = if state === nothing + iterate(vnv.varname_to_index) + else + iterate(vnv.varname_to_index, state) + end res === nothing && return nothing (vn, idx), state_new = res return vn => vnv.vals[getrange(vnv, idx)], state_new @@ -308,6 +297,6 @@ end function Base.convert(::Type{D}, vnv::VarNameVector) where {D<:AbstractDict} return ConstructionBase.constructorof(D)( keys(vnv.varname_to_index), - map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)) + map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)), ) end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index ed225c7bb..2d4374610 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -22,7 +22,7 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), - @varname(x[4]) => rand(2, 3, 4) + @varname(x[4]) => rand(2, 3, 4), ) @testset "constructor" begin @@ -50,7 +50,8 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) end # Should also work when mixing varnames with different symbols. - @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in Iterators.product( + @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in + Iterators.product( keys(test_pairs), keys(test_pairs) ) val_left = test_pairs[vn_left] From 9802811b5713193468222e1dbaf54708cf7ec4fd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 16:23:29 +0000 Subject: [PATCH 014/182] more testing for VarNameVector --- src/varnamevector.jl | 10 +++ test/Project.toml | 2 + test/runtests.jl | 2 + test/varnamevector.jl | 152 +++++++++++++++++++++++++----------------- 4 files changed, 104 insertions(+), 62 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index bb553835f..6db719a0c 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -25,6 +25,16 @@ struct VarNameVector{ metadata::MData end +function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) + return vnv_left.varname_to_index == vnv_right.varname_to_index && + vnv_left.varnames == vnv_right.varnames && + vnv_left.ranges == vnv_right.ranges && + vnv_left.vals == vnv_right.vals && + vnv_left.transforms == vnv_right.transforms && + vnv_left.inactive_ranges == vnv_right.inactive_ranges && + vnv_left.metadata == vnv_right.metadata +end + function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) return VarNameVector( varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing diff --git a/test/Project.toml b/test/Project.toml index ed8ae7971..c8e79f859 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -25,6 +26,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" AbstractMCMC = "2.1, 3.0, 4" AbstractPPL = "0.6" Bijectors = "0.13" +Combinatorics = "1" Compat = "4.3.0" Distributions = "0.25" DistributionsAD = "0.6.3" diff --git a/test/runtests.jl b/test/runtests.jl index 43d68386c..d15b17b42 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,6 +20,8 @@ using Random using Serialization using Test +using Combinatorics: combinations + using DynamicPPL: getargs_dottilde, getargs_tilde, Selector const DIRECTORY_DynamicPPL = dirname(dirname(pathof(DynamicPPL))) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 2d4374610..0f62e9f97 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -18,87 +18,115 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) # - `convert` to # - `AbstractDict` - test_pairs = OrderedDict( + test_pairs_x = OrderedDict( @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), @varname(x[4]) => rand(2, 3, 4), ) + test_pairs_y = OrderedDict( + @varname(y[1]) => rand(), + @varname(y[2]) => rand(2), + @varname(y[3]) => rand(2, 3), + @varname(y[4]) => rand(2, 3, 4), + ) + test_pairs = merge(test_pairs_x, test_pairs_y) - @testset "constructor" begin - @testset "no args" begin - # Empty. - vnv = VarNameVector() - @test isempty(vnv) - @test eltype(vnv) == Real + @testset "constructor: no args" begin + # Empty. + vnv = VarNameVector() + @test isempty(vnv) + @test eltype(vnv) == Real - # Empty with types. - vnv = VarNameVector{VarName,Float64}() - @test isempty(vnv) - @test eltype(vnv) == Float64 - end + # Empty with types. + vnv = VarNameVector{VarName,Float64}() + @test isempty(vnv) + @test eltype(vnv) == Float64 + end - # Should be able to handle different types of values. - @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in Iterators.product( - keys(test_pairs), keys(test_pairs) - ) - val_left = test_pairs[vn_left] - val_right = test_pairs[vn_right] - vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) - @test length(vnv) == length(val_left) + length(val_right) - @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) - end + test_varnames_iter = combinations(collect(keys(test_pairs)), 2) + @info "Testing varnames" collect( + map(Base.Fix1(convert, Vector{VarName}), test_varnames_iter) + ) + @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter + val_left = test_pairs[vn_left] + val_right = test_pairs[vn_right] + vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) - # Should also work when mixing varnames with different symbols. - @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in - Iterators.product( - keys(test_pairs), keys(test_pairs) + # Compare to alternative constructors. + vnv_from_dict = VarNameVector( + OrderedDict(vn_left => val_left, vn_right => val_right) ) - val_left = test_pairs[vn_left] - val_right = test_pairs[vn_right] - vnv = VarNameVector([vn_left, replace_sym(vn_right, :y)], [val_left, val_right]) - @test length(vnv) == length(val_left) + length(val_right) - @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + @test vnv == vnv_from_dict + + # We want the types of fields such as `varnames` and `transforms` to specialize + # whenever possible + some functionality, e.g. `push!`, is only sensible + # if the underlying containers can support it. + # Expected behavior + should_have_restricted_varname_type = typeof(vn_left) == typeof(vn_right) + should_have_restricted_transform_type = size(val_left) == size(val_right) + # Actual behavior + has_restricted_transform_type = isconcretetype(eltype(vnv.transforms)) + has_restricted_varname_type = isconcretetype(eltype(vnv.varnames)) + + @testset "type specialization" begin + @test !should_have_restricted_varname_type || has_restricted_varname_type + @test !should_have_restricted_transform_type || + has_restricted_transform_type end - end - @testset "basics" begin - vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] - vals = [1, 2:3, reshape(4:9, 2, 3)] - vnv = VarNameVector(vns, vals) + # `eltype` + @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + # `length` + @test length(vnv) == length(val_left) + length(val_right) # `getindex` - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val + @testset "getindex" begin + # `getindex` + @test vnv[vn_left] == val_left + @test vnv[vn_right] == val_right end - # `setindex!` - for (vn, val) in zip(vns, vals) - vnv[vn] = val .+ 100 - end - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val .+ 100 + @testset "setindex!" begin + vnv[vn_left] = val_left .+ 100 + @test vnv[vn_left] == val_left .+ 100 + vnv[vn_right] = val_right .+ 100 + @test vnv[vn_right] == val_right .+ 100 end # `push!` - vn = @varname(x[4]) - val = 10:12 - push!(vnv, vn, val) - @test vnv[vn] == val - - # `push!` existing varname is not allowed - @test_throws ArgumentError push!(vnv, vn, val) - - # `update!` works with both existing and new varname - # existing - val = 20:22 - DynamicPPL.update!(vnv, vn, val) - @test vnv[vn] == val + # `push!` and `update!` are only allowed for all the varnames if both the + # varname and the transform types used in the underlying containers are not concrete. + push_test_varnames = filter(keys(test_pairs)) do vn + val = test_pairs[vn] + transform_is_okay = + !has_restricted_transform_type || + size(val) == size(val_left) || + size(val) == size(val_right) + varname_is_okay = + !has_restricted_varname_type || + typeof(vn) == typeof(vn_left) || + typeof(vn) == typeof(vn_right) + return transform_is_okay && varname_is_okay + end + @testset "push! ($(vn))" for vn in push_test_varnames + val = test_pairs[vn] + if vn == vn_left || vn == vn_right + # Should not be possible to `push!` existing varname. + @test_throws ArgumentError push!(vnv, vn, val) + else + push!(vnv, vn, val) + @test vnv[vn] == val + end + end - # new - vn = @varname(x[5]) - val = 30:32 - DynamicPPL.update!(vnv, vn, val) - @test vnv[vn] == val + # `update!` + @testset "update! ($(vn))" for vn in push_test_varnames + val = test_pairs[vn] + # Perturb `val` a bit so we can also check that the existing `vn_left` and `vn_right` + # are also updated correctly. + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + end end end From 88b1721b8350a42df9239ebb9f184b1aa0e4aff4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 16:25:01 +0000 Subject: [PATCH 015/182] minor changes to some comments --- test/varnamevector.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 0f62e9f97..293f92e3e 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -94,9 +94,9 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @test vnv[vn_right] == val_right .+ 100 end - # `push!` - # `push!` and `update!` are only allowed for all the varnames if both the - # varname and the transform types used in the underlying containers are not concrete. + # `push!` & `update!` + # These are only allowed for all the varnames if both the varname and + # the transform types used in the underlying containers are not concrete. push_test_varnames = filter(keys(test_pairs)) do vn val = test_pairs[vn] transform_is_okay = @@ -119,8 +119,6 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @test vnv[vn] == val end end - - # `update!` @testset "update! ($(vn))" for vn in push_test_varnames val = test_pairs[vn] # Perturb `val` a bit so we can also check that the existing `vn_left` and `vn_right` From ca7b173b4273e24497d8d0412b4d3e4d5edfb54d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:28:16 +0000 Subject: [PATCH 016/182] added a bunch more tests for VarNameVector + several bugfixes in the process --- src/varnamevector.jl | 40 ++++----- test/varnamevector.jl | 202 +++++++++++++++++++++++++++++++++--------- 2 files changed, 179 insertions(+), 63 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 6db719a0c..c696aa20e 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -68,6 +68,7 @@ collect_maybe(x) = collect(x) collect_maybe(x::AbstractArray) = x VarNameVector() = VarNameVector{VarName,Real}() +VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) function VarNameVector( @@ -112,8 +113,12 @@ has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) -Base.length(vnv::VarNameVector) = length(vnv.vals) -Base.size(vnv::VarNameVector) = size(vnv.vals) +Base.length(vnv::VarNameVector) = if isempty(vnv.inactive_ranges) + length(vnv.vals) +else + sum(length, vnv.ranges) +end +Base.size(vnv::VarNameVector) = (length(vnv),) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() @@ -178,12 +183,12 @@ end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) function nextrange(vnd::VarNameVector, x) - n = length(vnd) + n = maximum(map(last, vnd.ranges)) return (n + 1):(n + length(x)) end # `push!` and `push!!`: add a variable to the varname vector. -function push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) +function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) return update!(vnv, vn, val, transform) @@ -194,9 +199,12 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) val_vec = tovec(val) if !haskey(vnv, vn) # Here we just add a new entry. + # NOTE: We need to compute the `nextrange` BEFORE we start mutating + # the underlying; otherwise we might get some strange behaviors. + r_new = nextrange(vnv, val_vec) vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 push!(vnv.varnames, vn) - push!(vnv.ranges, nextrange(vnv, val_vec)) + push!(vnv.ranges, r_new) append!(vnv.vals, val_vec) push!(vnv.transforms, transform) else @@ -208,11 +216,9 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Existing keys needs to be handled differently depending on # whether the size of the value is increasing or decreasing. if n_new > n_old - # Remove the old range. - delete!(vnv.ranges, vn) # Add the new range. r_new = nextrange(vnv, val_vec) - vnv.varname_to_ranges[vn] = r_new + vnv.ranges[idx] = r_new # Grow the underlying vector to accomodate the new value. resize!(vnv.vals, r_new[end]) # Keep track of the deleted ranges. @@ -242,7 +248,6 @@ end function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) offset = 0 - # NOTE: assumes `ranges` are ordered. for i in 1:length(ranges) r_old = ranges[i] ranges[i] = (offset + 1):(offset + length(r_old)) @@ -292,21 +297,14 @@ function group_by_symbol(vnv::VarNameVector) end # `iterate` +# TODO: Maybe implement `iterate` as a vector and then instead implement `pairs`. function Base.iterate(vnv::VarNameVector, state=nothing) res = if state === nothing - iterate(vnv.varname_to_index) + iterate(vnv.varnames) else - iterate(vnv.varname_to_index, state) + iterate(vnv.varnames, state) end res === nothing && return nothing - (vn, idx), state_new = res - return vn => vnv.vals[getrange(vnv, idx)], state_new -end - -# `convert` -function Base.convert(::Type{D}, vnv::VarNameVector) where {D<:AbstractDict} - return ConstructionBase.constructorof(D)( - keys(vnv.varname_to_index), - map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)), - ) + vn, state_new = res + return vn => getindex(vnv, vn), state_new end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 293f92e3e..c0c542c70 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,5 +1,75 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) +change_size_for_test(x::Real) = [x] +change_size_for_test(x::AbstractArray) = repeat(x, 2) + +function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.varnames)) + # If the container is concrete, we need to make sure that the varname types match. + # E.g. if `vnv.varnames` has `eltype` `VarName{:x, IndexLens{Tuple{Int64}}}` then + # we need `vn` to also be of this type. + # => If the varname types don't match, we need to relax the container type. + return any(keys(vnv)) do vn_present + typeof(vn_present) !== typeof(val) + end + end + + return false +end + +function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.vals)) + return promote_type(eltype(vnv.vals), eltype(val)) != eltype(vnv.vals) + end + + return false +end + +function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.transforms)) + # If the container is concrete, we need to make sure that the sizes match. + # => If the sizes don't match, we need to relax the container type. + return any(keys(vnv)) do vn_present + size(vnv[vn_present]) != size(val) + end + end + + return false +end + +relax_container_types(vnv::VarNameVector, vn::VarName, val) = relax_container_types(vnv, [vn], [val]) +function relax_container_types(vnv::VarNameVector, vns, vals) + if any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) + varnames_new = convert(Vector{VarName}, vnv.varnames) + else + varname_to_index_new = vnv.varname_to_index + varnames_new = vnv.varnames + end + + transforms_new = if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + convert(Vector{Any}, vnv.transforms) + else + vnv.transforms + end + + vals_new = if any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + convert(Vector{Any}, vnv.vals) + else + vnv.vals + end + + return VarNameVector( + varname_to_index_new, + varnames_new, + vnv.ranges, + vals_new, + transforms_new, + vnv.inactive_ranges, + vnv.metadata + ) +end + @testset "VarNameVector" begin # Need to test element-related operations: # - `getindex` @@ -18,19 +88,19 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) # - `convert` to # - `AbstractDict` - test_pairs_x = OrderedDict( + test_pairs = OrderedDict( @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), - @varname(x[4]) => rand(2, 3, 4), - ) - test_pairs_y = OrderedDict( + @varname(y[1]) => rand(), @varname(y[2]) => rand(2), @varname(y[3]) => rand(2, 3), - @varname(y[4]) => rand(2, 3, 4), + + @varname(z[1]) => rand(1:10), + @varname(z[2]) => rand(1:10, 2), + @varname(z[3]) => rand(1:10, 2, 3), ) - test_pairs = merge(test_pairs_x, test_pairs_y) @testset "constructor: no args" begin # Empty. @@ -51,13 +121,20 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] - vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) + vnv_base = VarNameVector([vn_left, vn_right], [val_left, val_right]) + + # We'll need the transformations later. + # TODO: Should we test other transformations than just `FromVec`? + from_vec_left = DynamicPPL.FromVec(val_left) + from_vec_right = DynamicPPL.FromVec(val_right) + to_vec_left = inverse(from_vec_left) + to_vec_right = inverse(from_vec_right) # Compare to alternative constructors. vnv_from_dict = VarNameVector( OrderedDict(vn_left => val_left, vn_right => val_right) ) - @test vnv == vnv_from_dict + @test vnv_base == vnv_from_dict # We want the types of fields such as `varnames` and `transforms` to specialize # whenever possible + some functionality, e.g. `push!`, is only sensible @@ -66,8 +143,8 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) should_have_restricted_varname_type = typeof(vn_left) == typeof(vn_right) should_have_restricted_transform_type = size(val_left) == size(val_right) # Actual behavior - has_restricted_transform_type = isconcretetype(eltype(vnv.transforms)) - has_restricted_varname_type = isconcretetype(eltype(vnv.varnames)) + has_restricted_transform_type = isconcretetype(eltype(vnv_base.transforms)) + has_restricted_varname_type = isconcretetype(eltype(vnv_base.varnames)) @testset "type specialization" begin @test !should_have_restricted_varname_type || has_restricted_varname_type @@ -76,55 +153,96 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) end # `eltype` - @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) # `length` - @test length(vnv) == length(val_left) + length(val_right) + @test length(vnv_base) == length(val_left) + length(val_right) # `getindex` @testset "getindex" begin # `getindex` - @test vnv[vn_left] == val_left - @test vnv[vn_right] == val_right + @test vnv_base[vn_left] == val_left + @test vnv_base[vn_right] == val_right end + # `setindex!` @testset "setindex!" begin + vnv = deepcopy(vnv_base) vnv[vn_left] = val_left .+ 100 @test vnv[vn_left] == val_left .+ 100 vnv[vn_right] = val_right .+ 100 @test vnv[vn_right] == val_right .+ 100 end - # `push!` & `update!` - # These are only allowed for all the varnames if both the varname and - # the transform types used in the underlying containers are not concrete. - push_test_varnames = filter(keys(test_pairs)) do vn - val = test_pairs[vn] - transform_is_okay = - !has_restricted_transform_type || - size(val) == size(val_left) || - size(val) == size(val_right) - varname_is_okay = - !has_restricted_varname_type || - typeof(vn) == typeof(vn_left) || - typeof(vn) == typeof(vn_right) - return transform_is_okay && varname_is_okay + # `getindex_raw` + @testset "getindex_raw" begin + @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) + @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) end - @testset "push! ($(vn))" for vn in push_test_varnames - val = test_pairs[vn] - if vn == vn_left || vn == vn_right - # Should not be possible to `push!` existing varname. - @test_throws ArgumentError push!(vnv, vn, val) - else - push!(vnv, vn, val) - @test vnv[vn] == val + + # `setindex_raw!` + @testset "setindex_raw!" begin + vnv = deepcopy(vnv_base) + DynamicPPL.setindex_raw!(vnv, to_vec_left(val_left .+ 100), vn_left) + @test vnv[vn_left] == val_left .+ 100 + DynamicPPL.setindex_raw!(vnv, to_vec_right(val_right .+ 100), vn_right) + @test vnv[vn_right] == val_right .+ 100 + end + + # `push!` & `update!` + @testset "push!" begin + vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + @testset "$vn" for vn in keys(test_pairs) + val = test_pairs[vn] + if vn == vn_left || vn == vn_right + # Should not be possible to `push!` existing varname. + @test_throws ArgumentError push!(vnv, vn, val) + else + push!(vnv, vn, val) + @test vnv[vn] == val + end end end - @testset "update! ($(vn))" for vn in push_test_varnames - val = test_pairs[vn] - # Perturb `val` a bit so we can also check that the existing `vn_left` and `vn_right` - # are also updated correctly. - DynamicPPL.update!(vnv, vn, val .+ 1) - @test vnv[vn] == val .+ 1 + @testset "update!" begin + vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + @testset "$vn" for vn in keys(test_pairs) + val = test_pairs[vn] + expected_length = if haskey(vnv, vn) + # If it's already present, the resulting length will be unchanged. + length(vnv) + else + length(vnv) + length(val) + end + + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(vnv[:]) == length(vnv) + + # There should be no redundant values in the underlying vector. + @test !DynamicPPL.has_inactive_ranges(vnv) + end + + # Need to recompute valid varnames for the changing of the sizes; before + # we required either a) the underlying `transforms` to be non-concrete, + # or b) the sizes of the values to match. But now the sizes of the values + # will change, so we can only test the former. + vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + @testset "$vn (different size)" for vn in keys(test_pairs) + val_original = test_pairs[vn] + val = change_size_for_test(val_original) + vn_already_present = haskey(vnv, vn) + expected_length = if vn_already_present + # If it's already present, the resulting length will be altered. + length(vnv) + length(val) - length(val_original) + else + length(vnv) + length(val) + end + + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(vnv[:]) == length(vnv) + end end end end From fb01b941df4883758b6d6f252bf98fb216ad7a7b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:29:20 +0000 Subject: [PATCH 017/182] formatting --- src/varnamevector.jl | 11 ++++++----- test/varnamevector.jl | 34 ++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index c696aa20e..a6da79b04 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -113,11 +113,12 @@ has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) -Base.length(vnv::VarNameVector) = if isempty(vnv.inactive_ranges) - length(vnv.vals) -else - sum(length, vnv.ranges) -end +Base.length(vnv::VarNameVector) = + if isempty(vnv.inactive_ranges) + length(vnv.vals) + else + sum(length, vnv.ranges) + end Base.size(vnv::VarNameVector) = (length(vnv),) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() diff --git a/test/varnamevector.jl b/test/varnamevector.jl index c0c542c70..0c46af83d 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -37,7 +37,9 @@ function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) return false end -relax_container_types(vnv::VarNameVector, vn::VarName, val) = relax_container_types(vnv, [vn], [val]) +function relax_container_types(vnv::VarNameVector, vn::VarName, val) + return relax_container_types(vnv, [vn], [val]) +end function relax_container_types(vnv::VarNameVector, vns, vals) if any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) @@ -47,11 +49,12 @@ function relax_container_types(vnv::VarNameVector, vns, vals) varnames_new = vnv.varnames end - transforms_new = if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) - convert(Vector{Any}, vnv.transforms) - else - vnv.transforms - end + transforms_new = + if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + convert(Vector{Any}, vnv.transforms) + else + vnv.transforms + end vals_new = if any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) convert(Vector{Any}, vnv.vals) @@ -66,7 +69,7 @@ function relax_container_types(vnv::VarNameVector, vns, vals) vals_new, transforms_new, vnv.inactive_ranges, - vnv.metadata + vnv.metadata, ) end @@ -92,11 +95,9 @@ end @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), - @varname(y[1]) => rand(), @varname(y[2]) => rand(2), @varname(y[3]) => rand(2, 3), - @varname(z[1]) => rand(1:10), @varname(z[2]) => rand(1:10, 2), @varname(z[3]) => rand(1:10, 2, 3), @@ -148,8 +149,7 @@ end @testset "type specialization" begin @test !should_have_restricted_varname_type || has_restricted_varname_type - @test !should_have_restricted_transform_type || - has_restricted_transform_type + @test !should_have_restricted_transform_type || has_restricted_transform_type end # `eltype` @@ -190,7 +190,9 @@ end # `push!` & `update!` @testset "push!" begin - vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + vnv = relax_container_types( + deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + ) @testset "$vn" for vn in keys(test_pairs) val = test_pairs[vn] if vn == vn_left || vn == vn_right @@ -203,7 +205,9 @@ end end end @testset "update!" begin - vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + vnv = relax_container_types( + deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + ) @testset "$vn" for vn in keys(test_pairs) val = test_pairs[vn] expected_length = if haskey(vnv, vn) @@ -226,7 +230,9 @@ end # we required either a) the underlying `transforms` to be non-concrete, # or b) the sizes of the values to match. But now the sizes of the values # will change, so we can only test the former. - vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + vnv = relax_container_types( + deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + ) @testset "$vn (different size)" for vn in keys(test_pairs) val_original = test_pairs[vn] val = change_size_for_test(val_original) From 9634839f22b684870afe5fb755530d640308fced Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:40:43 +0000 Subject: [PATCH 018/182] added `similar` implementation for `VarNameVector` --- src/varnamevector.jl | 22 ++++++++++++++++++++++ test/varnamevector.jl | 17 +++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a6da79b04..13d307d19 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -183,6 +183,28 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) + +similar_metadata(::Nothing) = nothing +similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) +function Base.similar(vnv::VarNameVector) + # NOTE: Whether or not we should empty the underlying containers or note + # is somewhat ambiguous. For example, `similar(vnv.varname_to_index)` will + # result in an empty `AbstractDict`, while the vectors, e.g. `vnv.ranges`, + # will result in non-empty vectors but with entries as `undef`. But it's + # much easier to write the rest of the code assuming that `undef` is not + # present, and so for now we empty the underlying containers, thus differing + # from the behavior of `similar` for `AbstractArray`s. + return VarNameVector( + similar(vnv.varname_to_index), + similar(vnv.varnames, 0), + similar(vnv.ranges, 0), + similar(vnv.vals, 0), + similar(vnv.transforms, 0), + similar(vnv.inactive_ranges, 0), + similar_metadata(vnv.metadata), + ) +end + function nextrange(vnd::VarNameVector, x) n = maximum(map(last, vnd.ranges)) return (n + 1):(n + length(x)) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 0c46af83d..b63d695c2 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -157,6 +157,23 @@ end # `length` @test length(vnv_base) == length(val_left) + length(val_right) + # `isempty` + @test !isempty(vnv_base) + + # `empty!` + @testset "empty!" begin + vnv = deepcopy(vnv_base) + empty!(vnv) + @test isempty(vnv) + end + + # `similar` + @testset "similar" begin + vnv = similar(vnv_base) + @test isempty(vnv) + @test typeof(vnv) == typeof(vnv_base) + end + # `getindex` @testset "getindex" begin # `getindex` From 5179f6fd603899cc4df4fa2d337062920eb12cad Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:41:09 +0000 Subject: [PATCH 019/182] formatting --- src/varnamevector.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 13d307d19..00d5149f2 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -183,7 +183,6 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) - similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) function Base.similar(vnv::VarNameVector) From 9f632bb319df4eab96c76ef0c71aa13816f8f148 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:41:29 +0000 Subject: [PATCH 020/182] removed debug statement --- test/varnamevector.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b63d695c2..92f402496 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -116,9 +116,6 @@ end end test_varnames_iter = combinations(collect(keys(test_pairs)), 2) - @info "Testing varnames" collect( - map(Base.Fix1(convert, Vector{VarName}), test_varnames_iter) - ) @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] From 3c210f7f85d734b34255eccd156bce6e8f4262d2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 21:36:08 +0000 Subject: [PATCH 021/182] made VarInfo slighly more generic wrt. underlying metadata --- src/varinfo.jl | 25 +++++++++++++++++++------ src/varnamevector.jl | 10 +++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 7551dc3fc..c50372aa7 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -867,7 +867,8 @@ function BangBang.empty!!(vi::VarInfo) reset_num_produce!(vi) return vi end -@inline _empty!(metadata::Metadata) = empty!(metadata) + +_empty!(metadata) = empty!(metadata) @generated function _empty!(metadata::NamedTuple{names}) where {names} expr = Expr(:block) for f in names @@ -1534,12 +1535,14 @@ end return map(vn -> vi[vn], f_vns) end +haskey(metadata::Metadata, vn::VarName) = haskey(metadata.idcs, vn) + """ haskey(vi::VarInfo, vn::VarName) Check whether `vn` has been sampled in `vi`. """ -haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn).idcs, vn) +haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn), vn) function haskey(vi::TypedVarInfo, vn::VarName) metadata = vi.metadata Tmeta = typeof(metadata) @@ -1603,9 +1606,14 @@ function BangBang.push!!( @assert ~(haskey(vi, vn)) "[push!!] attempt to add an exisitng variable $(getsym(vn)) ($(vn)) to TypedVarInfo of syms $(syms(vi)) with dist=$dist, gid=$gidset" end - val = vectorize(dist, r) - meta = getmetadata(vi, vn) + push!(meta, vn, r, dist, gidset, get_num_produce(vi)) + + return vi +end + +function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) + val = vectorize(dist, r) meta.idcs[vn] = length(meta.idcs) + 1 push!(meta.vns, vn) l = length(meta.vals) @@ -1614,11 +1622,16 @@ function BangBang.push!!( append!(meta.vals, val) push!(meta.dists, dist) push!(meta.gids, gidset) - push!(meta.orders, get_num_produce(vi)) + push!(meta.orders, num_produce) push!(meta.flags["del"], false) push!(meta.flags["trans"], false) - return vi + return meta +end + +function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) + # FIXME: Include `transform` in the `push!` call below. + return push!(vnv, vn, r) end """ diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 00d5149f2..a9094b28a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -172,6 +172,7 @@ function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) return vnv.vals[getrange(vnv, vn)] = val end +# `empty!(!)` function Base.empty!(vnv::VarNameVector) # TODO: Or should the semantics be different, e.g. keeping `varnames`? empty!(vnv.varname_to_index) @@ -183,6 +184,7 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) +# `similar` similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) function Base.similar(vnv::VarNameVector) @@ -204,9 +206,11 @@ function Base.similar(vnv::VarNameVector) ) end -function nextrange(vnd::VarNameVector, x) - n = maximum(map(last, vnd.ranges)) - return (n + 1):(n + length(x)) +function nextrange(vnv::VarNameVector, x) + # NOTE: Need to treat `isempty(vnv.ranges)` separately because `maximum` + # will error if `vnv.ranges` is empty. + offset = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) + return (offset + 1):(offset + length(x)) end # `push!` and `push!!`: add a variable to the varname vector. From 8b2720f76026d71ce9c08af5eda000008eb42253 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:16:02 +0000 Subject: [PATCH 022/182] fixed incorrect behavior in `keys` for `Metadata` --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index c50372aa7..90b61ea3e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -878,7 +878,7 @@ _empty!(metadata) = empty!(metadata) end # `keys` -Base.keys(md::Metadata) = keys(md.idcs) +Base.keys(md::Metadata) = md.vns Base.keys(vi::VarInfo) = keys(vi.metadata) # HACK: Necessary to avoid returning `Any[]` which won't dispatch correctly From 9fa6446b0c417d341dae3937879ba233d990a886 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:24:58 +0000 Subject: [PATCH 023/182] minor style changes to VarNameVector tests --- test/varnamevector.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 92f402496..1bbbf450f 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -90,7 +90,6 @@ end # - `iterate` # - `convert` to # - `AbstractDict` - test_pairs = OrderedDict( @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @@ -102,6 +101,8 @@ end @varname(z[2]) => rand(1:10, 2), @varname(z[3]) => rand(1:10, 2, 3), ) + test_vns = collect(keys(test_pairs)) + test_vals = collect(test_vals) @testset "constructor: no args" begin # Empty. @@ -115,7 +116,7 @@ end @test eltype(vnv) == Float64 end - test_varnames_iter = combinations(collect(keys(test_pairs)), 2) + test_varnames_iter = combinations(test_vns, 2) @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] @@ -205,9 +206,9 @@ end # `push!` & `update!` @testset "push!" begin vnv = relax_container_types( - deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + deepcopy(vnv_base), test_vns, test_vals ) - @testset "$vn" for vn in keys(test_pairs) + @testset "$vn" for vn in test_vns val = test_pairs[vn] if vn == vn_left || vn == vn_right # Should not be possible to `push!` existing varname. @@ -220,9 +221,9 @@ end end @testset "update!" begin vnv = relax_container_types( - deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + deepcopy(vnv_base), test_vns, test_vals ) - @testset "$vn" for vn in keys(test_pairs) + @testset "$vn" for vn in test_vns val = test_pairs[vn] expected_length = if haskey(vnv, vn) # If it's already present, the resulting length will be unchanged. @@ -245,9 +246,9 @@ end # or b) the sizes of the values to match. But now the sizes of the values # will change, so we can only test the former. vnv = relax_container_types( - deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + deepcopy(vnv_base), test_vns, test_vals ) - @testset "$vn (different size)" for vn in keys(test_pairs) + @testset "$vn (different size)" for vn in test_vns val_original = test_pairs[vn] val = change_size_for_test(val_original) vn_already_present = haskey(vnv, vn) From 0900c57bd1c69da53503355e254a5640fc347a3b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:25:18 +0000 Subject: [PATCH 024/182] style --- test/varnamevector.jl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 1bbbf450f..b6a7f1e33 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -205,9 +205,7 @@ end # `push!` & `update!` @testset "push!" begin - vnv = relax_container_types( - deepcopy(vnv_base), test_vns, test_vals - ) + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns val = test_pairs[vn] if vn == vn_left || vn == vn_right @@ -220,9 +218,7 @@ end end end @testset "update!" begin - vnv = relax_container_types( - deepcopy(vnv_base), test_vns, test_vals - ) + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns val = test_pairs[vn] expected_length = if haskey(vnv, vn) @@ -245,9 +241,7 @@ end # we required either a) the underlying `transforms` to be non-concrete, # or b) the sizes of the values to match. But now the sizes of the values # will change, so we can only test the former. - vnv = relax_container_types( - deepcopy(vnv_base), test_vns, test_vals - ) + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn (different size)" for vn in test_vns val_original = test_pairs[vn] val = change_size_for_test(val_original) From 1f7e633980220df465612841e5b9c490cb14c8af Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:58:18 +0000 Subject: [PATCH 025/182] added testing of `update!` with smaller sizes and fixed bug related to this --- src/varnamevector.jl | 13 ++++++- test/varnamevector.jl | 81 +++++++++++++++++++++++++++++++++---------- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a9094b28a..5f5233ecb 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -209,7 +209,18 @@ end function nextrange(vnv::VarNameVector, x) # NOTE: Need to treat `isempty(vnv.ranges)` separately because `maximum` # will error if `vnv.ranges` is empty. - offset = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) + max_active_range = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) + # Also need to consider inactive ranges, since we can have scenarios such as + # + # vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2, 3]) + # update!(vnv, @varname(y), [4]) # => `ranges = [1:1, 2:2], inactive_ranges = [3:3]` + # + # Here `nextrange(vnv, [5])` should return `4:4`, _not_ `3:3`. + # NOTE: We could of course attempt to make use of unused space, e.g. if we have an inactive + # range which can hold `x`, then we could just use that. Buuut the complexity of this is + # probably not worth it (at least at the moment). + max_inactive_range = isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) + offset = max(max_active_range, max_inactive_range) return (offset + 1):(offset + length(x)) end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b6a7f1e33..7eaa8285a 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,7 +1,11 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) -change_size_for_test(x::Real) = [x] -change_size_for_test(x::AbstractArray) = repeat(x, 2) +increase_size_for_test(x::Real) = [x] +increase_size_for_test(x::AbstractArray) = repeat(x, 2) + +decrease_size_for_test(x::Real) = x +decrease_size_for_test(x::AbstractVector) = first(x) +decrease_size_for_test(x::AbstractArray) = first(eachslice(x; dims=1)) function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) if isconcretetype(eltype(vnv.varnames)) @@ -16,6 +20,9 @@ function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) return false end +function need_varnames_relaxation(vnv::VarNameVector, vns, vals) + return any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) if isconcretetype(eltype(vnv.vals)) @@ -24,6 +31,9 @@ function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) return false end +function need_values_relaxation(vnv::VarNameVector, vns, vals) + return any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) if isconcretetype(eltype(vnv.transforms)) @@ -36,12 +46,35 @@ function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) return false end +function need_transforms_relaxation(vnv::VarNameVector, vns, vals) + return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end + +""" + relax_container_types(vnv::VarNameVector, vn::VarName, val) + relax_container_types(vnv::VarNameVector, vns, val) + +Relax the container types of `vnv` if necessary to accommodate `vn` and `val`. + +This attempts to avoid unnecessary container type relaxations by checking whether +the container types of `vnv` are already compatible with `vn` and `val`. +# Notes +For example, if `vn` is not compatible with the current keys in `vnv`, then +the underlying types will be changed to `VarName` to accommodate `vn`. + +Similarly: +- If `val` is not compatible with the current values in `vnv`, then + the underlying value type will be changed to `Real`. +- If `val` requires a transformation that is not compatible with the current + transformations type in `vnv`, then the underlying transformation type will + be changed to `Any`. +""" function relax_container_types(vnv::VarNameVector, vn::VarName, val) return relax_container_types(vnv, [vn], [val]) end function relax_container_types(vnv::VarNameVector, vns, vals) - if any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + if need_varnames_relaxation(vnv, vns, vals) varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) varnames_new = convert(Vector{VarName}, vnv.varnames) else @@ -49,15 +82,14 @@ function relax_container_types(vnv::VarNameVector, vns, vals) varnames_new = vnv.varnames end - transforms_new = - if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) - convert(Vector{Any}, vnv.transforms) - else - vnv.transforms - end + transforms_new = if need_transforms_relaxation(vnv, vns, vals) + convert(Vector{Any}, vnv.transforms) + else + vnv.transforms + end - vals_new = if any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) - convert(Vector{Any}, vnv.vals) + vals_new = if need_values_relaxation(vnv, vns, vals) + convert(Vector{Real}, vnv.vals) else vnv.vals end @@ -102,7 +134,7 @@ end @varname(z[3]) => rand(1:10, 2, 3), ) test_vns = collect(keys(test_pairs)) - test_vals = collect(test_vals) + test_vals = collect(values(test_pairs)) @testset "constructor: no args" begin # Empty. @@ -237,14 +269,10 @@ end @test !DynamicPPL.has_inactive_ranges(vnv) end - # Need to recompute valid varnames for the changing of the sizes; before - # we required either a) the underlying `transforms` to be non-concrete, - # or b) the sizes of the values to match. But now the sizes of the values - # will change, so we can only test the former. vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn (different size)" for vn in test_vns + @testset "$vn (increased size)" for vn in test_vns val_original = test_pairs[vn] - val = change_size_for_test(val_original) + val = increase_size_for_test(val_original) vn_already_present = haskey(vnv, vn) expected_length = if vn_already_present # If it's already present, the resulting length will be altered. @@ -258,6 +286,23 @@ end @test length(vnv) == expected_length @test length(vnv[:]) == length(vnv) end + + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn (decreased size)" for vn in test_vns + val_original = test_pairs[vn] + val = decrease_size_for_test(val_original) + vn_already_present = haskey(vnv, vn) + expected_length = if vn_already_present + # If it's already present, the resulting length will be altered. + length(vnv) + length(val) - length(val_original) + else + length(vnv) + length(val) + end + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(vnv[:]) == length(vnv) + end end end end From 8d05586cb16dfa91d0c8d350191c8cf9118a798f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:59:09 +0000 Subject: [PATCH 026/182] formatting --- src/varnamevector.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 5f5233ecb..93272a063 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -219,7 +219,8 @@ function nextrange(vnv::VarNameVector, x) # NOTE: We could of course attempt to make use of unused space, e.g. if we have an inactive # range which can hold `x`, then we could just use that. Buuut the complexity of this is # probably not worth it (at least at the moment). - max_inactive_range = isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) + max_inactive_range = + isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) offset = max(max_active_range, max_inactive_range) return (offset + 1):(offset + length(x)) end From 7801fe1c0afe694cda5c645e5b9846459abd10b8 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 11:11:55 +0000 Subject: [PATCH 027/182] move functionality related to `push!` for `VarNameVector` into `push!` --- src/varnamevector.jl | 85 +++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 93272a063..4f6a3b7f9 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -121,6 +121,7 @@ Base.length(vnv::VarNameVector) = end Base.size(vnv::VarNameVector) = (length(vnv),) +# TODO: We should probably remove this Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. @@ -229,59 +230,61 @@ end function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) - return update!(vnv, vn, val, transform) + # NOTE: We need to compute the `nextrange` BEFORE we start mutating + # the underlying; otherwise we might get some strange behaviors. + val_vec = tovec(val) + r_new = nextrange(vnv, val_vec) + vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 + push!(vnv.varnames, vn) + push!(vnv.ranges, r_new) + append!(vnv.vals, val_vec) + push!(vnv.transforms, transform) + return nothing end # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) - val_vec = tovec(val) if !haskey(vnv, vn) # Here we just add a new entry. - # NOTE: We need to compute the `nextrange` BEFORE we start mutating - # the underlying; otherwise we might get some strange behaviors. + return push!(vnv, vn, val, transform) + end + + # Here we update an existing entry. + val_vec = tovec(val) + idx = getidx(vnv, vn) + r_old = getrange(vnv, idx) + n_old = length(r_old) + n_new = length(val_vec) + # Existing keys needs to be handled differently depending on + # whether the size of the value is increasing or decreasing. + if n_new > n_old + # Add the new range. r_new = nextrange(vnv, val_vec) - vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 - push!(vnv.varnames, vn) - push!(vnv.ranges, r_new) - append!(vnv.vals, val_vec) - push!(vnv.transforms, transform) + vnv.ranges[idx] = r_new + # Grow the underlying vector to accomodate the new value. + resize!(vnv.vals, r_new[end]) + # Keep track of the deleted ranges. + push!(vnv.inactive_ranges, r_old) else - # Here we update the existing entry. - idx = getidx(vnv, vn) - r_old = getrange(vnv, idx) - n_old = length(r_old) - n_new = length(val_vec) - # Existing keys needs to be handled differently depending on - # whether the size of the value is increasing or decreasing. - if n_new > n_old - # Add the new range. - r_new = nextrange(vnv, val_vec) - vnv.ranges[idx] = r_new - # Grow the underlying vector to accomodate the new value. - resize!(vnv.vals, r_new[end]) - # Keep track of the deleted ranges. - push!(vnv.inactive_ranges, r_old) - else - # `n_new <= n_old` - # Just decrease the current range. - r_new = r_old[1]:(r_old[1] + n_new - 1) - vnv.ranges[idx] = r_new - # And mark the rest as inactive if needed. - if n_new < n_old - push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) - end + # `n_new <= n_old` + # Just decrease the current range. + r_new = r_old[1]:(r_old[1]+n_new-1) + vnv.ranges[idx] = r_new + # And mark the rest as inactive if needed. + if n_new < n_old + push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) end + end - # Update the value. - vnv.vals[r_new] = val_vec - # Update the transform. - vnv.transforms[idx] = transform + # Update the value. + vnv.vals[r_new] = val_vec + # Update the transform. + vnv.transforms[idx] = transform - # TODO: Should we maybe sweep over inactive ranges and re-contiguify - # if we the total number of inactive elements is "large" in some sense? - end + # TODO: Should we maybe sweep over inactive ranges and re-contiguify + # if we the total number of inactive elements is "large" in some sense? - return vnv + return nothing end function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) From cdc2373457b7fcfab18b1e09aa7ae182659b20f1 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 16 Nov 2023 14:20:08 +0000 Subject: [PATCH 028/182] Update src/varnamevector.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/varnamevector.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 4f6a3b7f9..cf3671594 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -268,7 +268,7 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) else # `n_new <= n_old` # Just decrease the current range. - r_new = r_old[1]:(r_old[1]+n_new-1) + r_new = r_old[1]:(r_old[1] + n_new - 1) vnv.ranges[idx] = r_new # And mark the rest as inactive if needed. if n_new < n_old From ae4bcb7e0ec1b1f4838daa2b21e2bf68a1f95fad Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 30 Dec 2023 10:03:51 +0000 Subject: [PATCH 029/182] several fixes to make sampling with VarNameVector + initiall tests for sampling with VarNameVector --- src/test_utils.jl | 15 +++++++-- src/utils.jl | 2 +- src/varinfo.jl | 76 ++++++++++++++++++++++++++++++++++++++----- src/varnamevector.jl | 11 ++++++- test/test_util.jl | 1 + test/varnamevector.jl | 61 ++++++++++++++++++++++++++++++++++ 6 files changed, 152 insertions(+), 14 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 6323f4dab..e79aad9d1 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -30,9 +30,9 @@ end Test that `vi[vn]` corresponds to the correct value in `vals` for every `vn` in `vns`. """ -function test_values(vi::AbstractVarInfo, vals::NamedTuple, vns; isequal=isequal, kwargs...) +function test_values(vi::AbstractVarInfo, vals::NamedTuple, vns; compare=isequal, kwargs...) for vn in vns - @test isequal(vi[vn], get(vals, vn); kwargs...) + @test compare(vi[vn], get(vals, vn); kwargs...) end end @@ -52,6 +52,8 @@ function setup_varinfos( vi_untyped = VarInfo() model(vi_untyped) vi_typed = DynamicPPL.TypedVarInfo(vi_untyped) + vi_vnv = DynamicPPL.VectorVarInfo(vi_untyped) + vi_vnv_typed = DynamicPPL.VectorVarInfo(vi_typed) # SimpleVarInfo svi_typed = SimpleVarInfo(example_values) svi_untyped = SimpleVarInfo(OrderedDict()) @@ -62,7 +64,14 @@ function setup_varinfos( lp = getlogp(vi_typed) varinfos = map(( - vi_untyped, vi_typed, svi_typed, svi_untyped, svi_typed_ref, svi_untyped_ref + vi_untyped, + vi_typed, + vi_vnv, + vi_vnv_typed, + svi_typed, + svi_untyped, + svi_typed_ref, + svi_untyped_ref, )) do vi # Set them all to the same values. DynamicPPL.setlogp!!(update_values!!(vi, example_values, varnames), lp) diff --git a/src/utils.jl b/src/utils.jl index ca068b1dc..07fb63b60 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -262,7 +262,7 @@ end reconstruct(d::Distribution, val::AbstractVector) = reconstruct(size(d), val) reconstruct(::Tuple{}, val::AbstractVector) = val[1] reconstruct(s::NTuple{1}, val::AbstractVector) = copy(val) -reconstruct(s::NTuple{2}, val::AbstractVector) = reshape(copy(val), s) +reconstruct(s::Tuple, val::AbstractVector) = reshape(copy(val), s) function reconstruct!(r, d::Distribution, val::AbstractVector) return reconstruct!(r, d, val) end diff --git a/src/varinfo.jl b/src/varinfo.jl index e417af00c..e450dd846 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -126,6 +126,47 @@ function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) ) end +# No-op if we're already working with a `VarNameVector`. +metadata_to_varnamevector(vnv::VarNameVector) = vnv +function metadata_to_varnamevector(md::Metadata) + idcs = md.idcs + vns = md.vns + ranges = md.ranges + vals = md.vals + transforms = map(md.dists) do dist + # TODO: Handle linked distributions. + FromVec(size(dist)) + end + + return VarNameVector( + OrderedDict{eltype(keys(idcs)),Int}(idcs), + vns, + ranges, + vals, + transforms, + ) +end + +function VectorVarInfo(vi::UntypedVarInfo) + md = metadata_to_varnamevector(vi.metadata) + lp = getlogp(vi) + return VarInfo( + md, + Base.RefValue{eltype(lp)}(lp), + Ref(get_num_produce(vi)), + ) +end + +function VectorVarInfo(vi::TypedVarInfo) + md = map(metadata_to_varnamevector, vi.metadata) + lp = getlogp(vi) + return VarInfo( + md, + Base.RefValue{eltype(lp)}(lp), + Ref(get_num_produce(vi)), + ) +end + function VarInfo( rng::Random.AbstractRNG, model::Model, @@ -549,6 +590,10 @@ end function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) end +function setval!(vnv::VarNameVector, val, vn::VarName) + return setindex_raw!(vnv, tovec(val), vn) +end + """ getval(vi::VarInfo, vns::Vector{<:VarName}) @@ -1421,12 +1466,15 @@ function getindex(vi::VarInfo, vn::VarName, dist::Distribution) val = getval(vi, vn) return maybe_invlink_and_reconstruct(vi, vn, dist, val) end -function getindex(vi::VectorVarInfo, vn::VarName, ::Nothing) +# HACK: Allows us to also work with `VarNameVector` where `dist` is not used, +# but we instead use a transformation stored with the variable. +function getindex(vi::VarInfo, vn::VarName, ::Nothing) if !haskey(vi, vn) throw(KeyError(vn)) end return getmetadata(vi, vn)[vn] end + function getindex(vi::VarInfo, vns::Vector{<:VarName}) # FIXME(torfjelde): Using `getdist(vi, first(vns))` won't be correct in cases # such as `x .~ [Normal(), Exponential()]`. @@ -1543,9 +1591,10 @@ Check whether `vn` has been sampled in `vi`. """ haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn), vn) function haskey(vi::TypedVarInfo, vn::VarName) - metadata = vi.metadata - Tmeta = typeof(metadata) - return getsym(vn) in fieldnames(Tmeta) && haskey(getmetadata(vi, vn).idcs, vn) + md_haskey = map(vi.metadata) do metadata + haskey(metadata, vn) + end + return any(md_haskey) end function Base.show(io::IO, ::MIME"text/plain", vi::UntypedVarInfo) @@ -1640,12 +1689,14 @@ Set the `order` of `vn` in `vi` to `index`, where `order` is the number of `obse statements run before sampling `vn`. """ function setorder!(vi::VarInfo, vn::VarName, index::Int) - metadata = getmetadata(vi, vn) - if metadata.orders[metadata.idcs[vn]] != index - metadata.orders[metadata.idcs[vn]] = index - end + setorder!(getmetadata(vi, vn), vn, index) return vi end +function setorder!(metadata::Metadata, vn::VarName, index::Int) + metadata.orders[metadata.idcs[vn]] = index + return metadata +end +setorder!(vnv::VarNameVector, ::VarName, ::Int) = vnv """ getorder(vi::VarInfo, vn::VarName) @@ -1671,6 +1722,8 @@ end function is_flagged(metadata::Metadata, vn::VarName, flag::String) return metadata.flags[flag][getidx(metadata, vn)] end +# HACK: This is bad. Should we always return `true` here? +is_flagged(::VarNameVector, ::VarName, flag::String) = flag == "del" ? true : false """ unset_flag!(vi::VarInfo, vn::VarName, flag::String) @@ -1678,9 +1731,14 @@ end Set `vn`'s value for `flag` to `false` in `vi`. """ function unset_flag!(vi::VarInfo, vn::VarName, flag::String) - getmetadata(vi, vn).flags[flag][getidx(vi, vn)] = false + unset_flag!(getmetadata(vi, vn), vn, flag) return vi end +function unset_flag!(metadata::Metadata, vn::VarName, flag::String) + metadata.flags[flag][getidx(vi, vn)] = false + return metadata +end +unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv """ set_retained_vns_del_by_spl!(vi::VarInfo, spl::Sampler) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 4f6a3b7f9..9e8c79b94 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,7 +1,15 @@ # Similar to `Metadata` but representing a `Vector` and simpler interface. # TODO: Should we subtype `AbstractVector` or `AbstractDict`? +""" + VarNameVector + +A container that works like a `Vector` and an `OrderedDict` but is neither. + +# Fields +$(FIELDS) +""" struct VarNameVector{ - K,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData + K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData } "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" varname_to_index::OrderedDict{K,Int} @@ -64,6 +72,7 @@ tovec(x::AbstractArray) = vec(x) Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec +# More convenient constructors. collect_maybe(x) = collect(x) collect_maybe(x::AbstractArray) = x diff --git a/test/test_util.jl b/test/test_util.jl index 64832f51e..22bfa46cc 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -86,6 +86,7 @@ short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" short_varinfo_name(::TypedVarInfo) = "TypedVarInfo" short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" +short_varinfo_name(::DynamicPPL.VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 7eaa8285a..65580c8b7 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -306,3 +306,64 @@ end end end end + +has_varnamevector(vi) = false +function has_varnamevector(vi::VarInfo) + return vi.metadata isa VarNameVector || + (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) +end + +@testset "VarInfo + VarNameVector" begin + models = [ + DynamicPPL.TestUtils.demo_assume_index_observe(), + DynamicPPL.TestUtils.demo_assume_observe_literal(), + DynamicPPL.TestUtils.demo_assume_literal_dot_observe(), + ] + + @testset "$(model.f)" for model in models + # TODO: Does not currently work with `get_and_set_val!` and thus not with + # `dot_tilde_assume`. + # NOTE: Need to set random seed explicitly to avoid using the same seed + # for initialization as for sampling in the inner testset below. + Random.seed!(42) + value_true = DynamicPPL.TestUtils.rand_prior_true(model) + vns = DynamicPPL.TestUtils.varnames(model) + varnames = DynamicPPL.TestUtils.varnames(model) + varinfos = DynamicPPL.TestUtils.setup_varinfos( + model, value_true, varnames; include_threadsafe=false + ) + # Filter out those which are not based on `VarNameVector`. + varinfos = filter(has_varnamevector, varinfos) + # Get the true log joint. + logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) + + @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos + # Need to make sure we're using a different random seed from the + # one used in the above call to `rand_prior_true`. + Random.seed!(43) + + # Are values correct? + DynamicPPL.TestUtils.test_values(varinfo, value_true, vns) + + # Is evaluation correct? + varinfo_eval = last( + DynamicPPL.evaluate!!(model, deepcopy(varinfo), DefaultContext()) + ) + # Log density should be the same. + @test getlogp(varinfo_eval) ≈ logp_true + # Values should be the same. + DynamicPPL.TestUtils.test_values(varinfo_eval, value_true, vns) + + # Is sampling correct? + varinfo_sample = last( + DynamicPPL.evaluate!!(model, deepcopy(varinfo), SamplingContext()) + ) + # Log density should be different. + @test getlogp(varinfo_sample) != getlogp(varinfo) + # Values should be different. + DynamicPPL.TestUtils.test_values( + varinfo_sample, value_true, vns; compare=!isequal + ) + end + end +end From 97e1bccb4c35ecbe9d97037c112502a96ac426b7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 30 Dec 2023 10:20:11 +0000 Subject: [PATCH 030/182] VarInfo + VarNameVector tests for all demo models --- test/varnamevector.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 65580c8b7..b52ad3880 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -314,15 +314,8 @@ function has_varnamevector(vi::VarInfo) end @testset "VarInfo + VarNameVector" begin - models = [ - DynamicPPL.TestUtils.demo_assume_index_observe(), - DynamicPPL.TestUtils.demo_assume_observe_literal(), - DynamicPPL.TestUtils.demo_assume_literal_dot_observe(), - ] - + models = DynamicPPL.TestUtils.DEMO_MODELS @testset "$(model.f)" for model in models - # TODO: Does not currently work with `get_and_set_val!` and thus not with - # `dot_tilde_assume`. # NOTE: Need to set random seed explicitly to avoid using the same seed # for initialization as for sampling in the inner testset below. Random.seed!(42) From ad343f36af414876e4203dd789617609a6b20ce5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 30 Dec 2023 14:25:04 +0000 Subject: [PATCH 031/182] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/varinfo.jl | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index e450dd846..dfc49c261 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -139,32 +139,20 @@ function metadata_to_varnamevector(md::Metadata) end return VarNameVector( - OrderedDict{eltype(keys(idcs)),Int}(idcs), - vns, - ranges, - vals, - transforms, + OrderedDict{eltype(keys(idcs)),Int}(idcs), vns, ranges, vals, transforms ) end function VectorVarInfo(vi::UntypedVarInfo) md = metadata_to_varnamevector(vi.metadata) lp = getlogp(vi) - return VarInfo( - md, - Base.RefValue{eltype(lp)}(lp), - Ref(get_num_produce(vi)), - ) + return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end function VectorVarInfo(vi::TypedVarInfo) md = map(metadata_to_varnamevector, vi.metadata) lp = getlogp(vi) - return VarInfo( - md, - Base.RefValue{eltype(lp)}(lp), - Ref(get_num_produce(vi)), - ) + return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end function VarInfo( @@ -594,7 +582,6 @@ function setval!(vnv::VarNameVector, val, vn::VarName) return setindex_raw!(vnv, tovec(val), vn) end - """ getval(vi::VarInfo, vns::Vector{<:VarName}) From f707b253ec031c6446d9433402df024a2b1e8c3e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 31 Dec 2023 18:25:22 +0000 Subject: [PATCH 032/182] added docs on the design of `VarNameVector` --- docs/src/api.md | 179 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 128 insertions(+), 51 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 9b98f9dc6..c562b6990 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -197,13 +197,139 @@ Please see the documentation of [AbstractPPL.jl](https://github.com/TuringLang/A ### Data Structures of Variables -DynamicPPL provides different data structures for samples from the model and their log density. -All of them are subtypes of [`AbstractVarInfo`](@ref). +DynamicPPL provides different data structures used in for storing samples and accumulation of the log-probabilities, all of which are subtypes of [`AbstractVarInfo`](@ref). ```@docs AbstractVarInfo ``` +But exactly how a [`AbstractVarInfo`](@ref) stores this information can vary. + +#### `VarInfo` + +```@docs +VarInfo +TypedVarInfo +``` + +One main characteristic of [`VarInfo`](@ref) is that samples are stored in a linearized form. + +```@docs +link! +invlink! +``` + +```@docs +set_flag! +unset_flag! +is_flagged +``` + +For Gibbs sampling the following functions were added. + +```@docs +setgid! +updategid! +``` + +The following functions were used for sequential Monte Carlo methods. + +```@docs +get_num_produce +set_num_produce! +increment_num_produce! +reset_num_produce! +setorder! +set_retained_vns_del_by_spl! +``` + +```@docs +Base.empty! +``` + +#### `SimpleVarInfo` + +```@docs +SimpleVarInfo +``` + +#### `VarNameVector` + +```@docs +VarNameVector +``` + +#### Design + +[`VarInfo`](@ref) is a fairly simple structure; it contains +- a `logp` field for accumulation of the log-density evaluation, and +- a `metadata` field for storing information about the realizations of the different variables. + +Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. + +**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. + +!!! note + We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. + +To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: +- `keys(::Dict)`: return all the `VarName`s present in `metadata`. +- `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. +- `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. +- `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. +- `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. +- `empty!(::Dict)`: delete all realizations in `metadata`. +- `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. + +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Hence we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: +- `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + - For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. +- `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. +- `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + +Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: +1. Type-stable when possible, but still functional when not. +2. Efficient storage and iteration. + +`VarNameVector` is a data structure that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. + +##### Type-stability + +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to construct a `NamedTuple` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```julia +x ~ Bernoulli(0.5) +y ~ Normal(0, 1) +``` + +then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where `Vx` is a container with `eltype` `Bool` and `Vy` is a container with `eltype` `Float64`. Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. + +!!! warning + Of course, this `NamedTuple` approach is *not* going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. + + ```julia + x[1] ~ Bernoulli(0.5) + x[2] ~ Normal(0, 1) + ``` + + In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is not type-stable but will still be functional. + + In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + +Hence, given a `VarNameVector` as outlined in the previous section, we can convert this into a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. + +##### Efficient storage and iteration + +We can achieve this nicely by storing the values for different `VarName`s contiguously in a `Vector{<:Real}` and store the `VarName`s in the order they appear in the `Vector{<:Real}` in a `OrderedDict{VarName, UnitRange{Int}}`, mapping each `VarName` to the range of indices in the `Vector{<:Real}` that correspond to its values. This is the approach taken in [`VarNameVector`](@ref). + +##### Additional methods + +We also want some additional methods that are not part of the `Dict` or `Vector` interface: +- `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + +In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: +- `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + ### Common API #### Accumulation of log-probabilities @@ -262,55 +388,6 @@ DynamicPPL.varname_leaves DynamicPPL.varname_and_value_leaves ``` -#### `SimpleVarInfo` - -```@docs -SimpleVarInfo -``` - -#### `VarInfo` - -Another data structure is [`VarInfo`](@ref). - -```@docs -VarInfo -TypedVarInfo -``` - -One main characteristic of [`VarInfo`](@ref) is that samples are stored in a linearized form. - -```@docs -link! -invlink! -``` - -```@docs -set_flag! -unset_flag! -is_flagged -``` - -For Gibbs sampling the following functions were added. - -```@docs -setgid! -updategid! -``` - -The following functions were used for sequential Monte Carlo methods. - -```@docs -get_num_produce -set_num_produce! -increment_num_produce! -reset_num_produce! -setorder! -set_retained_vns_del_by_spl! -``` - -```@docs -Base.empty! -``` ### Evaluation Contexts From f1faf18dfafb9fa371a93335f19cab58206dc027 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 31 Dec 2023 18:31:08 +0000 Subject: [PATCH 033/182] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/src/api.md | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index c562b6990..70a5ec99a 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -261,35 +261,41 @@ VarNameVector #### Design -[`VarInfo`](@ref) is a fairly simple structure; it contains -- a `logp` field for accumulation of the log-density evaluation, and -- a `metadata` field for storing information about the realizations of the different variables. +[`VarInfo`](@ref) is a fairly simple structure; it contains + + - a `logp` field for accumulation of the log-density evaluation, and + - a `metadata` field for storing information about the realizations of the different variables. Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. **Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. !!! note + We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: -- `keys(::Dict)`: return all the `VarName`s present in `metadata`. -- `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. -- `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. -- `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. -- `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. -- `empty!(::Dict)`: delete all realizations in `metadata`. -- `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. + + - `keys(::Dict)`: return all the `VarName`s present in `metadata`. + - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. + - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. + - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. + - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. + - `empty!(::Dict)`: delete all realizations in `metadata`. + - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. *But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Hence we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: -- `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. - - For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. -- `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. -- `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + + - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + + + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. + - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. + - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: -1. Type-stable when possible, but still functional when not. -2. Efficient storage and iteration. + + 1. Type-stable when possible, but still functional when not. + 2. Efficient storage and iteration. `VarNameVector` is a data structure that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. @@ -305,6 +311,7 @@ y ~ Normal(0, 1) then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where `Vx` is a container with `eltype` `Bool` and `Vy` is a container with `eltype` `Float64`. Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. !!! warning + Of course, this `NamedTuple` approach is *not* going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. ```julia @@ -325,10 +332,12 @@ We can achieve this nicely by storing the values for different `VarName`s contig ##### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: -- `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + + - `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: -- `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + + - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. ### Common API @@ -388,7 +397,6 @@ DynamicPPL.varname_leaves DynamicPPL.varname_and_value_leaves ``` - ### Evaluation Contexts Internally, both sampling and evaluation of log densities are performed with [`AbstractPPL.evaluate!!`](@ref). From 87d3d0110cbaf66973296a9f86dc01e1c3c1a36f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 31 Dec 2023 18:31:41 +0000 Subject: [PATCH 034/182] added note on `update!` --- docs/src/api.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/api.md b/docs/src/api.md index 70a5ec99a..b051ae2b8 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -333,7 +333,9 @@ We can achieve this nicely by storing the values for different `VarName`s contig We also want some additional methods that are not part of the `Dict` or `Vector` interface: - - `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. +- `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + +- `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: From 9c3b265ee55d8755a8d5ac57bbe5f71e10685894 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:05:24 +0000 Subject: [PATCH 035/182] further elaboration of the design of `VarInfo` and `VarNameVector` --- docs/src/api.md | 85 +++++++++++++++++++++++++++++++++++++++++-------- src/varinfo.jl | 19 +++++++++-- 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index b051ae2b8..52cd32061 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -284,31 +284,66 @@ To ensure that `varinfo` is simple and intuitive to work with, we need the under - `empty!(::Dict)`: delete all realizations in `metadata`. - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. -*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Hence we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. + - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + - `similar(::Vector{<:Real})`: return a new Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: 1. Type-stable when possible, but still functional when not. 2. Efficient storage and iteration. -`VarNameVector` is a data structure that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. +[`VarNameVector`](@ref) is a data structure that implements the above methods in a way that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. ##### Type-stability -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to construct a `NamedTuple` *for each distinct `Symbol` used*. For example, if we have a model of the form +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `VarNameVector` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```@example varnamevector-design +using DynamicPPL, Distributions + +@model function demo() + x ~ Bernoulli(0.5) + return y ~ Normal(0, 1) +end +``` + +then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where + + - `Vx` is a container with `eltype` `Bool`, and + - `Vy` is a container with `eltype` `Float64`. + +Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. + +For example, with the model above we have + +```@example varnamevector-design +# Type-unstable `VarInfo` +varinfo_untyped = DynamicPPL.VectorVarInfo(DynamicPPL.untyped_varinfo(demo())) +typeof(varinfo_untyped.metadata) +``` + +```@example varnamevector-design +# Type-stable `VarInfo` +varinfo_typed = DynamicPPL.VectorVarInfo(DynamicPPL.typed_varinfo(demo())) +typeof(varinfo_typed.metadata) +``` + +But they both work as expected: -```julia -x ~ Bernoulli(0.5) -y ~ Normal(0, 1) +```@example varnamevector-design +varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] ``` -then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where `Vx` is a container with `eltype` `Bool` and `Vy` is a container with `eltype` `Float64`. Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. +```@example varnamevector-design +varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] +``` !!! warning @@ -319,25 +354,49 @@ then we construct a type-stable representation by using a `NamedTuple{(:x, :y), x[2] ~ Normal(0, 1) ``` - In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is not type-stable but will still be functional. + In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. -Hence, given a `VarNameVector` as outlined in the previous section, we can convert this into a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. +!!! warning + + Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()`, this will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + +Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. ##### Efficient storage and iteration -We can achieve this nicely by storing the values for different `VarName`s contiguously in a `Vector{<:Real}` and store the `VarName`s in the order they appear in the `Vector{<:Real}` in a `OrderedDict{VarName, UnitRange{Int}}`, mapping each `VarName` to the range of indices in the `Vector{<:Real}` that correspond to its values. This is the approach taken in [`VarNameVector`](@ref). +In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. + +This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: + + - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. + - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. + - `transforms::Vector`: the transforms associated with each `VarName`. + +Mutating functions, e.g. `setindex!`, are then treated according to the following rules: + + 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. + + 2. If `VarName` is already present in the `VarNameVector`: + + 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. + 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. + 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + +!!! note + + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. ##### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: -- `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. -- `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. -In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: +In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. diff --git a/src/varinfo.jl b/src/varinfo.jl index dfc49c261..767fac7f6 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -155,15 +155,28 @@ function VectorVarInfo(vi::TypedVarInfo) return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end -function VarInfo( +function untyped_varinfo( rng::Random.AbstractRNG, model::Model, sampler::AbstractSampler=SampleFromPrior(), context::AbstractContext=DefaultContext(), ) varinfo = VarInfo() - model(rng, varinfo, sampler, context) - return TypedVarInfo(varinfo) + return last(evaluate!!(model, varinfo, SamplingContext(rng, sampler, context))) +end +function untyped_varinfo(model::Model, args...) + return untyped_varinfo(Random.default_rng(), model, args...) +end + +typed_varinfo(args...) = TypedVarInfo(untyped_varinfo(args...)) + +function VarInfo( + rng::Random.AbstractRNG, + model::Model, + sampler::AbstractSampler=SampleFromPrior(), + context::AbstractContext=DefaultContext(), +) + return typed_varinfo(rng, model, sampler, context) end VarInfo(model::Model, args...) = VarInfo(Random.default_rng(), model, args...) From 958c66bc850ef8273823074460941b8b3ed20906 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:27:49 +0000 Subject: [PATCH 036/182] more writing improvements --- docs/src/api.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 52cd32061..50e19395c 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -299,13 +299,15 @@ Moreover, we want also want the underlying representation used in `metadata` to 1. Type-stable when possible, but still functional when not. 2. Efficient storage and iteration. -[`VarNameVector`](@ref) is a data structure that implements the above methods in a way that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. +In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). ##### Type-stability -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `VarNameVector` *for each distinct `Symbol` used*. For example, if we have a model of the form +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. -```@example varnamevector-design +Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```@example varinfo-design using DynamicPPL, Distributions @model function demo() @@ -323,31 +325,31 @@ Since `VarName` contains the `Symbol` used in its type, something like `getindex For example, with the model above we have -```@example varnamevector-design +```@example varinfo-design # Type-unstable `VarInfo` -varinfo_untyped = DynamicPPL.VectorVarInfo(DynamicPPL.untyped_varinfo(demo())) +varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) typeof(varinfo_untyped.metadata) ``` -```@example varnamevector-design +```@example varinfo-design # Type-stable `VarInfo` -varinfo_typed = DynamicPPL.VectorVarInfo(DynamicPPL.typed_varinfo(demo())) +varinfo_typed = DynamicPPL.typed_varinfo(demo()) typeof(varinfo_typed.metadata) ``` But they both work as expected: -```@example varnamevector-design +```@example varinfo-design varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] ``` -```@example varnamevector-design +```@example varinfo-design varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] ``` !!! warning - Of course, this `NamedTuple` approach is *not* going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. + Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. ```julia x[1] ~ Bernoulli(0.5) @@ -360,7 +362,7 @@ varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] !!! warning - Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()`, this will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. @@ -384,6 +386,13 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: + +```@docs +DynamicPPL.has_inactive_ranges +DynamicPPL.inactive_ranges_sweep! +``` + !!! note Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. From 74c6efd6316e162195533c8a59728164605ae565 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:27:58 +0000 Subject: [PATCH 037/182] added docstring to `has_inactive_ranges` and `inactive_ranges_sweep!` --- src/varnamevector.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 783d1e9c3..12b48ce59 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -118,6 +118,11 @@ getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] +""" + has_inactive_ranges(vnv::VarNameVector) + +Returns `true` if `vnv` has inactive ranges. +""" has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) # Basic array interface. @@ -307,6 +312,11 @@ function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) return ranges end +""" + inactive_ranges_sweep!(vnv::VarNameVector) + +Sweep over the inactive ranges in `vnv` and re-contiguify the underlying +""" function inactive_ranges_sweep!(vnv::VarNameVector) # Extract the re-contiguified values. # NOTE: We need to do this before we update the ranges. From d9ea878695b8ad3a439fc5bfa6c020f086754442 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:32:16 +0000 Subject: [PATCH 038/182] moved docs on `VarInfo` design to a separate internals section --- docs/make.jl | 1 + docs/src/api.md | 150 ------------------------------------------ docs/src/internals.md | 149 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 150 deletions(-) create mode 100644 docs/src/internals.md diff --git a/docs/make.jl b/docs/make.jl index 109f28a9c..056b90d7f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,6 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], + "Internals" => "internals.md", ], checkdocs=:exports, ) diff --git a/docs/src/api.md b/docs/src/api.md index 50e19395c..a7137ccd7 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -259,156 +259,6 @@ SimpleVarInfo VarNameVector ``` -#### Design - -[`VarInfo`](@ref) is a fairly simple structure; it contains - - - a `logp` field for accumulation of the log-density evaluation, and - - a `metadata` field for storing information about the realizations of the different variables. - -Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. - -**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. - -!!! note - - We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. - -To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: - - - `keys(::Dict)`: return all the `VarName`s present in `metadata`. - - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. - - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. - - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. - - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. - - `empty!(::Dict)`: delete all realizations in `metadata`. - - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. - -*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: - - - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. - - + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. - - - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - - `similar(::Vector{<:Real})`: return a new - -Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: - - 1. Type-stable when possible, but still functional when not. - 2. Efficient storage and iteration. - -In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). - -##### Type-stability - -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. - -Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form - -```@example varinfo-design -using DynamicPPL, Distributions - -@model function demo() - x ~ Bernoulli(0.5) - return y ~ Normal(0, 1) -end -``` - -then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where - - - `Vx` is a container with `eltype` `Bool`, and - - `Vy` is a container with `eltype` `Float64`. - -Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. - -For example, with the model above we have - -```@example varinfo-design -# Type-unstable `VarInfo` -varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) -typeof(varinfo_untyped.metadata) -``` - -```@example varinfo-design -# Type-stable `VarInfo` -varinfo_typed = DynamicPPL.typed_varinfo(demo()) -typeof(varinfo_typed.metadata) -``` - -But they both work as expected: - -```@example varinfo-design -varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] -``` - -```@example varinfo-design -varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] -``` - -!!! warning - - Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. - - ```julia - x[1] ~ Bernoulli(0.5) - x[2] ~ Normal(0, 1) - ``` - - In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. - - In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. - -!!! warning - - Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. - -Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. - -##### Efficient storage and iteration - -In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. - -This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: - - - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. - - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - - `transforms::Vector`: the transforms associated with each `VarName`. - -Mutating functions, e.g. `setindex!`, are then treated according to the following rules: - - 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. - - 2. If `VarName` is already present in the `VarNameVector`: - - 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. - 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. - 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. - -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: - -```@docs -DynamicPPL.has_inactive_ranges -DynamicPPL.inactive_ranges_sweep! -``` - -!!! note - - Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. - -##### Additional methods - -We also want some additional methods that are not part of the `Dict` or `Vector` interface: - - - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. - - - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. - -In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - - - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. - ### Common API #### Accumulation of log-probabilities diff --git a/docs/src/internals.md b/docs/src/internals.md new file mode 100644 index 000000000..c1ce12325 --- /dev/null +++ b/docs/src/internals.md @@ -0,0 +1,149 @@ +## Design of `VarInfo` + +[`VarInfo`](@ref) is a fairly simple structure; it contains + + - a `logp` field for accumulation of the log-density evaluation, and + - a `metadata` field for storing information about the realizations of the different variables. + +Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. + +**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. + +!!! note + + We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. + +To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: + + - `keys(::Dict)`: return all the `VarName`s present in `metadata`. + - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. + - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. + - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. + - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. + - `empty!(::Dict)`: delete all realizations in `metadata`. + - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. + +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: + + - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + + + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. + + - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. + - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + - `similar(::Vector{<:Real})`: return a new + +Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: + + 1. Type-stable when possible, but still functional when not. + 2. Efficient storage and iteration. + +In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). + +### Type-stability + +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. + +Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```@example varinfo-design +using DynamicPPL, Distributions + +@model function demo() + x ~ Bernoulli(0.5) + return y ~ Normal(0, 1) +end +``` + +then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where + + - `Vx` is a container with `eltype` `Bool`, and + - `Vy` is a container with `eltype` `Float64`. + +Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. + +For example, with the model above we have + +```@example varinfo-design +# Type-unstable `VarInfo` +varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) +typeof(varinfo_untyped.metadata) +``` + +```@example varinfo-design +# Type-stable `VarInfo` +varinfo_typed = DynamicPPL.typed_varinfo(demo()) +typeof(varinfo_typed.metadata) +``` + +But they both work as expected: + +```@example varinfo-design +varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] +``` + +```@example varinfo-design +varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] +``` + +!!! warning + + Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. + + ```julia + x[1] ~ Bernoulli(0.5) + x[2] ~ Normal(0, 1) + ``` + + In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. + + In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + +!!! warning + + Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + +Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. + +### Efficient storage and iteration + +In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. + +This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: + + - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. + - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. + - `transforms::Vector`: the transforms associated with each `VarName`. + +Mutating functions, e.g. `setindex!`, are then treated according to the following rules: + + 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. + + 2. If `VarName` is already present in the `VarNameVector`: + + 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. + 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. + 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: + +```@docs +DynamicPPL.has_inactive_ranges +DynamicPPL.inactive_ranges_sweep! +``` + +!!! note + + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. + +### Additional methods + +We also want some additional methods that are not part of the `Dict` or `Vector` interface: + + - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + + - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + +In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: + + - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. From 5acce98bed0a4bdfe0e1d830807154871b174ccf Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:51:58 +0000 Subject: [PATCH 039/182] writing improvements for internal docs --- docs/src/api.md | 6 ------ docs/src/internals.md | 26 +++++++++++++++++++------- src/varnamevector.jl | 2 +- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index a7137ccd7..f9db6603c 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -253,12 +253,6 @@ Base.empty! SimpleVarInfo ``` -#### `VarNameVector` - -```@docs -VarNameVector -``` - ### Common API #### Accumulation of log-probabilities diff --git a/docs/src/internals.md b/docs/src/internals.md index c1ce12325..01fc47ddc 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -103,11 +103,17 @@ varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. -Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. +Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. ### Efficient storage and iteration -In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. +Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): + +```@docs +DynamicPPL.VarNameVector +``` + +In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: @@ -124,18 +130,24 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient. -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: +!!! note + + If we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. + +!!! note + + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. + +This does mean that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: ```@docs DynamicPPL.has_inactive_ranges DynamicPPL.inactive_ranges_sweep! ``` -!!! note - - Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. - ### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 12b48ce59..cbe72f16e 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -315,7 +315,7 @@ end """ inactive_ranges_sweep!(vnv::VarNameVector) -Sweep over the inactive ranges in `vnv` and re-contiguify the underlying +Re-contiguify the underlying vector and shrink if possible. """ function inactive_ranges_sweep!(vnv::VarNameVector) # Extract the re-contiguified values. From 6f95cddb16912f2af6125f24c947e252ef7763f5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 15:22:38 +0000 Subject: [PATCH 040/182] further motivation of the design choices made in `VarNameVector` --- docs/src/internals.md | 119 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 7 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 01fc47ddc..2040300b5 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -47,11 +47,11 @@ This is somewhat non-trivial to address since we want to achieve this for both c Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form ```@example varinfo-design -using DynamicPPL, Distributions - +using DynamicPPL, Distributions, FillArrays @model function demo() - x ~ Bernoulli(0.5) - return y ~ Normal(0, 1) + x ~ product_distribution(Fill(Bernoulli(0.5), 2)) + y ~ Normal(0, 1) + return nothing end ``` @@ -113,7 +113,7 @@ Efficient storage and iteration we achieve through implementation of the `metada DynamicPPL.VarNameVector ``` -In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. +In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: @@ -130,8 +130,67 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. - -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient. + +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient in the scenarios we encounter in practice. + +In particular, we want to make sure that the following scenario is efficient: + + 1. Construct a [`VarInfo`](@ref) from a `[Model`](@ref). + 2. Repeatedly call `rand!(rng, ::Model, ::VarInfo)`. + +There are typically a few scenarios where we encounter changing representation sizes of a random variable `x`: + + 1. We're working with a transformed version `x` which is represented in a lower-dimensional space, e.g. transforming a `x ~ LKJ(2, 1)` to unconstrained `y = f(x)` takes us from 2-by-2 `Matrix{Float64}` to a 1-length `Vector{Float64}`. + 2. `x` has a random size, e.g. in a mixture model with a prior on the number of components. Here the size of `x` can vary widly between every realization of the `Model`. + +In scenario (1), the we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". + +In scenario (2), we'll end up with quite a sub-optimal representation unless we do something to handle this. For example: + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: $(length(vnv.vals))") + +for i in 2:5 + # Insert value 1 size larger. + DynamicPPL.update!(vnv, @varname(x), fill(true, i)) + println("After insertion #$(i): $(length(vnv.vals))") +end +``` + +To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sweep!`](@ref) after every insertion: + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: $(length(vnv.vals))") + +for i in 2:5 + # Insert value 1 size larger. + DynamicPPL.update!(vnv, @varname(x), fill(true, i)) + DynamicPPL.inactive_ranges_sweep!(vnv) + println("After insertion #$(i): $(length(vnv.vals))") +end +``` + +The nice aspect of this is that `vnv` can grow as needed, but once it is sufficiently large to contain the all the different realizations of `x`, it stops growing! + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: $(length(vnv.vals))") + +for i in 1:100 + DynamicPPL.update!(vnv, @varname(x), fill(true, rand(1:5))) + if DynamicPPL.has_inactive_ranges(vnv) + DynamicPPL.inactive_ranges_sweep!(vnv) + end +end + +println("After insertions: $(length(vnv.vals))") +``` + +Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. + +`delete!` and similars are also implemented using the "delete-by-mark" technique outlined above. !!! note @@ -148,6 +207,52 @@ DynamicPPL.has_inactive_ranges DynamicPPL.inactive_ranges_sweep! ``` +Continuing from the example from the previous section: + +```@example varinfo-design +# Type-unstable +varinfo_untyped_vnv = DynamicPPL.VectorVarInfo(varinfo_untyped) +varinfo_untyped_vnv[@varname(x)], varinfo_untyped_vnv[@varname(y)] +``` + +```@example varinfo-design +# Type-stable +varinfo_typed_vnv = DynamicPPL.VectorVarInfo(varinfo_typed) +varinfo_typed_vnv[@varname(x)], varinfo_typed_vnv[@varname(y)] +``` + +If we now try to `delete!` `@varname(x)` + +```@example varinfo-design +haskey(varinfo_untyped_vnv, @varname(x)) +``` + +```@example varinfo-design +DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +``` + +```@example varinfo-design +# `delete!` +DynamicPPL.delete!(varinfo_untyped_vnv.metadata, @varname(x)) +DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +``` + +```@example varinfo-design +haskey(varinfo_untyped_vnv, @varname(x)) +``` + +If we try to insert a differently-sized value for `@varname(x)` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 1)) +varinfo_untyped_vnv[@varname(x)] +``` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 3)) +varinfo_untyped_vnv[@varname(x)] +``` + ### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: From 38a4b08e43fff9b949785394de1600e27a99cd0b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 15:57:56 +0000 Subject: [PATCH 041/182] improved writing --- docs/src/internals.md | 50 ++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 2040300b5..b8cfe2681 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -133,10 +133,20 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient in the scenarios we encounter in practice. -In particular, we want to make sure that the following scenario is efficient: +In particular, we want to optimize code-paths which effectively boil down to inner-loop in the following example:: - 1. Construct a [`VarInfo`](@ref) from a `[Model`](@ref). - 2. Repeatedly call `rand!(rng, ::Model, ::VarInfo)`. +```julia +# Construct a `VarInfo` with types inferred from `model`. +varinfo = VarInfo(model) + +# Repeatedly sample from `model`. +for _ = 1:num_samples + rand!(rng, model, varinfo) + + # Do something with `varinfo`. + # ... +end +``` There are typically a few scenarios where we encounter changing representation sizes of a random variable `x`: @@ -151,10 +161,10 @@ In scenario (2), we'll end up with quite a sub-optimal representation unless we vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: $(length(vnv.vals))") -for i in 2:5 - # Insert value 1 size larger. - DynamicPPL.update!(vnv, @varname(x), fill(true, i)) - println("After insertion #$(i): $(length(vnv.vals))") +for i in 1:5 + x = fill(true, rand(1:5)) + DynamicPPL.update!(vnv, @varname(x), x) + println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") end ``` @@ -164,31 +174,17 @@ To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sw vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: $(length(vnv.vals))") -for i in 2:5 - # Insert value 1 size larger. - DynamicPPL.update!(vnv, @varname(x), fill(true, i)) +for i in 1:5 + x = fill(true, rand(1:5)) + DynamicPPL.update!(vnv, @varname(x), x) DynamicPPL.inactive_ranges_sweep!(vnv) - println("After insertion #$(i): $(length(vnv.vals))") + println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") end ``` -The nice aspect of this is that `vnv` can grow as needed, but once it is sufficiently large to contain the all the different realizations of `x`, it stops growing! - -```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: $(length(vnv.vals))") - -for i in 1:100 - DynamicPPL.update!(vnv, @varname(x), fill(true, rand(1:5))) - if DynamicPPL.has_inactive_ranges(vnv) - DynamicPPL.inactive_ranges_sweep!(vnv) - end -end - -println("After insertions: $(length(vnv.vals))") -``` +Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. Of course, in this does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. -Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. +That is, we've decided to focus on optimizing scenario (1) but we still maintain functionality even in scenario (2), though at a greater computational cost. `delete!` and similars are also implemented using the "delete-by-mark" technique outlined above. From 60edd10f28e5c2759e2b681353ae8449b2c353c3 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 19:36:45 +0000 Subject: [PATCH 042/182] VarNameVector is now grown as much as needed --- src/varnamevector.jl | 116 ++++++++++++++++++++++++++++++++++++++---- test/varnamevector.jl | 26 ++++++++++ 2 files changed, 131 insertions(+), 11 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index cbe72f16e..7e9f1ccde 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -27,7 +27,7 @@ struct VarNameVector{ transforms::TTrans "inactive ranges" - inactive_ranges::Vector{UnitRange{Int}} + inactive_ranges::OrderedDict{Int,UnitRange{Int}} "metadata associated with the varnames" metadata::MData @@ -45,7 +45,13 @@ end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) return VarNameVector( - varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing + varname_to_index, + varnames, + ranges, + vals, + transforms, + OrderedDict{Int,UnitRange{Int}}(), + nothing, ) end # TODO: Do we need this? @@ -216,7 +222,7 @@ function Base.similar(vnv::VarNameVector) similar(vnv.ranges, 0), similar(vnv.vals, 0), similar(vnv.transforms, 0), - similar(vnv.inactive_ranges, 0), + similar(vnv.inactive_ranges), similar_metadata(vnv.metadata), ) end @@ -235,7 +241,7 @@ function nextrange(vnv::VarNameVector, x) # range which can hold `x`, then we could just use that. Buuut the complexity of this is # probably not worth it (at least at the moment). max_inactive_range = - isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) + isempty(vnv.inactive_ranges) ? 0 : maximum(last, values(vnv.inactive_ranges)) offset = max(max_active_range, max_inactive_range) return (offset + 1):(offset + length(x)) end @@ -256,6 +262,11 @@ function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val) return nothing end +function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + x[(start + n):end] = x[start:(end - n)] + return x +end + # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) @@ -272,13 +283,47 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Existing keys needs to be handled differently depending on # whether the size of the value is increasing or decreasing. if n_new > n_old - # Add the new range. - r_new = nextrange(vnv, val_vec) - vnv.ranges[idx] = r_new + n_extra = n_new - n_old # Grow the underlying vector to accomodate the new value. - resize!(vnv.vals, r_new[end]) - # Keep track of the deleted ranges. - push!(vnv.inactive_ranges, r_old) + resize!(vnv.vals, length(vnv.vals) + n_extra) + # Shift the existing values to make room for the new value. + shift_right!(vnv.vals, r_old[end] + 1, n_extra) + # Compute the new range. + r_new = r_old[1]:(r_old[1] + n_new - 1) + vnv.ranges[idx] = r_new + # If we have an inactive range for this variable, we need to update it. + if haskey(vnv.inactive_ranges, idx) + # Then we need to change this to reflect the new range. + let inactive_range = vnv.inactive_ranges[idx] + # Inactive range should always start before the new range + # in the scenario where the new range is larger than the old range. + @assert inactive_range[1] <= r_new[end] + if inactive_range[end] <= r_new[end] + # Inactive range ends before the new range ends. + delete!(vnv.inactive_ranges, idx) + else + # Then we have `inactive_range[1] < r_new[end]`, i.e. + # Inactive range starts before the new range ends. + # AND the inactive range ends after the new range ends. + # => We need to split the inactive range. + vnv.inactive_ranges[idx] = (r_new[end] + 1):inactive_range[end] + end + end + end + + # Shift existing ranges which are after the current range. + for (i, r_i) in enumerate(vnv.ranges) + if r_i[1] > r_old[end] + vnv.ranges[i] = r_i .+ n_extra + end + end + # Shift inactive ranges coming after `r_old` (unless they are + # for the current variable). + for (i, r_i) in pairs(vnv.inactive_ranges) + if i !== idx && r_i[1] > r_old[end] + vnv.inactive_ranges[i] = r_i .+ n_extra + end + end else # `n_new <= n_old` # Just decrease the current range. @@ -286,7 +331,13 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) vnv.ranges[idx] = r_new # And mark the rest as inactive if needed. if n_new < n_old - push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) + if haskey(vnv.inactive_ranges, idx) + vnv.inactive_ranges[idx] = ( + (r_old[n_new] + 1):vnv.inactive_ranges[idx][end] + ) + else + vnv.inactive_ranges[idx] = ((r_old[n_new] + 1):r_old[end]) + end end end @@ -368,3 +419,46 @@ function Base.iterate(vnv::VarNameVector, state=nothing) vn, state_new = res return vn => getindex(vnv, vn), state_new end + +function Base.pairs(vnv::VarNameVector) + return Iterators.zip( + vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) + ) +end + +function Base.delete!(vnv::VarNameVector, vn::VarName) + # Error if we don't have the variable. + !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) + + # Get the index of the variable. + idx = getidx(vnv, vn) + # Get the range of the variable. + r_old = getrange(vnv, idx) + # Delete the variable. + delete!(vnv.varname_to_index, vn) + deleteat!(vnv.varnames, idx) + deleteat!(vnv.ranges, idx) + deleteat!(vnv.transforms, idx) + # Mark the range as inactive. + push!(vnv.inactive_ranges, r_old) + + return vnv +end + +function Base.deleteat!(vnv::VarNameVector, vn::VarName) + # Error if we don't have the variable. + !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) + # Get the index of the variable. + idx = getidx(vnv, vn) + # Get the range of the variable. + r_old = getrange(vnv, idx) + # Delete the variable. + delete!(vnv.varname_to_index, vn) + deleteat!(vnv.varnames, idx) + deleteat!(vnv.ranges, idx) + deleteat!(vnv.transforms, idx) + # Delete the range. + deleteat!(vnv.vals, r_old) + + return vnv +end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b52ad3880..97cde9ba1 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -305,6 +305,32 @@ end end end end + + @testset "growing and shrinking" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive_ranges(vnv) + # Growing should not create inactive ranges. + for i in 1:n + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive_ranges(vnv) + end + + # Same size should not create inactive ranges. + x = fill(true, n) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive_ranges(vnv) + + # Shrinking should create inactive ranges. + for i in (n - 1):-1:1 + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test DynamicPPL.has_inactive_ranges(vnv) + @test vnv.inactive_ranges[1] == ((i + 1):n) + end + end end has_varnamevector(vi) = false From 3f9d34f2262c473d23dd1f3616aa3dda84a3f27e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 10:40:06 +0000 Subject: [PATCH 043/182] updated `delete!` --- src/varnamevector.jl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 7e9f1ccde..07b74e4c4 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -436,11 +436,21 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) r_old = getrange(vnv, idx) # Delete the variable. delete!(vnv.varname_to_index, vn) + # `deleteat!` deletes and shifts the rest of the vector. + # So after this, we need to re-adjust the indices in `varname_to_index`. deleteat!(vnv.varnames, idx) deleteat!(vnv.ranges, idx) deleteat!(vnv.transforms, idx) - # Mark the range as inactive. - push!(vnv.inactive_ranges, r_old) + + # Delete any inactive ranges corresponding to the variable. + if haskey(vnv.inactive_ranges, idx) + delete!(vnv.inactive_ranges, idx) + end + + # Re-adjust the indices in `varname_to_index`. + for (vn, idx) in vnv.varname_to_index + idx > idx && (vnv.varname_to_index[vn] = idx - 1) + end return vnv end From fb822b52643550f98331c97b55c0490ba683b833 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 12:30:54 +0000 Subject: [PATCH 044/182] Significant changes to implementation of `VarNameVector`: - "delete-by-mark" is now replaced by proper deletion. - `inactive_ranges` replaced by `num_inactive`, which only keeps track of the number of inactive entries for a given `VarName. - `VarNameVector` is now a "grow-as-needed" structure where the underlying also mimics the order that the user experiences.` --- src/varnamevector.jl | 207 +++++++++++++++++++++++++----------------- test/varnamevector.jl | 65 +++++++++---- 2 files changed, 167 insertions(+), 105 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 07b74e4c4..896c0c3a2 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -26,8 +26,8 @@ struct VarNameVector{ "vector of transformations whose inverse takes us back to the original space" transforms::TTrans - "inactive ranges" - inactive_ranges::OrderedDict{Int,UnitRange{Int}} + "additional entries which are considered inactive" + num_inactive::OrderedDict{Int,Int} "metadata associated with the varnames" metadata::MData @@ -39,7 +39,7 @@ function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) vnv_left.ranges == vnv_right.ranges && vnv_left.vals == vnv_right.vals && vnv_left.transforms == vnv_right.transforms && - vnv_left.inactive_ranges == vnv_right.inactive_ranges && + vnv_left.num_inactive == vnv_right.num_inactive && vnv_left.metadata == vnv_right.metadata end @@ -50,7 +50,7 @@ function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) ranges, vals, transforms, - OrderedDict{Int,UnitRange{Int}}(), + OrderedDict{Int,Int}(), nothing, ) end @@ -119,22 +119,47 @@ end # Some `VarNameVector` specific functions. getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] -getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] +getrange(vnv::VarNameVector, idx::Int) = vnv.ranges[idx] getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] """ - has_inactive_ranges(vnv::VarNameVector) + has_inactive(vnv::VarNameVector) Returns `true` if `vnv` has inactive ranges. """ -has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) +has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) + +""" + num_inactive(vnv::VarNameVector, vn::VarName) + +Returns the number of inactive entries for `vn` in `vnv`. +""" +num_inactive(vnv::VarNameVector, vn::VarName) = num_inactive(vnv, getidx(vnv, vn)) +num_inactive(vnv::VarNameVector, idx::Int) = get(vnv.num_inactive, idx, 0) + +""" + num_allocated(vnv::VarNameVector) + +Returns the number of allocated entries in `vnv`. +""" +num_allocated(vnv::VarNameVector) = length(vnv.vals) + +""" + num_allocated(vnv::VarNameVector, vn::VarName) + +Returns the number of allocated entries for `vn` in `vnv`. +""" +num_allocated(vnv::VarNameVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) +function num_allocated(vnv::VarNameVector, idx::Int) + return length(getrange(vnv, idx)) + num_inactive(vnv, idx) +end # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) Base.length(vnv::VarNameVector) = - if isempty(vnv.inactive_ranges) + if isempty(vnv.num_inactive) length(vnv.vals) else sum(length, vnv.ranges) @@ -164,7 +189,7 @@ end # `getindex` for `Colon` function Base.getindex(vnv::VarNameVector, ::Colon) - return if has_inactive_ranges(vnv) + return if has_inactive(vnv) mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) else vnv.vals @@ -172,7 +197,7 @@ function Base.getindex(vnv::VarNameVector, ::Colon) end function getindex_raw(vnv::VarNameVector, ::Colon) - return if has_inactive_ranges(vnv) + return if has_inactive(vnv) mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) else vnv.vals @@ -222,27 +247,20 @@ function Base.similar(vnv::VarNameVector) similar(vnv.ranges, 0), similar(vnv.vals, 0), similar(vnv.transforms, 0), - similar(vnv.inactive_ranges), + similar(vnv.num_inactive), similar_metadata(vnv.metadata), ) end function nextrange(vnv::VarNameVector, x) - # NOTE: Need to treat `isempty(vnv.ranges)` separately because `maximum` - # will error if `vnv.ranges` is empty. - max_active_range = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) - # Also need to consider inactive ranges, since we can have scenarios such as - # - # vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2, 3]) - # update!(vnv, @varname(y), [4]) # => `ranges = [1:1, 2:2], inactive_ranges = [3:3]` - # - # Here `nextrange(vnv, [5])` should return `4:4`, _not_ `3:3`. - # NOTE: We could of course attempt to make use of unused space, e.g. if we have an inactive - # range which can hold `x`, then we could just use that. Buuut the complexity of this is - # probably not worth it (at least at the moment). - max_inactive_range = - isempty(vnv.inactive_ranges) ? 0 : maximum(last, values(vnv.inactive_ranges)) - offset = max(max_active_range, max_inactive_range) + # If `vnv` is empty, return immediately. + isempty(vnv) && return 1:length(x) + + # The offset will be the last range's end + its number of inactive entries. + vn_last = vnv.varnames[end] + idx = getidx(vnv, vn_last) + offset = last(getrange(vnv, idx)) + num_inactive(vnv, idx) + return (offset + 1):(offset + length(x)) end @@ -267,6 +285,11 @@ function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) return x end +function shift_left!(x::AbstractVector{<:Real}, start::Int, n::Int) + x[start:(end - n)] = x[(start + n):end] + return x +end + # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) @@ -277,70 +300,84 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Here we update an existing entry. val_vec = tovec(val) idx = getidx(vnv, vn) + # Extract the old range. r_old = getrange(vnv, idx) + start_old, end_old = first(r_old), last(r_old) n_old = length(r_old) + # Compute the new range. n_new = length(val_vec) - # Existing keys needs to be handled differently depending on - # whether the size of the value is increasing or decreasing. - if n_new > n_old - n_extra = n_new - n_old - # Grow the underlying vector to accomodate the new value. - resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift the existing values to make room for the new value. - shift_right!(vnv.vals, r_old[end] + 1, n_extra) - # Compute the new range. - r_new = r_old[1]:(r_old[1] + n_new - 1) - vnv.ranges[idx] = r_new - # If we have an inactive range for this variable, we need to update it. - if haskey(vnv.inactive_ranges, idx) - # Then we need to change this to reflect the new range. - let inactive_range = vnv.inactive_ranges[idx] - # Inactive range should always start before the new range - # in the scenario where the new range is larger than the old range. - @assert inactive_range[1] <= r_new[end] - if inactive_range[end] <= r_new[end] - # Inactive range ends before the new range ends. - delete!(vnv.inactive_ranges, idx) - else - # Then we have `inactive_range[1] < r_new[end]`, i.e. - # Inactive range starts before the new range ends. - # AND the inactive range ends after the new range ends. - # => We need to split the inactive range. - vnv.inactive_ranges[idx] = (r_new[end] + 1):inactive_range[end] - end - end - end + start_new = start_old + end_new = start_old + n_new - 1 + r_new = start_new:end_new - # Shift existing ranges which are after the current range. - for (i, r_i) in enumerate(vnv.ranges) - if r_i[1] > r_old[end] - vnv.ranges[i] = r_i .+ n_extra - end - end - # Shift inactive ranges coming after `r_old` (unless they are - # for the current variable). - for (i, r_i) in pairs(vnv.inactive_ranges) - if i !== idx && r_i[1] > r_old[end] - vnv.inactive_ranges[i] = r_i .+ n_extra - end + #= + Suppose we currently have the following: + + | x | x | o | o | o | y | y | y | <- Current entries + + where 'O' denotes an inactive entry, and we're going to + update the variable `x` to be of size `k` instead of 2. + + We then have a few different scenarios: + 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. + E.g. if `k = 7`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | x | x | y | y | y | <- New entries + + 2. `k = 5`: All inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | y | y | y | <- New entries + + 3. `k < 5`: Some inactive entries become active, some remain inactive. + E.g. if `k = 3`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | o | o | y | y | y | <- New entries + + 4. `k = 2`: No inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | o | o | o | y | y | y | <- New entries + + 5. `k < 2`: More entries become inactive. + E.g. if `k = 1`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | o | o | o | o | y | y | y | <- New entries + =# + + # Compute the allocated space for `vn`. + had_inactive = haskey(vnv.num_inactive, idx) + n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old + + if n_new > n_allocated + # Then we need to grow the underlying vector. + n_extra = n_new - n_allocated + # Allocate. + resize!(vnv.vals, length(vnv.vals) + n_extra) + # Shift current values. + shift_right!(vnv.vals, end_old + 1, n_extra) + # No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + # Update the ranges for all variables after this one. + for i in (idx + 1):length(vnv.varnames) + vnv.ranges[i] = vnv.ranges[i] .+ n_extra end + elseif n_new == n_allocated + # => No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) else - # `n_new <= n_old` - # Just decrease the current range. - r_new = r_old[1]:(r_old[1] + n_new - 1) - vnv.ranges[idx] = r_new - # And mark the rest as inactive if needed. - if n_new < n_old - if haskey(vnv.inactive_ranges, idx) - vnv.inactive_ranges[idx] = ( - (r_old[n_new] + 1):vnv.inactive_ranges[idx][end] - ) - else - vnv.inactive_ranges[idx] = ((r_old[n_new] + 1):r_old[end]) - end - end + # `n_new < n_allocated` + # => Need to update the number of inactive entries. + vnv.num_inactive[idx] = n_allocated - n_new end + # Update the range for this variable. + vnv.ranges[idx] = r_new # Update the value. vnv.vals[r_new] = val_vec # Update the transform. @@ -375,7 +412,7 @@ function inactive_ranges_sweep!(vnv::VarNameVector) # And then we re-contiguify the ranges. recontiguify_ranges!(vnv.ranges) # Clear the inactive ranges. - empty!(vnv.inactive_ranges) + empty!(vnv.num_inactive) # Now we update the values. for (i, r) in enumerate(vnv.ranges) vnv.vals[r] = vals[r] @@ -443,8 +480,8 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) deleteat!(vnv.transforms, idx) # Delete any inactive ranges corresponding to the variable. - if haskey(vnv.inactive_ranges, idx) - delete!(vnv.inactive_ranges, idx) + if haskey(vnv.num_inactive, idx) + delete!(vnv.num_inactive, idx) end # Re-adjust the indices in `varname_to_index`. diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 97cde9ba1..cb5ac0632 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -100,7 +100,7 @@ function relax_container_types(vnv::VarNameVector, vns, vals) vnv.ranges, vals_new, transforms_new, - vnv.inactive_ranges, + vnv.num_inactive, vnv.metadata, ) end @@ -266,7 +266,7 @@ end @test length(vnv[:]) == length(vnv) # There should be no redundant values in the underlying vector. - @test !DynamicPPL.has_inactive_ranges(vnv) + @test !DynamicPPL.has_inactive(vnv) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -307,28 +307,53 @@ end end @testset "growing and shrinking" begin - n = 5 - vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) - @test !DynamicPPL.has_inactive_ranges(vnv) - # Growing should not create inactive ranges. - for i in 1:n - x = fill(true, i) + @testset "deterministic" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive(vnv) + # Growing should not create inactive ranges. + for i in 1:n + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive(vnv) + end + + # Same size should not create inactive ranges. + x = fill(true, n) DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive_ranges(vnv) + @test !DynamicPPL.has_inactive(vnv) + + # Shrinking should create inactive ranges. + for i in (n - 1):-1:1 + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test DynamicPPL.has_inactive(vnv) + @test DynamicPPL.num_inactive(vnv, vn) == n - i + end end - # Same size should not create inactive ranges. - x = fill(true, n) - DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive_ranges(vnv) + @testset "random" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive(vnv) - # Shrinking should create inactive ranges. - for i in (n - 1):-1:1 - x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) - @test DynamicPPL.has_inactive_ranges(vnv) - @test vnv.inactive_ranges[1] == ((i + 1):n) + # Insert a bunch of random-length vectors. + for i in 1:100 + x = fill(true, rand(1:n)) + DynamicPPL.update!(vnv, vn, x) + end + # Should never be allocating more than `n` elements. + @test DynamicPPL.num_allocated(vnv, vn) ≤ n + + # If we compaticfy, then it should always be the same size as just inserted. + for i in 1:10 + x = fill(true, rand(1:n)) + DynamicPPL.update!(vnv, vn, x) + DynamicPPL.inactive_ranges_sweep!(vnv) + @test DynamicPPL.num_allocated(vnv, vn) == length(x) + end end end end From 66bc090e8faf325be1f0d2cd5d90097aae965742 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:12:27 +0000 Subject: [PATCH 045/182] added `copy` when constructing `VectorVarInfo` from `VarInfo` --- src/varinfo.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 767fac7f6..48ad94392 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -129,10 +129,10 @@ end # No-op if we're already working with a `VarNameVector`. metadata_to_varnamevector(vnv::VarNameVector) = vnv function metadata_to_varnamevector(md::Metadata) - idcs = md.idcs - vns = md.vns - ranges = md.ranges - vals = md.vals + idcs = copy(md.idcs) + vns = copy(md.vns) + ranges = copy(md.ranges) + vals = copy(md.vals) transforms = map(md.dists) do dist # TODO: Handle linked distributions. FromVec(size(dist)) From ccd86f2683169ddf5e314a3f509a020b24a2ee87 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:13:03 +0000 Subject: [PATCH 046/182] added missing `isempty` impl --- src/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 896c0c3a2..758cc75d5 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -165,6 +165,7 @@ Base.length(vnv::VarNameVector) = sum(length, vnv.ranges) end Base.size(vnv::VarNameVector) = (length(vnv),) +Base.isempty(vnv::VarNameVector) = isempty(vnv.varnames) # TODO: We should probably remove this Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() From 1d4a000548b395fe2e53dfeb56a48276e27e12c6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:13:21 +0000 Subject: [PATCH 047/182] remove impl of `iterate` and instead implemented `pairs` and `values` iterators --- src/varnamevector.jl | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 758cc75d5..a26927394 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -172,6 +172,13 @@ Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. Base.keys(vnv::VarNameVector) = vnv.varnames +Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) + +function Base.pairs(vnv::VarNameVector) + return Iterators.zip( + vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) + ) +end Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) @@ -445,25 +452,6 @@ function group_by_symbol(vnv::VarNameVector) return NamedTuple{Tuple(keys(d))}(nt_vals) end -# `iterate` -# TODO: Maybe implement `iterate` as a vector and then instead implement `pairs`. -function Base.iterate(vnv::VarNameVector, state=nothing) - res = if state === nothing - iterate(vnv.varnames) - else - iterate(vnv.varnames, state) - end - res === nothing && return nothing - vn, state_new = res - return vn => getindex(vnv, vn), state_new -end - -function Base.pairs(vnv::VarNameVector) - return Iterators.zip( - vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) - ) -end - function Base.delete!(vnv::VarNameVector, vn::VarName) # Error if we don't have the variable. !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) From 9a16dd15b6562c3dc00a4c4b0477f6e9d2a0f499 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:18 +0000 Subject: [PATCH 048/182] added missing `empty!` for `num_inactive` --- src/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a26927394..90d2d8847 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -234,6 +234,7 @@ function Base.empty!(vnv::VarNameVector) empty!(vnv.ranges) empty!(vnv.vals) empty!(vnv.transforms) + empty!(vnv.num_inactive) return nothing end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) From e49b762303dda3c63d67d47fe1e1706d50cb639c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:37 +0000 Subject: [PATCH 049/182] removed redundant `shift_left!` methd --- src/varnamevector.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 90d2d8847..d238b8d87 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -294,11 +294,6 @@ function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) return x end -function shift_left!(x::AbstractVector{<:Real}, start::Int, n::Int) - x[start:(end - n)] = x[(start + n):end] - return x -end - # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) From 2b445c96ffe1fa21127dc593e6c13457e2e2b93b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:48 +0000 Subject: [PATCH 050/182] fixed `delete!` for `VarNameVector` --- src/varnamevector.jl | 57 ++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index d238b8d87..5ab11465b 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -454,43 +454,44 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) # Get the index of the variable. idx = getidx(vnv, vn) - # Get the range of the variable. - r_old = getrange(vnv, idx) - # Delete the variable. + + # Delete the values. + r_start = first(getrange(vnv, idx)) + n_allocated = num_allocated(vnv, idx) + deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) + + # Delete `vn` from the lookup table. delete!(vnv.varname_to_index, vn) - # `deleteat!` deletes and shifts the rest of the vector. - # So after this, we need to re-adjust the indices in `varname_to_index`. - deleteat!(vnv.varnames, idx) - deleteat!(vnv.ranges, idx) - deleteat!(vnv.transforms, idx) - # Delete any inactive ranges corresponding to the variable. - if haskey(vnv.num_inactive, idx) - delete!(vnv.num_inactive, idx) + # Delete any inactive ranges corresponding to `vn`. + haskey(vnv.num_inactive, idx) && delete!(vnv.num_inactive, idx) + + # Re-adjust the indices for varnames occuring after `vn` so + # that they point to the correct indices after the deletions below. + for idx_to_shift in (idx + 1):length(vnv.varnames) + vn = vnv.varnames[idx_to_shift] + if idx_to_shift > idx + # Shift the index in the lookup table. + vnv.varname_to_index[vn] = idx_to_shift - 1 + # Shift the index in the inactive ranges. + if haskey(vnv.num_inactive, idx_to_shift) + # Done in increasing order => don't need to worry about + # potentially shifting the same index twice. + vnv.num_inactive[idx_to_shift - 1] = pop!(vnv.num_inactive, idx_to_shift) + end + end end - # Re-adjust the indices in `varname_to_index`. - for (vn, idx) in vnv.varname_to_index - idx > idx && (vnv.varname_to_index[vn] = idx - 1) + # Re-adjust the ranges for varnames occuring after `vn`. + for idx_to_shift in (idx + 1):length(vnv.varnames) + vnv.ranges[idx_to_shift] = vnv.ranges[idx_to_shift] .- n_allocated end - return vnv -end - -function Base.deleteat!(vnv::VarNameVector, vn::VarName) - # Error if we don't have the variable. - !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) - # Get the index of the variable. - idx = getidx(vnv, vn) - # Get the range of the variable. - r_old = getrange(vnv, idx) - # Delete the variable. - delete!(vnv.varname_to_index, vn) + # Delete references from vector fields, thus shifting the indices of + # varnames occuring after `vn` by one to the left, as we adjusted for above. deleteat!(vnv.varnames, idx) deleteat!(vnv.ranges, idx) deleteat!(vnv.transforms, idx) - # Delete the range. - deleteat!(vnv.vals, r_old) return vnv end From e3c26339bf575a58d1b3c8228aa875561d843778 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:59 +0000 Subject: [PATCH 051/182] added `is_contiguous` as an alterantive to `!has_inactive` --- src/varnamevector.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 5ab11465b..d95e7cca0 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -261,6 +261,15 @@ function Base.similar(vnv::VarNameVector) ) end +""" + is_continugous(vnv::VarNameVector) + +Returns `true` if the underlying data of `vnv` is stored in a contiguous array. + +This is equivalent to [`!has_inactive(vnv)`](@ref). +""" +is_continugous(vnv::VarNameVector) = !has_inactive(vnv) + function nextrange(vnv::VarNameVector, x) # If `vnv` is empty, return immediately. isempty(vnv) && return 1:length(x) From 19a829cdccbb46f67267f1b0f6fabdf56a1569b2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:57:18 +0000 Subject: [PATCH 052/182] updates to internal docs --- docs/src/internals.md | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index b8cfe2681..7cac77cfb 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -86,6 +86,8 @@ varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] ``` +Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entries while the typed uses `Vector{Bool}`. This is because the untyped version needs the underlying container to be able to handle both the `Bool` for `x` and the `Float64` for `y`, while the typed version can use a `Vector{Bool}` for `x` and a `Vector{Float64}` for `y` due to its usage of `NamedTuple`. + !!! warning Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. @@ -155,16 +157,17 @@ There are typically a few scenarios where we encounter changing representation s In scenario (1), the we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". + In scenario (2), we'll end up with quite a sub-optimal representation unless we do something to handle this. For example: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: $(length(vnv.vals))") +println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 x = fill(true, rand(1:5)) DynamicPPL.update!(vnv, @varname(x), x) - println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") + println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` @@ -172,25 +175,17 @@ To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sw ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: $(length(vnv.vals))") +println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 x = fill(true, rand(1:5)) DynamicPPL.update!(vnv, @varname(x), x) DynamicPPL.inactive_ranges_sweep!(vnv) - println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") + println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` -Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. Of course, in this does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. - -That is, we've decided to focus on optimizing scenario (1) but we still maintain functionality even in scenario (2), though at a greater computational cost. - -`delete!` and similars are also implemented using the "delete-by-mark" technique outlined above. - -!!! note - - If we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. +This does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. However, this also ensures that the the underlying `Vector{T}` is contiguous, which is important for performance. Hence, if we're about to do a lot of work with the `VarNameVector` without insertions, etc., it can be worth it to do a sweep to ensure that the underlying `Vector{T}` is contiguous. !!! note @@ -199,7 +194,7 @@ That is, we've decided to focus on optimizing scenario (1) but we still maintain This does mean that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: ```@docs -DynamicPPL.has_inactive_ranges +DynamicPPL.has_inactive DynamicPPL.inactive_ranges_sweep! ``` @@ -224,13 +219,13 @@ haskey(varinfo_untyped_vnv, @varname(x)) ``` ```@example varinfo-design -DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) ``` ```@example varinfo-design # `delete!` DynamicPPL.delete!(varinfo_untyped_vnv.metadata, @varname(x)) -DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) ``` ```@example varinfo-design @@ -249,6 +244,20 @@ DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 3)) varinfo_untyped_vnv[@varname(x)] ``` +#### Performance summary + +In the end, we have the following "rough" performance characteristics: + +| Method | Is blazingly fast? | +| :-: | :-: | +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if same size / ${\color{red} \times}$ if different size | +| `delete!` | ${\color{red} \times}$ | + + + ### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: From a358bc40b11b52b80b285570b380dcbd9c74bae6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 15:00:54 +0000 Subject: [PATCH 053/182] renamed `sweep_inactive_ranges!` to `contiguify!` --- docs/src/internals.md | 6 +++--- src/varnamevector.jl | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 7cac77cfb..f13407fca 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -171,7 +171,7 @@ for i in 1:5 end ``` -To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sweep!`](@ref) after every insertion: +To alleviate this issue, we can insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) @@ -180,7 +180,7 @@ println("Before insertion: number of allocated entries $(DynamicPPL.num_allocat for i in 1:5 x = fill(true, rand(1:5)) DynamicPPL.update!(vnv, @varname(x), x) - DynamicPPL.inactive_ranges_sweep!(vnv) + DynamicPPL.contiguify!(vnv) println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` @@ -195,7 +195,7 @@ This does mean that the underlying `Vector{T}` can grow without bound, so we hav ```@docs DynamicPPL.has_inactive -DynamicPPL.inactive_ranges_sweep! +DynamicPPL.contiguify! ``` Continuing from the example from the previous section: diff --git a/src/varnamevector.jl b/src/varnamevector.jl index d95e7cca0..0fba49997 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -414,11 +414,11 @@ function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) end """ - inactive_ranges_sweep!(vnv::VarNameVector) + contiguify!(vnv::VarNameVector) Re-contiguify the underlying vector and shrink if possible. """ -function inactive_ranges_sweep!(vnv::VarNameVector) +function contiguify!(vnv::VarNameVector) # Extract the re-contiguified values. # NOTE: We need to do this before we update the ranges. vals = vnv[:] From 46be8d59b0dd57aaf84d4b3c205498f6d6ba52ce Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 15:04:34 +0000 Subject: [PATCH 054/182] improvements to internal docs --- docs/src/internals.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index f13407fca..98f9737c4 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -246,17 +246,22 @@ varinfo_untyped_vnv[@varname(x)] #### Performance summary -In the end, we have the following "rough" performance characteristics: +In the end, we have the following "rough" performance characteristics for `VarNameVector`: | Method | Is blazingly fast? | | :-: | :-: | | `getindex` | ${\color{green} \checkmark}$ | | `setindex!` | ${\color{green} \checkmark}$ | | `push!` | ${\color{green} \checkmark}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if same size / ${\color{red} \times}$ if different size | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `delete!` | ${\color{red} \times}$ | +#### Methods +```@docs +DynamicPPL.num_inactive +DynamicPPL.num_allocated +``` ### Additional methods From 57d688e84529bfcb3ade21445b7f81e3e44bff98 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:43:42 +0000 Subject: [PATCH 055/182] more improvements to internal docs --- docs/src/internals.md | 95 ++++++++++++++++++++++++------------------- src/varnamevector.jl | 6 +-- 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 98f9737c4..3c47a3d97 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -7,7 +7,7 @@ Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. -**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. +**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding to a variable `@varname(x)`. !!! note @@ -31,23 +31,26 @@ To ensure that `varinfo` is simple and intuitive to work with, we need the under - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - - `similar(::Vector{<:Real})`: return a new + - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: - 1. Type-stable when possible, but still functional when not. - 2. Efficient storage and iteration. + 1. Type-stable when possible, but functional when not. + 2. Efficient storage and iteration when possible, but functional when not. + +The "but functional when not" is important as we want to support arbitrary models, which means that we can't always have these performance properties. In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). ### Type-stability -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. +Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form ```@example varinfo-design using DynamicPPL, Distributions, FillArrays + @model function demo() x ~ product_distribution(Fill(Bernoulli(0.5), 2)) y ~ Normal(0, 1) @@ -76,7 +79,7 @@ varinfo_typed = DynamicPPL.typed_varinfo(demo()) typeof(varinfo_typed.metadata) ``` -But they both work as expected: +But they both work as expected but one results in concrete typing and the other does not: ```@example varinfo-design varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] @@ -99,11 +102,13 @@ Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entri In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. - In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + In practice, rarely observe such mixing of types, therefore in DynamicPPL, and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. !!! warning - Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + Another downside with such a `NamedTuple` approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + + For these scenarios it can be useful to fall back to "untyped" representations. Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. @@ -123,19 +128,19 @@ This does require a bit of book-keeping, in particular when it comes to insertio - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - `transforms::Vector`: the transforms associated with each `VarName`. -Mutating functions, e.g. `setindex!`, are then treated according to the following rules: +Mutating functions, e.g. `setindex!(vnv::VarNameVector, val, vn::VarName)`, are then treated according to the following rules: - 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. + 1. If `vn` is not already present: add it to the end of `vnv.varnames`, add the `val` to the underlying `vnv.vals`, etc. - 2. If `VarName` is already present in the `VarNameVector`: + 2. If `vn` is already present in `vnv`: - 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. - 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. - 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + 1. If `val` has the *same length* as the existing value for `vn`: replace existing value. + 2. If `val` has a *smaller length* than the existing value for `vn`: replace existing value and mark the remaining indices as "inactive" by increasing the entry in `vnv.num_inactive` field. + 3. If `val` has a *larger length* than the existing value for `vn`: expand the underlying `vnv.vals` to accommodate the new value, update all `VarName`s occuring after `vn`, and update the `vnv.ranges` to point to the new range for `vn`. -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient in the scenarios we encounter in practice. +This means that `VarNameVector` is allowed to grow as needed, while "shrinking" (i.e. insertion of smaller elements) is handled by simply marking the redundant indices as "inactive". This turns out to be efficient for use-cases that we are generally interested in. -In particular, we want to optimize code-paths which effectively boil down to inner-loop in the following example:: +For example, we want to optimize code-paths which effectively boil down to inner-loop in the following example: ```julia # Construct a `VarInfo` with types inferred from `model`. @@ -155,32 +160,45 @@ There are typically a few scenarios where we encounter changing representation s 1. We're working with a transformed version `x` which is represented in a lower-dimensional space, e.g. transforming a `x ~ LKJ(2, 1)` to unconstrained `y = f(x)` takes us from 2-by-2 `Matrix{Float64}` to a 1-length `Vector{Float64}`. 2. `x` has a random size, e.g. in a mixture model with a prior on the number of components. Here the size of `x` can vary widly between every realization of the `Model`. -In scenario (1), the we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". +In scenario (1), we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". +In scenario (2), we end up increasing the allocated memory for the randomly sized `x`, eventually leading to a vector that is large enough to hold realizations without needing to reallocate. But this can still lead to unnecessary memory usage, which might be undesirable. Hence one has to make a decision regarding the trade-off between memory usage and performance for the use-case at hand. -In scenario (2), we'll end up with quite a sub-optimal representation unless we do something to handle this. For example: +To help with this, we have the following functions: + +```@docs +DynamicPPL.has_inactive +DynamicPPL.num_inactive +DynamicPPL.num_allocated +DynamicPPL.is_contiguous +DynamicPPL.contiguify! +``` + +For example, one might encounter the following scenario: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 - x = fill(true, rand(1:5)) + x = fill(true, rand(1:100)) DynamicPPL.update!(vnv, @varname(x), x) println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` -To alleviate this issue, we can insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion: +We can then insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion whenever the allocation grows too large to reduce overall memory usage: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 - x = fill(true, rand(1:5)) + x = fill(true, rand(1:100)) DynamicPPL.update!(vnv, @varname(x), x) - DynamicPPL.contiguify!(vnv) + if DynamicPPL.num_allocated(vnv) > 10 + DynamicPPL.contiguify!(vnv) + end println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` @@ -191,14 +209,7 @@ This does incur a runtime cost as it requires re-allocation of the `ranges` in a Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. -This does mean that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: - -```@docs -DynamicPPL.has_inactive -DynamicPPL.contiguify! -``` - -Continuing from the example from the previous section: +Continuing from the example from the previous section, we can use a `VarInfo` with a `VarNameVector` as the `metadata` field: ```@example varinfo-design # Type-unstable @@ -232,18 +243,26 @@ DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) haskey(varinfo_untyped_vnv, @varname(x)) ``` -If we try to insert a differently-sized value for `@varname(x)` +Or insert a differently-sized value for `@varname(x)` ```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 1)) +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 1)) varinfo_untyped_vnv[@varname(x)] ``` ```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 3)) +DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) +``` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 4)) varinfo_untyped_vnv[@varname(x)] ``` +```@example varinfo-design +DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) +``` + #### Performance summary In the end, we have the following "rough" performance characteristics for `VarNameVector`: @@ -253,15 +272,9 @@ In the end, we have the following "rough" performance characteristics for `VarNa | `getindex` | ${\color{green} \checkmark}$ | | `setindex!` | ${\color{green} \checkmark}$ | | `push!` | ${\color{green} \checkmark}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `delete!` | ${\color{red} \times}$ | - -#### Methods - -```@docs -DynamicPPL.num_inactive -DynamicPPL.num_allocated -``` +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| `convert(Vector, ::VarNameVector)` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | ### Additional methods diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 0fba49997..3fa295112 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -262,13 +262,13 @@ function Base.similar(vnv::VarNameVector) end """ - is_continugous(vnv::VarNameVector) + is_contiguous(vnv::VarNameVector) Returns `true` if the underlying data of `vnv` is stored in a contiguous array. -This is equivalent to [`!has_inactive(vnv)`](@ref). +This is equivalent to negating [`has_inactive(vnv)`](@ref). """ -is_continugous(vnv::VarNameVector) = !has_inactive(vnv) +is_contiguous(vnv::VarNameVector) = !has_inactive(vnv) function nextrange(vnv::VarNameVector, x) # If `vnv` is empty, return immediately. From 0968a076b9ccbdc088433288a6e78ca26502952b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:46:54 +0000 Subject: [PATCH 056/182] moved additional methods description in internals to earlier in the doc --- docs/src/internals.md | 52 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 3c47a3d97..6e1e8b78b 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -33,11 +33,21 @@ To ensure that `varinfo` is simple and intuitive to work with, we need the under - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. -Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: +We also want some additional methods that are *not* part of the `Dict` or `Vector` interface: + + - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + + - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + +In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: + + - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + +Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: 1. Type-stable when possible, but functional when not. 2. Efficient storage and iteration when possible, but functional when not. - + The "but functional when not" is important as we want to support arbitrary models, which means that we can't always have these performance properties. In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). @@ -147,7 +157,7 @@ For example, we want to optimize code-paths which effectively boil down to inner varinfo = VarInfo(model) # Repeatedly sample from `model`. -for _ = 1:num_samples +for _ in 1:num_samples rand!(rng, model, varinfo) # Do something with `varinfo`. @@ -183,7 +193,9 @@ println("Before insertion: number of allocated entries $(DynamicPPL.num_allocat for i in 1:5 x = fill(true, rand(1:100)) DynamicPPL.update!(vnv, @varname(x), x) - println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") + println( + "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", + ) end ``` @@ -199,7 +211,9 @@ for i in 1:5 if DynamicPPL.num_allocated(vnv) > 10 DynamicPPL.contiguify!(vnv) end - println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") + println( + "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", + ) end ``` @@ -267,23 +281,11 @@ DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) In the end, we have the following "rough" performance characteristics for `VarNameVector`: -| Method | Is blazingly fast? | -| :-: | :-: | -| `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | -| `delete!` | ${\color{red} \times}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | -| `convert(Vector, ::VarNameVector)` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | - -### Additional methods - -We also want some additional methods that are not part of the `Dict` or `Vector` interface: - - - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. - - - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. - -In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - - - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. +| Method | Is blazingly fast? | +|:--------------------------------------------------:|:--------------------------------------------------------------------------------------------:| +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `delete!` | ${\color{red} \times}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| `convert(Vector{T}, ::VarNameVector{<:VarName,T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | From 0d008a4ed05ae1172b4991c998dfbf3d7b2f0258 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:52:48 +0000 Subject: [PATCH 057/182] moved internals docs to a separate directory and split into files --- docs/make.jl | 2 +- docs/src/{internals.md => internals/varinfo.md} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename docs/src/{internals.md => internals/varinfo.md} (99%) diff --git a/docs/make.jl b/docs/make.jl index 056b90d7f..38c520d3a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,7 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], - "Internals" => "internals.md", + "Internals" => ["internals/varinfo.md"], ], checkdocs=:exports, ) diff --git a/docs/src/internals.md b/docs/src/internals/varinfo.md similarity index 99% rename from docs/src/internals.md rename to docs/src/internals/varinfo.md index 6e1e8b78b..40932d2f4 100644 --- a/docs/src/internals.md +++ b/docs/src/internals/varinfo.md @@ -1,4 +1,4 @@ -## Design of `VarInfo` +# Design of `VarInfo` # [`VarInfo`](@ref) is a fairly simple structure; it contains @@ -52,7 +52,7 @@ The "but functional when not" is important as we want to support arbitrary model In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). -### Type-stability +## Type-stability ## Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. @@ -122,7 +122,7 @@ Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entri Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. -### Efficient storage and iteration +## Efficient storage and iteration ## Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): @@ -277,7 +277,7 @@ varinfo_untyped_vnv[@varname(x)] DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) ``` -#### Performance summary +### Performance summary ### In the end, we have the following "rough" performance characteristics for `VarNameVector`: From ccd0d64ef40c9e363b8590ec0be3757bcd379267 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:58:09 +0000 Subject: [PATCH 058/182] more improvements to internals doc --- docs/src/internals/varinfo.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 40932d2f4..ae4335807 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -1,6 +1,12 @@ # Design of `VarInfo` # -[`VarInfo`](@ref) is a fairly simple structure; it contains +[`VarInfo`](@ref) is a fairly simple structure. + +```@docs; canonical=false +VarInfo +``` + +It contains - a `logp` field for accumulation of the log-density evaluation, and - a `metadata` field for storing information about the realizations of the different variables. @@ -13,7 +19,7 @@ Representing `logp` is fairly straight-forward: we'll just use a `Real` or an ar We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. -To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: +To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo`, and hence the underlying `metadata`, to replicate the following functionality of `Dict`: - `keys(::Dict)`: return all the `VarName`s present in `metadata`. - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. From 7c45e67fff93da3b94104a9541bffa12a09736bd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:58:34 +0000 Subject: [PATCH 059/182] formatting --- docs/src/internals/varinfo.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index ae4335807..b5e1aed60 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -1,6 +1,6 @@ -# Design of `VarInfo` # +# Design of `VarInfo` -[`VarInfo`](@ref) is a fairly simple structure. +[`VarInfo`](@ref) is a fairly simple structure. ```@docs; canonical=false VarInfo @@ -58,7 +58,7 @@ The "but functional when not" is important as we want to support arbitrary model In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). -## Type-stability ## +## Type-stability Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. @@ -128,7 +128,7 @@ Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entri Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. -## Efficient storage and iteration ## +## Efficient storage and iteration Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): @@ -283,7 +283,7 @@ varinfo_untyped_vnv[@varname(x)] DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) ``` -### Performance summary ### +### Performance summary In the end, we have the following "rough" performance characteristics for `VarNameVector`: From 373215b7c752f16f0e3a3fd8c54953edf718c0de Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:03:34 +0000 Subject: [PATCH 060/182] added tests for `delete!` and fixed reference to old method --- test/varnamevector.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index cb5ac0632..bdb6ecb6b 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -235,6 +235,15 @@ end @test vnv[vn_right] == val_right .+ 100 end + # `delete!` + @testset "delete!" begin + vnv = deepcopy(vnv_base) + delete!(vnv, vn_left) + @test !haskey(vnv, vn_left) + delete!(vnv, vn_right) + @test !haskey(vnv, vn_right) + end + # `push!` & `update!` @testset "push!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -351,7 +360,7 @@ end for i in 1:10 x = fill(true, rand(1:n)) DynamicPPL.update!(vnv, vn, x) - DynamicPPL.inactive_ranges_sweep!(vnv) + DynamicPPL.contiguify!(vnv) @test DynamicPPL.num_allocated(vnv, vn) == length(x) end end From 0cdafbf4133f26311c752ea598a5ea90c5622ba2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:04:08 +0000 Subject: [PATCH 061/182] addition to `delete!` test --- test/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index bdb6ecb6b..a37d23a6f 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -240,6 +240,7 @@ end vnv = deepcopy(vnv_base) delete!(vnv, vn_left) @test !haskey(vnv, vn_left) + @test haskey(vnv, vn_right) delete!(vnv, vn_right) @test !haskey(vnv, vn_right) end From 51c041febae9257a46dda3983fd9383c9b37d410 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:17:21 +0000 Subject: [PATCH 062/182] added `values_as` impls for `VarNameVector` --- docs/src/internals/varinfo.md | 2 +- src/varinfo.jl | 2 +- src/varnamevector.jl | 13 ++++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index b5e1aed60..70721d35c 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -294,4 +294,4 @@ In the end, we have the following "rough" performance characteristics for `VarNa | `push!` | ${\color{green} \checkmark}$ | | `delete!` | ${\color{red} \times}$ | | `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | -| `convert(Vector{T}, ::VarNameVector{<:VarName,T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | +| `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | diff --git a/src/varinfo.jl b/src/varinfo.jl index 48ad94392..37098f68a 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -901,7 +901,7 @@ TypedVarInfo(vi::TypedVarInfo) = vi function TypedVarInfo(vi::VectorVarInfo) logp = getlogp(vi) num_produce = get_num_produce(vi) - nt = group_by_symbol(vi.metadata) + nt = NamedTuple(group_by_symbol(vi.metadata)) return VarInfo(nt, Ref(logp), Ref(num_produce)) end diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 3fa295112..279795d8a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -454,7 +454,7 @@ function group_by_symbol(vnv::VarNameVector) ) end - return NamedTuple{Tuple(keys(d))}(nt_vals) + return OrderedDict(zip(keys(d), nt_vals)) end function Base.delete!(vnv::VarNameVector, vn::VarName) @@ -504,3 +504,14 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) return vnv end + +values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] +function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} + return convert(Vector{T}, values_as(vnv, Vector)) +end +function values_as(vnv::VarNameVector, ::Type{NamedTuple}) + return NamedTuple(zip(map(Symbol, keys(vnv)), values(vnv))) +end +function values_as(vnv::VarNameVector, ::Type{D}) where {D<:AbstractDict} + return ConstructionBase.constructorof(D)(pairs(vnv)) +end From 20b37427c2b640f98d7576235206856156343a8f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:41:18 +0000 Subject: [PATCH 063/182] added docs for `replace_valus` and `values_as` for `VarNameVector` --- docs/src/internals/varinfo.md | 10 +++++ src/varnamevector.jl | 70 ++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 70721d35c..8fe456ab7 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -295,3 +295,13 @@ In the end, we have the following "rough" performance characteristics for `VarNa | `delete!` | ${\color{red} \times}$ | | `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | + +## Other methods + +```@docs +DynamicPPL.replace_values(::VarNameVector, vals::AbstractVector) +``` + +```@docs; canonical=false +DynamicPPL.values_as(::VarNameVector) +``` diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 279795d8a..eaad004c6 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,5 +1,3 @@ -# Similar to `Metadata` but representing a `Vector` and simpler interface. -# TODO: Should we subtype `AbstractVector` or `AbstractDict`? """ VarNameVector @@ -116,6 +114,46 @@ function VarNameVector( ) end +""" + replace_values(vnv::VarNameVector, vals::AbstractVector) + +Replace the values in `vnv` with `vals`. + +This is useful when we want to update the entire underlying vector of values +in one go or if we want to change the how the values are stored, e.g. alter the `eltype`. + +!!! warning + This replaces the raw underlying values, and so care should be taken when using this + function. For example, if `vnv` has any inactive entries, then the provided `vals` + should also contain the inactive entries to avoid unexpected behavior. + +# Example + +```jldoctest varnamevector-replace-values +julia> using DynamicPPL: VarNameVector, replace_values + +julia> vnv = VarNameVector(@varname(x) => [1.0]); + +julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] +true +``` + +This is also useful when we want to differentiate wrt. the values +using automatic differentiation, e.g. ForwardDiff.jl. + +```jldoctest varnamevector-replace-values +julia> using ForwardDiff: ForwardDiff + +julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) +f (generic function with 1 method) + +julia> ForwardDiff.gradient(f, [1.0]) +1-element Vector{Float64}: + 2.0 +``` +""" +replace_values(vnv::VarNameVector, vals) = Setfield.@set vnv.vals = vals + # Some `VarNameVector` specific functions. getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] @@ -505,6 +543,34 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) return vnv end +""" + values_as(vnv::VarNameVector[, T]) + +Return the values/realizations in `vnv` as type `T`, if implemented. + +If no type `T` is provided, return values as stored in `vnv`. + +# Examples + +```jldoctest +julia> using DynamicPPL: VarNameVector + +julia> vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2.0]); + +julia> values_as(vnv) == [1.0, 2.0] +true + +julia> values_as(vnv, Vector{Float32}) == Vector{Float32}([1.0, 2.0]) +true + +julia> values_as(vnv, OrderedDict) == OrderedDict(@varname(x) => 1.0, @varname(y) => [2.0]) +true + +julia> values_as(vnv, NamedTuple) == (x = 1.0, y = [2.0]) +true +``` +""" +values_as(vnv::VarNameVector) = values_as(vnv, Vector) values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} return convert(Vector{T}, values_as(vnv, Vector)) From ef6c618e2983abefa4c7565788434368939e5026 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:44:08 +0000 Subject: [PATCH 064/182] fixed doctest --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 37098f68a..cbea585f6 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1735,7 +1735,7 @@ function unset_flag!(vi::VarInfo, vn::VarName, flag::String) return vi end function unset_flag!(metadata::Metadata, vn::VarName, flag::String) - metadata.flags[flag][getidx(vi, vn)] = false + metadata.flags[flag][getidx(metadata, vn)] = false return metadata end unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv From 8a1209c67e1c3399de186a22f56b48ad8bcd81d5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:44:24 +0000 Subject: [PATCH 065/182] formatting --- docs/src/internals/varinfo.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 8fe456ab7..1312f152c 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -287,13 +287,13 @@ DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) In the end, we have the following "rough" performance characteristics for `VarNameVector`: -| Method | Is blazingly fast? | -|:--------------------------------------------------:|:--------------------------------------------------------------------------------------------:| -| `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | -| `delete!` | ${\color{red} \times}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| Method | Is blazingly fast? | +|:---------------------------------------:|:--------------------------------------------------------------------------------------------:| +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `delete!` | ${\color{red} \times}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | ## Other methods From adeadf0b747baa775e1137fa923b58871c3daa6b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:56:50 +0000 Subject: [PATCH 066/182] temporarily disable doctests so we can build docs --- docs/make.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/make.jl b/docs/make.jl index 38c520d3a..c38853d5b 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -23,6 +23,7 @@ makedocs(; "Internals" => ["internals/varinfo.md"], ], checkdocs=:exports, + doctest=false, ) deploydocs(; repo="github.com/TuringLang/DynamicPPL.jl.git", push_preview=true) From 7ff179d77c6e6386abfa5fd8b120c8df75ba9cb2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:59:05 +0000 Subject: [PATCH 067/182] added missing compat entry for ForwardDiff in docs --- docs/Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Project.toml b/docs/Project.toml index 48ebe173c..5b8351de9 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,6 +3,7 @@ DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c" MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" @@ -14,6 +15,7 @@ DataStructures = "0.18" Distributions = "0.25" Documenter = "1" FillArrays = "0.13, 1" +ForwardDiff = "0.10" LogDensityProblems = "2" MCMCChains = "5, 6" MLUtils = "0.3, 0.4" From c7ec08af999afb65838cfe752541e8306102eb2c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 08:49:26 +0000 Subject: [PATCH 068/182] moved some shared code into methods to make things a bit cleaner --- src/varnamevector.jl | 91 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 21 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index eaad004c6..96fadacbe 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -336,12 +336,36 @@ function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val) return nothing end +""" + shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + +Shifts the elements of `x` starting from index `start` by `n` to the right. +""" function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) x[(start + n):end] = x[start:(end - n)] return x end +""" + shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) + +Shifts the ranges of variables in `vnv` starting from index `idx` by `n`. +""" +function shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) + for i in (idx + 1):length(vnv.ranges) + vnv.ranges[i] = vnv.ranges[i] .+ n + end + return nothing +end + # `update!` and `update!!`: update a variable in the varname vector. +""" + update!(vnv::VarNameVector, vn::VarName, val[, transform]) + +Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. + +If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). +""" function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) # Here we just add a new entry. @@ -410,14 +434,12 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) n_extra = n_new - n_allocated # Allocate. resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift current values. + # Shift current values. shift_right!(vnv.vals, end_old + 1, n_extra) # No more inactive entries. had_inactive && delete!(vnv.num_inactive, idx) # Update the ranges for all variables after this one. - for i in (idx + 1):length(vnv.varnames) - vnv.ranges[i] = vnv.ranges[i] .+ n_extra - end + shift_subsequent_ranges_by!(vnv, idx, n_extra) elseif n_new == n_allocated # => No more inactive entries. had_inactive && delete!(vnv.num_inactive, idx) @@ -474,7 +496,12 @@ function contiguify!(vnv::VarNameVector) return vnv end -# Typed version. +""" + group_by_symbol(vnv::VarNameVector) + +Return a dictionary mapping symbols to `VarNameVector`s with +varnames containing that symbol. +""" function group_by_symbol(vnv::VarNameVector) # Group varnames in `vnv` by the symbol. d = OrderedDict{Symbol,Vector{VarName}}() @@ -495,6 +522,41 @@ function group_by_symbol(vnv::VarNameVector) return OrderedDict(zip(keys(d), nt_vals)) end +""" + shift_index_left!(vnv::VarNameVector, idx::Int) + +Shift the index `idx` to the left by one and update the relevant fields. + +!!! warning + This does not check if index we're shifting to is already occupied. +""" +function shift_index_left!(vnv::VarNameVector, idx::Int) + # Shift the index in the lookup table. + vn = vnv.varnames[idx] + vnv.varname_to_index[vn] = idx - 1 + # Shift the index in the inactive ranges. + if haskey(vnv.num_inactive, idx) + # Done in increasing order => don't need to worry about + # potentially shifting the same index twice. + vnv.num_inactive[idx - 1] = pop!(vnv.num_inactive, idx) + end +end + +""" + shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) + +Shift the indices for all variables after `idx` to the left by one and update +the relevant fields. + +This just +""" +function shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) + # Shift the indices for all variables after `idx`. + for idx_to_shift in (idx + 1):length(vnv.varnames) + shift_index_left!(vnv, idx_to_shift) + end +end + function Base.delete!(vnv::VarNameVector, vn::VarName) # Error if we don't have the variable. !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) @@ -505,6 +567,7 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) # Delete the values. r_start = first(getrange(vnv, idx)) n_allocated = num_allocated(vnv, idx) + # NOTE: `deleteat!` also results in a `resize!` so we don't need to do that. deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) # Delete `vn` from the lookup table. @@ -515,24 +578,10 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) # Re-adjust the indices for varnames occuring after `vn` so # that they point to the correct indices after the deletions below. - for idx_to_shift in (idx + 1):length(vnv.varnames) - vn = vnv.varnames[idx_to_shift] - if idx_to_shift > idx - # Shift the index in the lookup table. - vnv.varname_to_index[vn] = idx_to_shift - 1 - # Shift the index in the inactive ranges. - if haskey(vnv.num_inactive, idx_to_shift) - # Done in increasing order => don't need to worry about - # potentially shifting the same index twice. - vnv.num_inactive[idx_to_shift - 1] = pop!(vnv.num_inactive, idx_to_shift) - end - end - end + shift_subsequent_indices_left!(vnv, idx) # Re-adjust the ranges for varnames occuring after `vn`. - for idx_to_shift in (idx + 1):length(vnv.varnames) - vnv.ranges[idx_to_shift] = vnv.ranges[idx_to_shift] .- n_allocated - end + shift_subsequent_ranges_by!(vnv, idx, -n_allocated) # Delete references from vector fields, thus shifting the indices of # varnames occuring after `vn` by one to the left, as we adjusted for above. From c5a5e585907fede7970d6b2138a648289ce2de8a Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 09:38:18 +0000 Subject: [PATCH 069/182] added impl of `merge` for `VarNameVector` --- src/varnamevector.jl | 73 +++++++++++++++++++++++++++++++++++++++++++ test/varnamevector.jl | 27 ++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 96fadacbe..30d848518 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -277,6 +277,79 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) +function Base.merge(left::VarNameVector, right::VarNameVector) + # Return early if possible. + isempty(left) && return deepcopy(right) + isempty(right) && return deepcopy(left) + + # A very simple way of doing this would be to just + # convert both to `OrderedDict` and merge. + # However, we need to also account of transformations. + + # Determine varnames. + vns_left = left.varnames + vns_right = right.varnames + vns_both = union(vns_left, vns_right) + + # Determine `eltype` of `vals`. + T_left = eltype(left.vals) + T_right = eltype(right.vals) + T = promote_type(T_left, T_right) + # TODO: Is this necessary? + if !(T <: Real) + T = Real + end + + # Determine `eltype` of `varnames`. + V_left = eltype(left.varnames) + V_right = eltype(right.varnames) + V = promote_type(V_left, V_right) + if !(V <: VarName) + V = VarName + end + + # Determine `eltype` of `transforms`. + F_left = eltype(left.transforms) + F_right = eltype(right.transforms) + F = promote_type(F_left, F_right) + + # Allocate. + varnames_to_index = OrderedDict{V,Int}() + ranges = UnitRange{Int}[] + vals = T[] + transforms = F[] + + # Range offset. + offset = 0 + + for (idx, vn) in enumerate(vns_both) + # Extract the necessary information from `left` or `right`. + if vn in vns_left && !(vn in vns_right) + # `vn` is only in `left`. + varnames_to_index[vn] = idx + val = getindex_raw(left, vn) + n = length(val) + r = (offset + 1):(offset + n) + f = gettransform(left, vn) + else + # `vn` is either in both or just `right`. + varnames_to_index[vn] = idx + val = getindex_raw(right, vn) + n = length(val) + r = (offset + 1):(offset + n) + f = gettransform(right, vn) + end + # Update. + append!(vals, val) + push!(ranges, r) + push!(transforms, f) + # Increment `offset`. + offset += n + end + + return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) +end + # `similar` similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index a37d23a6f..b30135e74 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -245,6 +245,33 @@ end @test !haskey(vnv, vn_right) end + # `merge` + @testset "merge" begin + # When there are no inactive entries, `merge` on itself result in the same. + @test merge(vnv_base, vnv_base) == vnv_base + + # Merging with empty should result in the same. + @test merge(vnv_base, similar(vnv_base)) == vnv_base + @test merge(similar(vnv_base), vnv_base) == vnv_base + + # With differences. + vnv_left_only = deepcopy(vnv_base) + delete!(vnv_left_only, vn_right) + vnv_right_only = deepcopy(vnv_base) + delete!(vnv_right_only, vn_left) + + # `(x,)` and `(x, y)` should be `(x, y)`. + @test merge(vnv_left_only, vnv_base) == vnv_base + # `(x, y)` and `(x,)` should be `(x, y)`. + @test merge(vnv_base, vnv_left_only) == vnv_base + # `(x, y)` and `(y,)` should be `(x, y)`. + @test merge(vnv_base, vnv_right_only) == vnv_base + # `(y,)` and `(x, y)` should be `(y, x)`. + vnv_merged = merge(vnv_right_only, vnv_base) + @test vnv_merged != vnv_base + @test collect(keys(vnv_merged)) == [vn_right, vn_left] + end + # `push!` & `update!` @testset "push!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) From c376d956f77a96904828bd9a50bb6ecb50c43dbf Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 09:41:02 +0000 Subject: [PATCH 070/182] renamed a few variables in `merge` impl for `VarNameVector` --- src/varnamevector.jl | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 30d848518..f33f35365 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -277,23 +277,19 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) -function Base.merge(left::VarNameVector, right::VarNameVector) +function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) # Return early if possible. - isempty(left) && return deepcopy(right) - isempty(right) && return deepcopy(left) - - # A very simple way of doing this would be to just - # convert both to `OrderedDict` and merge. - # However, we need to also account of transformations. + isempty(left_vnv) && return deepcopy(right_vnv) + isempty(right_vnv) && return deepcopy(left_vnv) # Determine varnames. - vns_left = left.varnames - vns_right = right.varnames + vns_left = left_vnv.varnames + vns_right = right_vnv.varnames vns_both = union(vns_left, vns_right) # Determine `eltype` of `vals`. - T_left = eltype(left.vals) - T_right = eltype(right.vals) + T_left = eltype(left_vnv.vals) + T_right = eltype(right_vnv.vals) T = promote_type(T_left, T_right) # TODO: Is this necessary? if !(T <: Real) From f71baa54014ab3a798d69c0baf2b7d9c5e559369 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 09:41:27 +0000 Subject: [PATCH 071/182] forgot to include some changes in previous commit --- src/varnamevector.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f33f35365..8082e3068 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -297,16 +297,16 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) end # Determine `eltype` of `varnames`. - V_left = eltype(left.varnames) - V_right = eltype(right.varnames) + V_left = eltype(left_vnv.varnames) + V_right = eltype(right_vnv.varnames) V = promote_type(V_left, V_right) if !(V <: VarName) V = VarName end # Determine `eltype` of `transforms`. - F_left = eltype(left.transforms) - F_right = eltype(right.transforms) + F_left = eltype(left_vnv.transforms) + F_right = eltype(right_vnv.transforms) F = promote_type(F_left, F_right) # Allocate. @@ -323,17 +323,17 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) if vn in vns_left && !(vn in vns_right) # `vn` is only in `left`. varnames_to_index[vn] = idx - val = getindex_raw(left, vn) + val = getindex_raw(left_vnv, vn) n = length(val) r = (offset + 1):(offset + n) - f = gettransform(left, vn) + f = gettransform(left_vnv, vn) else # `vn` is either in both or just `right`. varnames_to_index[vn] = idx - val = getindex_raw(right, vn) + val = getindex_raw(right_vnv, vn) n = length(val) r = (offset + 1):(offset + n) - f = gettransform(right, vn) + f = gettransform(right_vnv, vn) end # Update. append!(vals, val) From af25f3cfc6561d292a482db5ba1f8f5a91b9176f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:54:26 +0000 Subject: [PATCH 072/182] added impl of `subset` for `VarNameVector` --- src/varnamevector.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 8082e3068..219c93416 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -346,6 +346,19 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) end +function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) + # NOTE: This does not specialize types when possible. + vnv_new = similar(vnv) + # Return early if possible. + isempty(vnv) && return vnv_new + + for vn in vns + push!(vnv_new, vn, getval(vnv, vn), gettransform(vnv, vn)) + end + + return vnv_new +end + # `similar` similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) From c28f0762ac73b36e4dda5fa0782bdb0781647f29 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:54:38 +0000 Subject: [PATCH 073/182] fixed `pairs` impl for `VarNameVector` --- src/varnamevector.jl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 219c93416..02c659c79 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -211,12 +211,7 @@ Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. Base.keys(vnv::VarNameVector) = vnv.varnames Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) - -function Base.pairs(vnv::VarNameVector) - return Iterators.zip( - vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) - ) -end +Base.pairs(vnv::VarNameVector) = (vn => vnv[vn] for vn in keys(vnv)) Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) From f5d2c6391a4511f29129ee463aa167fcbf378c01 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:55:02 +0000 Subject: [PATCH 074/182] added missing impl of `subset` for `VectorVarInfo` --- src/varinfo.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index cbea585f6..ffe792886 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -284,6 +284,11 @@ function subset(varinfo::UntypedVarInfo, vns::AbstractVector{<:VarName}) return VarInfo(metadata, varinfo.logp, varinfo.num_produce) end +function subset(varinfo::VectorVarInfo, vns::AbstractVector{<:VarName}) + metadata = subset(varinfo.metadata, vns) + return VarInfo(metadata, varinfo.logp, varinfo.num_produce) +end + function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName{sym}}) where {sym} # If all the variables are using the same symbol, then we can just extract that field from the metadata. metadata = subset(getfield(varinfo.metadata, sym), vns) From 3eb6c7f2fa72bebd696de4389c89d682485c569a Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:55:16 +0000 Subject: [PATCH 075/182] added missing impl of `merge_metadata` for `VarNameVector` --- src/varinfo.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index ffe792886..d9e71335e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -364,6 +364,10 @@ function _merge(varinfo_left::VarInfo, varinfo_right::VarInfo) ) end +function merge_metadata(vnv_left::VarNameVector, vnv_right::VarNameVector) + return merge(vnv_left, vnv_right) +end + @generated function merge_metadata( metadata_left::NamedTuple{names_left}, metadata_right::NamedTuple{names_right} ) where {names_left,names_right} From 9ba8144dc9874d8d8548cf3563008d2054410375 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:55:29 +0000 Subject: [PATCH 076/182] added a bunch of `from_vec_transform` and `tovec` impls to make `VarNameVector` work with `Cholesky`, etc. --- src/varnamevector.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 02c659c79..8a5be6656 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -70,8 +70,17 @@ FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) +from_vec_transform(x::Real) = FromVec(()) +from_vec_transform(x::AbstractArray) = FromVec(size(x)) +from_vec_transform(C::Cholesky) = from_vec_transform(C.UL) +from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu +from_vec_transform(L::LinearAlgebra.LowerTriangular) = transpose ∘ from_vec_transform + tovec(x::Real) = [x] tovec(x::AbstractArray) = vec(x) +tovec(C::Cholesky) = tovec(C.UL) +tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) +tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec From acd695143ebf67201db631c0446a3796380ae083 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:56:39 +0000 Subject: [PATCH 077/182] make default args use `from_vec_transform` rather than `FromVec` --- src/varnamevector.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 8a5be6656..e98d0b09f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -94,7 +94,7 @@ VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) function VarNameVector( - varnames::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) + varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) ) # TODO: Check uniqueness of `varnames`? @@ -407,7 +407,7 @@ function nextrange(vnv::VarNameVector, x) end # `push!` and `push!!`: add a variable to the varname vector. -function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) +function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) # NOTE: We need to compute the `nextrange` BEFORE we start mutating @@ -452,7 +452,7 @@ Either add a new entry or update existing entry for `vn` in `vnv` with the valu If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). """ -function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) +function update!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) if !haskey(vnv, vn) # Here we just add a new entry. return push!(vnv, vn, val, transform) From 790f743a445a90067b8b5dd80ae444fb3c008f12 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:56:58 +0000 Subject: [PATCH 078/182] fixed `values_as` fro `VarInfo` with `VarNameVector` as `metadata` --- src/varinfo.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index d9e71335e..545fcbdb3 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2079,6 +2079,10 @@ end function values_as(vi::UntypedVarInfo, ::Type{D}) where {D<:AbstractDict} return ConstructionBase.constructorof(D)(values_from_metadata(vi.metadata)) end +values_as(vi::VectorVarInfo, ::Type{NamedTuple}) = values_as(vi.metadata, NamedTuple) +function values_as(vi::VectorVarInfo, ::Type{D}) where {D<:AbstractDict} + return values_as(vi.metadata, D) +end function values_as(vi::VarInfo{<:NamedTuple{names}}, ::Type{NamedTuple}) where {names} iter = Iterators.flatten(values_from_metadata(getfield(vi.metadata, n)) for n in names) @@ -2098,3 +2102,5 @@ function values_from_metadata(md::Metadata) vn in md.vns ) end + +values_from_metadata(md::VarNameVector) = pairs(md) From c474bb08e68ba7bee306bf47a2c669350ee53c7f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 4 Jan 2024 09:51:17 +0000 Subject: [PATCH 079/182] fixed impl of `getindex_raw` when using integer index for `VarNameVector` --- src/varnamevector.jl | 54 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index e98d0b09f..4fedfdeb6 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -178,6 +178,13 @@ Returns `true` if `vnv` has inactive ranges. """ has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) +""" + num_inactive(vnv::VarNameVector) + +Return the number of inactive entries in `vnv`. +""" +num_inactive(vnv::VarNameVector) = sum(values(vnv.num_inactive)) + """ num_inactive(vnv::VarNameVector, vn::VarName) @@ -232,11 +239,52 @@ function Base.getindex(vnv::VarNameVector, vn::VarName) return f(x) end -getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[i] -function getindex_raw(vnv::VarNameVector, vn::VarName) - return vnv.vals[getrange(vnv, vn)] +function find_range_from_sorted(ranges::AbstractVector{<:AbstractRange}, x) + # TODO: Assume `ranges` to be sorted and contiguous, and use `searchsortedfirst` + # for a more efficient approach. + range_idx = findfirst(Base.Fix1(∈, x), ranges) + + # If we're out of bounds, we raise an error. + if range_idx === nothing + throw(ArgumentError("Value $x is not in any of the ranges.")) + end + + return range_idx end +function adjusted_ranges(vnv::VarNameVector) + # Every range following inactive entries needs to be shifted. + offset = 0 + ranges_adj = similar(vnv.ranges) + for (idx, r) in enumerate(vnv.ranges) + # Remove the `offset` in `r` due to inactive entries. + ranges_adj[idx] = r .- offset + # Update `offset`. + offset += get(vnv.num_inactive, idx, 0) + end + + return ranges_adj +end + +function index_to_raw_index(vnv::VarNameVector, i::Int) + # If we don't have any inactive entries, there's nothing to do. + has_inactive(vnv) || return i + + # Get the adjusted ranges. + ranges_adj = adjusted_ranges(vnv) + # Determine the adjusted range that the index corresponds to. + r_idx = find_range_from_sorted(ranges_adj, i) + r = vnv.ranges[r_idx] + # Determine how much of the index `i` is used to get to this range. + i_used = r_idx == 1 ? 0 : sum(length, ranges_adj[1:(r_idx - 1)]) + # Use remainder to index into `r`. + i_remainder = i - i_used + return r[i_remainder] +end + +getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] +getindex_raw(vnv::VarNameVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] + # `getindex` for `Colon` function Base.getindex(vnv::VarNameVector, ::Colon) return if has_inactive(vnv) From 8251463ae509001dbd300ffb96f5827e9fce1d58 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 4 Jan 2024 12:39:03 +0000 Subject: [PATCH 080/182] added tests for `getindex` with `Int` index for `VarNameVector` --- test/varnamevector.jl | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b30135e74..ce46e86fc 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -206,9 +206,13 @@ end # `getindex` @testset "getindex" begin - # `getindex` + # With `VarName` index. @test vnv_base[vn_left] == val_left @test vnv_base[vn_right] == val_right + + # With `Int` index. + val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) + @test all(vnv_base[i] == val_vec[i] for i in 1:length(val_vec)) end # `setindex!` @@ -222,8 +226,15 @@ end # `getindex_raw` @testset "getindex_raw" begin + # With `VarName` index. @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) + # With `Int` index. + val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) + @test all( + DynamicPPL.getindex_raw(vnv_base, i) == val_vec[i] for + i in 1:length(val_vec) + ) end # `setindex_raw!` @@ -298,12 +309,16 @@ end end DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] @test vnv[vn] == val .+ 1 @test length(vnv) == expected_length - @test length(vnv[:]) == length(vnv) + @test length(x) == length(vnv) # There should be no redundant values in the underlying vector. @test !DynamicPPL.has_inactive(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -319,9 +334,13 @@ end end DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] @test vnv[vn] == val .+ 1 @test length(vnv) == expected_length - @test length(vnv[:]) == length(vnv) + @test length(x) == length(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -336,9 +355,13 @@ end length(vnv) + length(val) end DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] @test vnv[vn] == val .+ 1 @test length(vnv) == expected_length - @test length(vnv[:]) == length(vnv) + @test length(x) == length(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) end end end @@ -395,12 +418,6 @@ end end end -has_varnamevector(vi) = false -function has_varnamevector(vi::VarInfo) - return vi.metadata isa VarNameVector || - (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) -end - @testset "VarInfo + VarNameVector" begin models = DynamicPPL.TestUtils.DEMO_MODELS @testset "$(model.f)" for model in models From 5df703170190b1234d2525693506fc8cac82fd1e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 4 Jan 2024 12:39:22 +0000 Subject: [PATCH 081/182] fix for `setindex!` and `setindex_raw!` for `VarNameVector` --- src/varnamevector.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 4fedfdeb6..f15f86078 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -311,7 +311,7 @@ function Base.setindex!(vnv::VarNameVector, val, vn::VarName) return setindex_raw!(vnv, f(val), vn) end -setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] = val function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) return vnv.vals[getrange(vnv, vn)] = val end From 683b776bed196fd060325b250ef9d0cb289ee8dc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 11:32:36 +0000 Subject: [PATCH 082/182] introduction of `from_vec_transform` and `tovec` and its usage in `VarInfo` --- src/utils.jl | 7 +++---- src/varinfo.jl | 2 +- src/varnamevector.jl | 32 ++++++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 70dec4e28..b00425c94 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -212,9 +212,7 @@ invlink_transform(dist) = inverse(link_transform(dist)) ##################################################### vectorize(d, r) = vectorize(r) -vectorize(r::Real) = [r] -vectorize(r::AbstractArray{<:Real}) = copy(vec(r)) -vectorize(r::Cholesky) = copy(vec(r.UL)) +vectorize(r) = tovec(r) # NOTE: # We cannot use reconstruct{T} because val is always Vector{Real} then T will be Real. @@ -240,7 +238,8 @@ reconstruct(::MatrixDistribution, val::AbstractMatrix{<:Real}) = copy(val) reconstruct(::Inverse{Bijectors.VecCorrBijector}, ::LKJ, val::AbstractVector) = copy(val) function reconstruct(dist::LKJCholesky, val::AbstractVector{<:Real}) - return reconstruct(dist, Matrix(reshape(val, size(dist)))) + f = from_vec_transform(dist) + return reconstruct(dist, f(val)) end function reconstruct(dist::LKJCholesky, val::AbstractMatrix{<:Real}) return Cholesky(val, dist.uplo, 0) diff --git a/src/varinfo.jl b/src/varinfo.jl index 545fcbdb3..b3174c20a 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -135,7 +135,7 @@ function metadata_to_varnamevector(md::Metadata) vals = copy(md.vals) transforms = map(md.dists) do dist # TODO: Handle linked distributions. - FromVec(size(dist)) + from_vec_transform(dist) end return VarNameVector( diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f15f86078..ebf7a8e79 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -58,7 +58,7 @@ function VarNameVector{K,V}() where {K,V} end # Useful transformation going from the flattened representation. -struct FromVec{Sz} +struct FromVec{Sz} <: Bijectors.Bijector sz::Sz end @@ -67,24 +67,44 @@ FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) # TODO: Should we materialize the `reshape`? (f::FromVec)(x) = reshape(x, f.sz) (f::FromVec{Tuple{}})(x) = only(x) +# TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) +struct ToChol <: Bijectors.Bijector + uplo::Char +end + +Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) +Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) + from_vec_transform(x::Real) = FromVec(()) +from_vec_transform(x::AbstractVector) = identity from_vec_transform(x::AbstractArray) = FromVec(size(x)) -from_vec_transform(C::Cholesky) = from_vec_transform(C.UL) +function from_vec_transform(C::Cholesky) + return ToChol(C.uplo) ∘ from_vec_transform(C.UL) +end from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu -from_vec_transform(L::LinearAlgebra.LowerTriangular) = transpose ∘ from_vec_transform +function from_vec_transform(L::LinearAlgebra.LowerTriangular) + return transpose ∘ from_vec_transform(transpose(L)) +end + +# FIXME: remove the `rand` below. +from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) +# We want to use the inverse of `FromVec` so it preserves the size information. +Bijectors.transform(::Bijectors.Inverse{<:FromVec}, x) = tovec(x) + +# FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix +# flattened, while using `tovec` below flattenes only the necessary entries. +# => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, +# or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. tovec(x::Real) = [x] tovec(x::AbstractArray) = vec(x) tovec(C::Cholesky) = tovec(C.UL) tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) -Bijectors.inverse(f::FromVec) = tovec -Bijectors.inverse(f::FromVec{Tuple{}}) = tovec - # More convenient constructors. collect_maybe(x) = collect(x) collect_maybe(x::AbstractArray) = x From 4dae00d9e8a1b328b2e287a52b6ae4da3f9736f1 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 11:33:55 +0000 Subject: [PATCH 083/182] moved definition of `is_splat_symbol` to the file where it's used --- src/compiler.jl | 2 -- src/model.jl | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler.jl b/src/compiler.jl index e7c44d16b..081a4bca9 100644 --- a/src/compiler.jl +++ b/src/compiler.jl @@ -591,8 +591,6 @@ function namedtuple_from_splitargs(splitargs) return :(NamedTuple{$names_expr}($vals)) end -is_splat_symbol(s::Symbol) = startswith(string(s), "#splat#") - """ build_output(modeldef, linenumbernode) diff --git a/src/model.jl b/src/model.jl index c0cc2f26f..0b451e36a 100644 --- a/src/model.jl +++ b/src/model.jl @@ -963,6 +963,8 @@ function _evaluate!!(model::Model, varinfo::AbstractVarInfo, context::AbstractCo return model.f(args...; kwargs...) end +is_splat_symbol(s::Symbol) = startswith(string(s), "#splat#") + """ make_evaluate_args_and_kwargs(model, varinfo, context) From e3b52a48557b219c585b37e28dd0db9bf26fa899 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:30:59 +0000 Subject: [PATCH 084/182] added `VarInfo` constructor with vector input for `VectorVarInfo` --- src/varinfo.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index b3174c20a..7771e64d2 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -119,6 +119,12 @@ function VarInfo(old_vi::UntypedVarInfo, spl, x::AbstractVector) return new_vi end +function VarInfo(old_vi::VectorVarInfo, spl, x::AbstractVector) + new_vi = deepcopy(old_vi) + new_vi[spl] = x + return new_vi +end + function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) md = newmetadata(old_vi.metadata, Val(getspace(spl)), x) return VarInfo( From 9626be14d23691d597b10fee277bd069ef97402d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:31:26 +0000 Subject: [PATCH 085/182] make `extract_priors` take the `rng` as an argument --- src/extract_priors.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index bb6721a9c..a22e65fad 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -110,9 +110,15 @@ julia> length(extract_priors(rng, model)[@varname(x)]) 9 ``` """ -extract_priors(model::Model) = extract_priors(Random.default_rng(), model) +extract_priors(args...) = extract_priors(Random.default_rng(), args...) function extract_priors(rng::Random.AbstractRNG, model::Model) context = PriorExtractorContext(SamplingContext(rng)) evaluate!!(model, VarInfo(), context) return context.priors end + +function extract_priors(::Random.AbstractRNG, model::Model, varinfo::AbstractVarInfo) + context = PriorExtractorContext(DefaultContext()) + evaluate!!(model, varinfo, context) + return context.priors +end From e731fd69c19fb74b1dc0032c720153502e4ed7d4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:31:44 +0000 Subject: [PATCH 086/182] added `replace_values` for `Metadata` --- src/varinfo.jl | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 7771e64d2..ff6617db5 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -196,6 +196,12 @@ function VarInfo(rng::Random.AbstractRNG, model::Model, context::AbstractContext return VarInfo(rng, model, SampleFromPrior(), context) end +function replace_values(md::Metadata, vals) + return Metadata( + md.idcs, md.vns, md.ranges, vals, md.dists, md.gids, md.orders, md.flags + ) +end + @generated function newmetadata( metadata::NamedTuple{names}, ::Val{space}, x ) where {names,space} @@ -205,21 +211,7 @@ end mdf = :(metadata.$f) if inspace(f, space) || length(space) == 0 len = :(sum(length, $mdf.ranges)) - push!( - exprs, - :( - $f = Metadata( - $mdf.idcs, - $mdf.vns, - $mdf.ranges, - x[($offset + 1):($offset + $len)], - $mdf.dists, - $mdf.gids, - $mdf.orders, - $mdf.flags, - ) - ), - ) + push!(exprs, :($f = replace_values($mdf, x[($offset + 1):($offset + $len)]))) offset = :($offset + $len) else push!(exprs, :($f = $mdf)) From 0785abf100b17e18d8fa5500a5db2a7614c37674 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:32:48 +0000 Subject: [PATCH 087/182] make link and invlink act on the `metadata` field for `VarInfo` + implementations of these for `Metadata` and `VarNameVector` --- src/varinfo.jl | 174 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 151 insertions(+), 23 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ff6617db5..1e25ff6e1 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -634,12 +634,19 @@ Set the values of all the variables in `vi` to `val`. The values may or may not be transformed to Euclidean space. """ -function setall!(vi::UntypedVarInfo, val) - for r in vi.metadata.ranges - vi.metadata.vals[r] .= val[r] +setall!(vi::VarInfo, val) = _setall!(vi.metadata, val) + +function _setall!(metadata::Metadata, val) + for r in metadata.ranges + metadata.vals[r] .= val[r] + end +end +function _setall!(vnv::VarNameVector, val) + # TODO: Do something more efficient here. + for i in 1:length(vnv) + vnv[i] = val[i] end end -setall!(vi::TypedVarInfo, val) = _setall!(vi.metadata, val) @generated function _setall!(metadata::NamedTuple{names}, val) where {names} expr = Expr(:block) start = :(1) @@ -1192,6 +1199,10 @@ end end function _inner_transform!(vi::VarInfo, vn::VarName, dist, f) + return _inner_transform!(getmetadata(vi, vn), vi, vn, dist, f) +end + +function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, dist, f) # TODO: Use inplace versions to avoid allocations y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, getval(vi, vn)) yvec = vectorize(dist, y) @@ -1210,14 +1221,15 @@ end # an empty iterable for `SampleFromPrior`, so we need to override it here. # This is quite hacky, but seems safer than changing the behavior of `_getvns`. _getvns_link(varinfo::VarInfo, spl::AbstractSampler) = _getvns(varinfo, spl) -_getvns_link(varinfo::UntypedVarInfo, spl::SampleFromPrior) = nothing +_getvns_link(varinfo::VarInfo, spl::SampleFromPrior) = nothing function _getvns_link(varinfo::TypedVarInfo, spl::SampleFromPrior) return map(Returns(nothing), varinfo.metadata) end function link(::DynamicTransformation, varinfo::VarInfo, spl::AbstractSampler, model::Model) - return _link(varinfo, spl) + return _link(model, varinfo, spl) end + function link( ::DynamicTransformation, varinfo::ThreadSafeVarInfo{<:VarInfo}, @@ -1229,30 +1241,43 @@ function link( return Setfield.@set varinfo.varinfo = link(varinfo.varinfo, spl, model) end -function _link(varinfo::UntypedVarInfo, spl::AbstractSampler) +function _link(model::Model, varinfo::UntypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) return VarInfo( - _link_metadata!(varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo)), ) end -function _link(varinfo::TypedVarInfo, spl::AbstractSampler) +function _link(model::Model, varinfo::VectorVarInfo, spl::AbstractSampler) + varinfo = deepcopy(varinfo) + return VarInfo( + _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + Base.Ref(getlogp(varinfo)), + Ref(get_num_produce(varinfo)), + ) +end + +function _link(model::Model, varinfo::TypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) md = _link_metadata_namedtuple!( - varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) + model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) ) return VarInfo(md, Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo))) end @generated function _link_metadata_namedtuple!( - varinfo::VarInfo, metadata::NamedTuple{names}, vns::NamedTuple, ::Val{space} + model::Model, + varinfo::VarInfo, + metadata::NamedTuple{names}, + vns::NamedTuple, + ::Val{space}, ) where {names,space} vals = Expr(:tuple) for f in names if inspace(f, space) || length(space) == 0 - push!(vals.args, :(_link_metadata!(varinfo, metadata.$f, vns.$f))) + push!(vals.args, :(_link_metadata!(model, varinfo, metadata.$f, vns.$f))) else push!(vals.args, :(metadata.$f)) end @@ -1260,7 +1285,7 @@ end return :(NamedTuple{$names}($vals)) end -function _link_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) +function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, target_vns) vns = metadata.vns # Construct the new transformed values, and keep track of their lengths. @@ -1272,8 +1297,8 @@ function _link_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) end # Transform to constrained space. - x = getval(varinfo, vn) - dist = getdist(varinfo, vn) + x = getval(metadata, vn) + dist = getdist(metadata, vn) f = link_transform(dist) y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) # Vectorize value. @@ -1308,10 +1333,65 @@ function _link_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) ) end +function _link_metadata!( + model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns +) + vns = keys(metadata) + # Need to extract the priors from the model. + dists = extract_priors(model, varinfo) + + # Construct the linking transformations. + link_transforms = map(vns) do vn + # If `vn` is not part of `target_vns`, the `identity` transformation is used. + if (target_vns !== nothing && vn ∉ target_vns) + return identity + end + + # Otherwise, we derive the transformation from the distribution. + link_transform(getindex(dists, vn)) + end + # Compute the transformed values. + ys = map(vns, link_transforms) do vn, f + # TODO: Do we need to handle scenarios where `vn` is not in `dists`? + dist = dists[vn] + x = getval(metadata, vn) + y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) + # Accumulate the log-abs-det jacobian correction. + acclogp!!(varinfo, -logjac) + # Return the transformed value. + return y + end + # Extract the from-vec transformations. + fromvec_transforms = map(from_vec_transform, ys) + # Compose the transformations to form a full transformation from + # unconstrained vector representation to constrained space. + transforms = map(∘, fromvec_transforms, link_transforms) + # Convert to vector representation. + yvecs = map(tovec, ys) + + # Determine new ranges. + ranges_new = similar(metadata.ranges) + offset = 0 + for (i, v) in enumerate(yvecs) + r_start, r_end = offset + 1, length(v) + offset + offset = r_end + ranges_new[i] = r_start:r_end + end + + # Now we just create a new metadata with the new `vals` and `ranges`. + return VarNameVector( + metadata.varname_to_index, + metadata.varnames, + ranges_new, + reduce(vcat, yvecs), + transforms, + ) +end + function invlink( ::DynamicTransformation, varinfo::VarInfo, spl::AbstractSampler, model::Model ) - return _invlink(varinfo, spl) + return _invlink(model, varinfo, spl) end function invlink( ::DynamicTransformation, @@ -1324,30 +1404,34 @@ function invlink( return Setfield.@set varinfo.varinfo = invlink(varinfo.varinfo, spl, model) end -function _invlink(varinfo::UntypedVarInfo, spl::AbstractSampler) +function _invlink(model::Model, varinfo::VarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) return VarInfo( - _invlink_metadata!(varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + _invlink_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo)), ) end -function _invlink(varinfo::TypedVarInfo, spl::AbstractSampler) +function _invlink(model::Model, varinfo::TypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) md = _invlink_metadata_namedtuple!( - varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) + model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) ) return VarInfo(md, Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo))) end @generated function _invlink_metadata_namedtuple!( - varinfo::VarInfo, metadata::NamedTuple{names}, vns::NamedTuple, ::Val{space} + model::Model, + varinfo::VarInfo, + metadata::NamedTuple{names}, + vns::NamedTuple, + ::Val{space}, ) where {names,space} vals = Expr(:tuple) for f in names if inspace(f, space) || length(space) == 0 - push!(vals.args, :(_invlink_metadata!(varinfo, metadata.$f, vns.$f))) + push!(vals.args, :(_invlink_metadata!(model, varinfo, metadata.$f, vns.$f))) else push!(vals.args, :(metadata.$f)) end @@ -1355,7 +1439,7 @@ end return :(NamedTuple{$names}($vals)) end -function _invlink_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) +function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, target_vns) vns = metadata.vns # Construct the new transformed values, and keep track of their lengths. @@ -1404,6 +1488,50 @@ function _invlink_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) ) end +function _invlink_metadata!( + model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns +) + # TODO: Make use of `update!` to aovid copying values. + # => Only need to allocate for transformations. + + vns = keys(metadata) + + # Compute the transformed values. + xs = map(vns) do vn + f = inverse(gettransform(metadata, vn)) + y = getval(metadata, vn) + # TODO: Can we remove this `_reconstruct` part? + x, logjac = with_logabsdet_jacobian(f, y) + # Accumulate the log-abs-det jacobian correction. + acclogp!!(varinfo, -logjac) + # Return the transformed value. + return x + end + # Compose the transformations to form a full transformation from + # unconstrained vector representation to constrained space. + transforms = map(from_vec_transform, xs) + # Convert to vector representation. + xvecs = map(tovec, xs) + + # Determine new ranges. + ranges_new = similar(metadata.ranges) + offset = 0 + for (i, v) in enumerate(xvecs) + r_start, r_end = offset + 1, length(v) + offset + offset = r_end + ranges_new[i] = r_start:r_end + end + + # Now we just create a new metadata with the new `vals` and `ranges`. + return VarNameVector( + metadata.varname_to_index, + metadata.varnames, + ranges_new, + reduce(vcat, xvecs), + transforms, + ) +end + """ islinked(vi::VarInfo, spl::Union{Sampler, SampleFromPrior}) From b3e09559ba1aa30b6a40b45a5033b5c94f8be16c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:33:50 +0000 Subject: [PATCH 088/182] added temporary defs of `with_logabsdet_jacobian` and `inverse` for `transpose` and `Bijectors.vec_to_triu` --- src/varinfo.jl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index 1e25ff6e1..baca712d0 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2230,3 +2230,30 @@ function values_from_metadata(md::Metadata) end values_from_metadata(md::VarNameVector) = pairs(md) + +# TODO: Move these. +Bijectors.with_logabsdet_jacobian(::typeof(transpose), x) = (transpose(x), 0) +Bijectors.inverse(::typeof(transpose)) = transpose + +function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) + return (Bijectors.vec_to_triu(x), 0) +end + +# HACK: Overload of `invlink_with_logpdf` so we can fix it for `VarNameVector`. +function invlink_with_logpdf(vi::VarInfo, vn::VarName, dist, y) + return _invlink_with_logpdf(getmetadata(vi, vn), vn, dist, y) +end + +function _invlink_with_logpdf(md::Metadata, vn::VarName, dist, y) + # NOTE: Will this cause type-instabilities or will union-splitting save us? + f = istrans(md, vn) ? invlink_transform(dist) : identity + x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) + return x, logpdf(dist, x) + logjac +end + +function _invlink_with_logpdf(vnv::VarNameVector, vn::VarName, dist, y) + # Here the transformation is stored in `vnv` so we just extract and use this. + f = gettransform(vnv, vn) + x, logjac = with_logabsdet_jacobian(f, y) + return x, logpdf(dist, x) + logjac +end From ff963cef47532d9c666fe694fc9dac1a9a886725 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:34:24 +0000 Subject: [PATCH 089/182] added invlink_with_logpdf overload for `ThreadSafeVarInfo` --- src/threadsafe.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index fb1cc1c0c..9b6525097 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -238,3 +238,7 @@ function Base.merge(varinfo_left::ThreadSafeVarInfo, varinfo_right::ThreadSafeVa varinfo_left.varinfo, varinfo_right.varinfo ) end + +function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) + return invlink_with_logpdf(vi.varinfo, vn, dist, y) +end From 03f2b2bdea5dc277694cb341c2a9d87405232248 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:23:31 +0000 Subject: [PATCH 090/182] added `is_transformed` field to `VarNameVector` --- src/utils.jl | 62 +++++++++++++++++++++++++++++++++++++-- src/varinfo.jl | 19 ++++++++---- src/varnamevector.jl | 69 +++++++++++--------------------------------- 3 files changed, 91 insertions(+), 59 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index b00425c94..a2007196c 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -211,6 +211,59 @@ invlink_transform(dist) = inverse(link_transform(dist)) # Helper functions for vectorize/reconstruct values # ##################################################### +# Useful transformation going from the flattened representation. +struct FromVec{Sz} <: Bijectors.Bijector + sz::Sz +end + +FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) + +# TODO: Should we materialize the `reshape`? +(f::FromVec)(x) = reshape(x, f.sz) +(f::FromVec{Tuple{}})(x) = only(x) +# TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. + +Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) +# We want to use the inverse of `FromVec` so it preserves the size information. +Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:FromVec}, x) = (tovec(x), 0) + +struct ToChol <: Bijectors.Bijector + uplo::Char +end + +Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) +Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) + +from_vec_transform(x::Real) = FromVec(()) +from_vec_transform(x::AbstractVector) = identity +from_vec_transform(x::AbstractArray) = FromVec(size(x)) +function from_vec_transform(C::Cholesky) + return ToChol(C.uplo) ∘ from_vec_transform(C.UL) +end +from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu +function from_vec_transform(L::LinearAlgebra.LowerTriangular) + return transpose ∘ from_vec_transform(transpose(L)) +end + +# FIXME: drop the `rand` below and instead implement on a case-by-case basis. +from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) + +# TODO: Move these. +function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) + return (Bijectors.vec_to_triu(x), 0) +end + +# FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix +# flattened, while using `tovec` below flattenes only the necessary entries. +# => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, +# or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. +tovec(x::Real) = [x] +tovec(x::AbstractArray) = vec(x) +tovec(C::Cholesky) = tovec(C.UL) +tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) +tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) + +# TODO: Remove these. vectorize(d, r) = vectorize(r) vectorize(r) = tovec(r) @@ -340,8 +393,13 @@ end ####################### # Convenience methods # ####################### -collectmaybe(x) = x -collectmaybe(x::Base.AbstractSet) = collect(x) +""" + collect_maybe(x) + +Return `collect(x)` if `x` is an array, otherwise return `x`. +""" +collect_maybe(x) = collect(x) +collect_maybe(x::AbstractArray) = x ####################### # BangBang.jl related # diff --git a/src/varinfo.jl b/src/varinfo.jl index baca712d0..cbe2f6428 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -964,7 +964,6 @@ end istrans(vi::VarInfo, vn::VarName) = istrans(getmetadata(vi, vn), vn) istrans(md::Metadata, vn::VarName) = is_flagged(md, vn, "trans") -istrans(vnv::VarNameVector, vn::VarName) = !(gettransform(vnv, vn) isa FromVec) getlogp(vi::VarInfo) = vi.logp[] @@ -1336,10 +1335,13 @@ end function _link_metadata!( model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns ) + # HACK: We ignore `target_vns` here. vns = keys(metadata) # Need to extract the priors from the model. dists = extract_priors(model, varinfo) + is_transformed = copy(metadata.is_transformed) + # Construct the linking transformations. link_transforms = map(vns) do vn # If `vn` is not part of `target_vns`, the `identity` transformation is used. @@ -1348,6 +1350,7 @@ function _link_metadata!( end # Otherwise, we derive the transformation from the distribution. + is_transformed[getidx(metadata, vn)] = true link_transform(getindex(dists, vn)) end # Compute the transformed values. @@ -1385,6 +1388,7 @@ function _link_metadata!( ranges_new, reduce(vcat, yvecs), transforms, + is_transformed, ) end @@ -1491,19 +1495,23 @@ end function _invlink_metadata!( model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns ) + # HACK: We ignore `target_vns` here. # TODO: Make use of `update!` to aovid copying values. # => Only need to allocate for transformations. vns = keys(metadata) + is_transformed = copy(varinfo.is_transformed) # Compute the transformed values. xs = map(vns) do vn f = inverse(gettransform(metadata, vn)) y = getval(metadata, vn) - # TODO: Can we remove this `_reconstruct` part? + # No need to use `with_reconstruct` as `f` will include this. x, logjac = with_logabsdet_jacobian(f, y) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) + # Mark as no longer transformed. + is_transformed[getidx(metadata, vn)] = false # Return the transformed value. return x end @@ -1529,6 +1537,7 @@ function _invlink_metadata!( ranges_new, reduce(vcat, xvecs), transforms, + is_transformed, ) end @@ -1949,7 +1958,7 @@ end Calls `kernel!(vi, vn, values, keys)` for every `vn` in `vi`. """ function _apply!(kernel!, vi::VarInfoOrThreadSafeVarInfo, values, keys) - keys_strings = map(string, collectmaybe(keys)) + keys_strings = map(string, collect_maybe(keys)) num_indices_seen = 0 for vn in Base.keys(vi) @@ -1971,7 +1980,7 @@ function _apply!(kernel!, vi::VarInfoOrThreadSafeVarInfo, values, keys) end function _apply!(kernel!, vi::TypedVarInfo, values, keys) - return _typed_apply!(kernel!, vi, vi.metadata, values, collectmaybe(keys)) + return _typed_apply!(kernel!, vi, vi.metadata, values, collect_maybe(keys)) end @generated function _typed_apply!( @@ -2007,7 +2016,7 @@ end end function _find_missing_keys(vi::VarInfoOrThreadSafeVarInfo, keys) - string_vns = map(string, collectmaybe(Base.keys(vi))) + string_vns = map(string, collect_maybe(Base.keys(vi))) # If `key` isn't subsumed by any element of `string_vns`, it is not present in `vi`. missing_keys = filter(keys) do key !any(Base.Fix2(subsumes_string, key), string_vns) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index ebf7a8e79..c2a3b489f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -18,12 +18,15 @@ struct VarNameVector{ "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" ranges::Vector{UnitRange{Int}} - "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" + "vector of values of all variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" vals::TVal # AbstractVector{<:Real} "vector of transformations whose inverse takes us back to the original space" transforms::TTrans + "specifies whether a variable is transformed or not " + is_transformed::BitVector + "additional entries which are considered inactive" num_inactive::OrderedDict{Int,Int} @@ -37,17 +40,26 @@ function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) vnv_left.ranges == vnv_right.ranges && vnv_left.vals == vnv_right.vals && vnv_left.transforms == vnv_right.transforms && + vnv_left.is_transformed == vnv_right.is_transformed && vnv_left.num_inactive == vnv_right.num_inactive && vnv_left.metadata == vnv_right.metadata end -function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) +function VarNameVector( + varname_to_index, + varnames, + ranges, + vals, + transforms, + is_transformed=fill!(BitVector(undef, length(varnames)), 0), +) return VarNameVector( varname_to_index, varnames, ranges, vals, transforms, + is_transformed, OrderedDict{Int,Int}(), nothing, ) @@ -57,58 +69,11 @@ function VarNameVector{K,V}() where {K,V} return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) end -# Useful transformation going from the flattened representation. -struct FromVec{Sz} <: Bijectors.Bijector - sz::Sz -end - -FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) - -# TODO: Should we materialize the `reshape`? -(f::FromVec)(x) = reshape(x, f.sz) -(f::FromVec{Tuple{}})(x) = only(x) -# TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. - -Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) - -struct ToChol <: Bijectors.Bijector - uplo::Char -end - -Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) -Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) - -from_vec_transform(x::Real) = FromVec(()) -from_vec_transform(x::AbstractVector) = identity -from_vec_transform(x::AbstractArray) = FromVec(size(x)) -function from_vec_transform(C::Cholesky) - return ToChol(C.uplo) ∘ from_vec_transform(C.UL) -end -from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu -function from_vec_transform(L::LinearAlgebra.LowerTriangular) - return transpose ∘ from_vec_transform(transpose(L)) +istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] +function settrans!(vnv::VarNameVector, vn::VarName, val) + return vnv.is_transformed[vnv.varname_to_index[vn]] = val end -# FIXME: remove the `rand` below. -from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) - -# We want to use the inverse of `FromVec` so it preserves the size information. -Bijectors.transform(::Bijectors.Inverse{<:FromVec}, x) = tovec(x) - -# FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix -# flattened, while using `tovec` below flattenes only the necessary entries. -# => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, -# or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. -tovec(x::Real) = [x] -tovec(x::AbstractArray) = vec(x) -tovec(C::Cholesky) = tovec(C.UL) -tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) -tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) - -# More convenient constructors. -collect_maybe(x) = collect(x) -collect_maybe(x::AbstractArray) = x - VarNameVector() = VarNameVector{VarName,Real}() VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) From 949b33a3465ad02fcb2729bfefa9fadb79f9ec1c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:23:58 +0000 Subject: [PATCH 091/182] removed unnecessary defintions of `with_logabsdet_jacobian` and `inverse` for `transpose` --- src/varinfo.jl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index cbe2f6428..acdffdc2f 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2240,14 +2240,6 @@ end values_from_metadata(md::VarNameVector) = pairs(md) -# TODO: Move these. -Bijectors.with_logabsdet_jacobian(::typeof(transpose), x) = (transpose(x), 0) -Bijectors.inverse(::typeof(transpose)) = transpose - -function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) - return (Bijectors.vec_to_triu(x), 0) -end - # HACK: Overload of `invlink_with_logpdf` so we can fix it for `VarNameVector`. function invlink_with_logpdf(vi::VarInfo, vn::VarName, dist, y) return _invlink_with_logpdf(getmetadata(vi, vn), vn, dist, y) From cc5ecc40c6300796ff7b284162e28cdb36dfd540 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:34:19 +0000 Subject: [PATCH 092/182] fixed issue where we were storing the wrong transformations in `VarNameVector` --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index acdffdc2f..c8f9d274c 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1368,7 +1368,7 @@ function _link_metadata!( fromvec_transforms = map(from_vec_transform, ys) # Compose the transformations to form a full transformation from # unconstrained vector representation to constrained space. - transforms = map(∘, fromvec_transforms, link_transforms) + transforms = map(∘, map(inverse, link_transforms), fromvec_transforms) # Convert to vector representation. yvecs = map(tovec, ys) From 1aae1b413aaf19120096889e87746e5609e8def6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:56:28 +0000 Subject: [PATCH 093/182] make sure `extract_priors` doesn't mutate the `varinfo` --- src/extract_priors.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index a22e65fad..1487b9049 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -119,6 +119,6 @@ end function extract_priors(::Random.AbstractRNG, model::Model, varinfo::AbstractVarInfo) context = PriorExtractorContext(DefaultContext()) - evaluate!!(model, varinfo, context) + evaluate!!(model, deepcopy(varinfo), context) return context.priors end From 8e0853d32d44b6893b9f2edd53990b688631b4c7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:14:58 +0000 Subject: [PATCH 094/182] updated `similar` for `VarNameVector` and fixed `invlink` for `VarNameVector` --- src/varinfo.jl | 4 ++-- src/varnamevector.jl | 1 + test/varinfo.jl | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index c8f9d274c..ab85d57f1 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1500,11 +1500,11 @@ function _invlink_metadata!( # => Only need to allocate for transformations. vns = keys(metadata) - is_transformed = copy(varinfo.is_transformed) + is_transformed = copy(metadata.is_transformed) # Compute the transformed values. xs = map(vns) do vn - f = inverse(gettransform(metadata, vn)) + f = gettransform(metadata, vn) y = getval(metadata, vn) # No need to use `with_reconstruct` as `f` will include this. x, logjac = with_logabsdet_jacobian(f, y) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index c2a3b489f..630f7f870 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -413,6 +413,7 @@ function Base.similar(vnv::VarNameVector) similar(vnv.ranges, 0), similar(vnv.vals, 0), similar(vnv.transforms, 0), + BitVector(), similar(vnv.num_inactive), similar_metadata(vnv.metadata), ) diff --git a/test/varinfo.jl b/test/varinfo.jl index 71e341767..c51c5fb82 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -413,6 +413,9 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) else DynamicPPL.link(varinfo, model) end + for vn in keys(varinfo) + @test DynamicPPL.istrans(varinfo_linked, vn) + end @test length(varinfo[:]) > length(varinfo_linked[:]) varinfo_linked_unflattened = DynamicPPL.unflatten( varinfo_linked, varinfo_linked[:] From 229b168aca8736f6915ceb8100b8d64ce8779173 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:22:24 +0000 Subject: [PATCH 095/182] added handling of `is_transformed` in `merge` for `VarNameVector` --- src/varnamevector.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 630f7f870..92d413196 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -351,6 +351,7 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) ranges = UnitRange{Int}[] vals = T[] transforms = F[] + is_transformed = BitVector(undef, length(vns_both)) # Range offset. offset = 0 @@ -364,6 +365,7 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) n = length(val) r = (offset + 1):(offset + n) f = gettransform(left_vnv, vn) + is_transformed[idx] = istrans(left_vnv, vn) else # `vn` is either in both or just `right`. varnames_to_index[vn] = idx @@ -371,6 +373,7 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) n = length(val) r = (offset + 1):(offset + n) f = gettransform(right_vnv, vn) + is_transformed[idx] = istrans(right_vnv, vn) end # Update. append!(vals, val) From c581dcf323f448c1e825a18f26adb1ccb07fd13f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:22:42 +0000 Subject: [PATCH 096/182] removed unnecesasry `deepcopy` from outer `link` --- src/abstract_varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index fe2c0b3e5..cfa2d7914 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -563,7 +563,7 @@ If `t` is not provided, `default_transformation(model, vi)` will be used. See also: [`default_transformation`](@ref), [`invlink`](@ref). """ -link(vi::AbstractVarInfo, model::Model) = link(deepcopy(vi), SampleFromPrior(), model) +link(vi::AbstractVarInfo, model::Model) = link(vi, SampleFromPrior(), model) function link(t::AbstractTransformation, vi::AbstractVarInfo, model::Model) return link(t, deepcopy(vi), SampleFromPrior(), model) end From b4d3f5541d244cb67656b5e4fe5f4adf117234b7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:44:47 +0000 Subject: [PATCH 097/182] updated `push!` to also `push!` on `is_transformed` --- src/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 92d413196..1172ae01a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -456,6 +456,7 @@ function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_tra push!(vnv.ranges, r_new) append!(vnv.vals, val_vec) push!(vnv.transforms, transform) + push!(vnv.is_transformed, false) return nothing end From ed1d00698b21807933fea1b4432ce0b70af3c130 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:47:07 +0000 Subject: [PATCH 098/182] skip tests for mutating linking when using VarNameVector --- test/test_util.jl | 9 +++++++++ test/varinfo.jl | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/test/test_util.jl b/test/test_util.jl index 22bfa46cc..707d21733 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -104,3 +104,12 @@ function modify_value_representation(nt::NamedTuple) end return modified_nt end + +has_varnamevector(vi) = false +function has_varnamevector(vi::VarInfo) + return vi.metadata isa VarNameVector || + (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) +end +function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) + return has_varnamevector(vi.varinfo) +end diff --git a/test/varinfo.jl b/test/varinfo.jl index c51c5fb82..c8aadef35 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -405,6 +405,12 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end + if has_varnamevector(varinfo) && mutating + # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. + @test_broken false + continue + end + # Evaluate the model once to update the logp of the varinfo. varinfo = last(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())) From f13220995654aed074dfe751685685429b5fc009 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 22:38:12 +0000 Subject: [PATCH 099/182] use same projection for `Cholesky` in `VarNameVector` as in `VarInfo` --- src/utils.jl | 17 ++--------------- src/varinfo.jl | 22 +++++++++++++++++++++- test/test_util.jl | 9 --------- test/varinfo.jl | 2 +- test/varnamevector.jl | 2 +- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index a2007196c..f1ad6bb0e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -237,31 +237,18 @@ Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = from_vec_transform(x::Real) = FromVec(()) from_vec_transform(x::AbstractVector) = identity from_vec_transform(x::AbstractArray) = FromVec(size(x)) -function from_vec_transform(C::Cholesky) - return ToChol(C.uplo) ∘ from_vec_transform(C.UL) -end -from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu -function from_vec_transform(L::LinearAlgebra.LowerTriangular) - return transpose ∘ from_vec_transform(transpose(L)) -end +from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ FromVec(size(C.UL)) # FIXME: drop the `rand` below and instead implement on a case-by-case basis. from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) -# TODO: Move these. -function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) - return (Bijectors.vec_to_triu(x), 0) -end - # FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix # flattened, while using `tovec` below flattenes only the necessary entries. # => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, # or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. tovec(x::Real) = [x] tovec(x::AbstractArray) = vec(x) -tovec(C::Cholesky) = tovec(C.UL) -tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) -tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) +tovec(C::Cholesky) = tovec(Matrix(C.UL)) # TODO: Remove these. vectorize(d, r) = vectorize(r) diff --git a/src/varinfo.jl b/src/varinfo.jl index ab85d57f1..b225eae28 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -161,6 +161,20 @@ function VectorVarInfo(vi::TypedVarInfo) return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end +""" + has_varnamevector(varinfo::VarInfo) + +Returns `true` if `varinfo` uses `VarNameVector` as metadata. +""" +has_varnamevector(vi) = false +function has_varnamevector(vi::VarInfo) + return vi.metadata isa VarNameVector || + (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) +end +function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) + return has_varnamevector(vi.varinfo) +end + function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -1018,6 +1032,8 @@ end # X -> R for all variables associated with given sampler function link!!(t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) + # If we're working with a `VarNameVector`, we always use immutable. + has_varnamevector(vi) && return link(t, vi, spl, model) # Call `_link!` instead of `link!` to avoid deprecation warning. _link!(vi, spl) return vi @@ -1103,7 +1119,11 @@ end end # R -> X for all variables associated with given sampler -function invlink!!(::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) +function invlink!!( + t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model +) + # If we're working with a `VarNameVector`, we always use immutable. + has_varnamevector(vi) && return invlink(t, vi, spl, model) # Call `_invlink!` instead of `invlink!` to avoid deprecation warning. _invlink!(vi, spl) return vi diff --git a/test/test_util.jl b/test/test_util.jl index 707d21733..22bfa46cc 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -104,12 +104,3 @@ function modify_value_representation(nt::NamedTuple) end return modified_nt end - -has_varnamevector(vi) = false -function has_varnamevector(vi::VarInfo) - return vi.metadata isa VarNameVector || - (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) -end -function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) - return has_varnamevector(vi.varinfo) -end diff --git a/test/varinfo.jl b/test/varinfo.jl index c8aadef35..ddd1b7ea8 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -405,7 +405,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end - if has_varnamevector(varinfo) && mutating + if DynamicPPL.has_varnamevector(varinfo) && mutating # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. @test_broken false continue diff --git a/test/varnamevector.jl b/test/varnamevector.jl index ce46e86fc..210812932 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -431,7 +431,7 @@ end model, value_true, varnames; include_threadsafe=false ) # Filter out those which are not based on `VarNameVector`. - varinfos = filter(has_varnamevector, varinfos) + varinfos = filter(DynamicPPL.has_varnamevector, varinfos) # Get the true log joint. logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) From 49454de728025a9753138ba79d32f09304bacab0 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:05:29 +0000 Subject: [PATCH 100/182] fixed `settrans!!` for `VarInfo` with `VarNameVector` --- src/varinfo.jl | 13 ++++++++++--- src/varnamevector.jl | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index b225eae28..ccb8b97ad 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -681,14 +681,18 @@ Return the set of sampler selectors associated with `vn` in `vi`. getgid(vi::VarInfo, vn::VarName) = getmetadata(vi, vn).gids[getidx(vi, vn)] function settrans!!(vi::VarInfo, trans::Bool, vn::VarName) + return settrans!!(getmetadata(vi, vn), trans, vn) +end +function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) if trans - set_flag!(vi, vn, "trans") + set_flag!(metadata, vn, "trans") else - unset_flag!(vi, vn, "trans") + unset_flag!(metadata, vn, "trans") end return vi end +settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) = settrans!(vnv, trans, vn) function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) @@ -851,7 +855,10 @@ end Set `vn`'s value for `flag` to `true` in `vi`. """ function set_flag!(vi::VarInfo, vn::VarName, flag::String) - return getmetadata(vi, vn).flags[flag][getidx(vi, vn)] = true + return set_flag!(getmetadata(vi, vn), vn, flag) +end +function set_flag!(md::Metadata, vn::VarName, flag::String) + return md.flags[flag][getidx(vi, vn)] = true end #### diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 1172ae01a..7bbb8bda8 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -70,7 +70,7 @@ function VarNameVector{K,V}() where {K,V} end istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] -function settrans!(vnv::VarNameVector, vn::VarName, val) +function settrans!(vnv::VarNameVector, val::Bool, vn::VarName) return vnv.is_transformed[vnv.varname_to_index[vn]] = val end From 01ff2dd9403e485d9b78589868ff67cc27b46f38 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:06:01 +0000 Subject: [PATCH 101/182] fixed bug in `set_flag!` --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ccb8b97ad..ca5be142d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -858,7 +858,7 @@ function set_flag!(vi::VarInfo, vn::VarName, flag::String) return set_flag!(getmetadata(vi, vn), vn, flag) end function set_flag!(md::Metadata, vn::VarName, flag::String) - return md.flags[flag][getidx(vi, vn)] = true + return md.flags[flag][getidx(md, vn)] = true end #### From 20adedfd71a88af1abc280b337e6631f7b68d6f4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:09:31 +0000 Subject: [PATCH 102/182] fixed another typo --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ca5be142d..ffb19b79e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -690,7 +690,7 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) unset_flag!(metadata, vn, "trans") end - return vi + return metadata end settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) = settrans!(vnv, trans, vn) From 8f9566a7f6690ad3aefc8d90b8434da292853c6f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:33:08 +0000 Subject: [PATCH 103/182] fixed return values of `settrans!!` --- src/varinfo.jl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ffb19b79e..f599c3a2c 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -681,7 +681,8 @@ Return the set of sampler selectors associated with `vn` in `vi`. getgid(vi::VarInfo, vn::VarName) = getmetadata(vi, vn).gids[getidx(vi, vn)] function settrans!!(vi::VarInfo, trans::Bool, vn::VarName) - return settrans!!(getmetadata(vi, vn), trans, vn) + settrans!!(getmetadata(vi, vn), trans, vn) + return vi end function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) if trans @@ -692,7 +693,10 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end -settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) = settrans!(vnv, trans, vn) +function settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) + settrans!(vnv, trans, vn) + return vnv +end function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) @@ -855,7 +859,8 @@ end Set `vn`'s value for `flag` to `true` in `vi`. """ function set_flag!(vi::VarInfo, vn::VarName, flag::String) - return set_flag!(getmetadata(vi, vn), vn, flag) + set_flag!(getmetadata(vi, vn), vn, flag) + return vi end function set_flag!(md::Metadata, vn::VarName, flag::String) return md.flags[flag][getidx(md, vn)] = true From 553204663199deb91304b115a5a68e3e595798b6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 20 Jan 2024 02:23:15 +0000 Subject: [PATCH 104/182] updated static transformation tests --- src/varinfo.jl | 9 +++++++++ test/simple_varinfo.jl | 16 +++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index f599c3a2c..3d97b748d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1668,6 +1668,15 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) end getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) +function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) + # FIXME: This is too hacky. + # We know this will only be hit if we're working with `VarNameVector`, + # so we can just use the `getindex_raw` for `VarNameVector`. + # NOTE: This won't result in the same behavior as `getindex_raw` + # for the other `VarInfo`s since we don't have access to the `dist` + # and so can't call `reconstruct`. + return getindex_raw(getmetadata(vi, vn), vn) +end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) return reconstruct(dist, getval(vi, vn)) end diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 869fb82b3..ca1e05c80 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -223,6 +223,7 @@ @testset "Static transformation" begin model = DynamicPPL.TestUtils.demo_static_transformation() + priors = extract_priors(model) varinfos = DynamicPPL.TestUtils.setup_varinfos( model, DynamicPPL.TestUtils.rand_prior_true(model), [@varname(s), @varname(m)] @@ -251,8 +252,11 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s)) ≠ retval.s # `s` is unconstrained in original - @test DynamicPPL.getindex_raw(vi_linked_result, @varname(s)) == retval.s # `s` is constrained in result + @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≠ + retval.s # `s` is unconstrained in original + @test DynamicPPL.getindex_raw( + vi_linked_result, @varname(s), priors[@varname(s)] + ) == retval.s # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -263,9 +267,11 @@ model, retval.s, retval.m ) - # Realizations in `vi_linked` should all be equal to the unconstrained realization. - @test DynamicPPL.getindex_raw(vi_linked, @varname(s)) ≈ retval_unconstrained.s - @test DynamicPPL.getindex_raw(vi_linked, @varname(m)) ≈ retval_unconstrained.m + + @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≈ + retval_unconstrained.s + @test DynamicPPL.getindex_raw(vi_linked, @varname(m), priors[@varname(m)]) ≈ + retval_unconstrained.m # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) From 3c5d2acb1c9cb77e4b47faae8043ad674dd25f8e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 20 Jan 2024 02:25:48 +0000 Subject: [PATCH 105/182] Update test/simple_varinfo.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/simple_varinfo.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index ca1e05c80..223d0dd49 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -267,7 +267,6 @@ model, retval.s, retval.m ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≈ retval_unconstrained.s @test DynamicPPL.getindex_raw(vi_linked, @varname(m), priors[@varname(m)]) ≈ From a9be219c08212ab58e90ddfefdf552de5494ae4c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:15:41 +0000 Subject: [PATCH 106/182] removed unnecessary impl of `extract_priors` --- src/extract_priors.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index 1487b9049..da9b176a0 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -116,9 +116,3 @@ function extract_priors(rng::Random.AbstractRNG, model::Model) evaluate!!(model, VarInfo(), context) return context.priors end - -function extract_priors(::Random.AbstractRNG, model::Model, varinfo::AbstractVarInfo) - context = PriorExtractorContext(DefaultContext()) - evaluate!!(model, deepcopy(varinfo), context) - return context.priors -end From 53c8d3345ceec0c293dc077526da9a6f8328f64b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:19:36 +0000 Subject: [PATCH 107/182] make `short_varinfo_name` of `TypedVarInfo` a bit more informative --- test/test_util.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_util.jl b/test/test_util.jl index 22bfa46cc..563e13df1 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -84,7 +84,10 @@ Return string representing a short description of `vi`. """ short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" -short_varinfo_name(::TypedVarInfo) = "TypedVarInfo" +function short_varinfo_name(vi::TypedVarInfo) + DynamicPPL.has_varnamevector(vi) && return "TypedVarInfo with VarNameVector" + return "TypedVarInfo" +end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" short_varinfo_name(::DynamicPPL.VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" From 61d85ad09ea4c80cdf3d0167e8c3f3fd18d9e596 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:22:59 +0000 Subject: [PATCH 108/182] moved impl of `has_varnamevector` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 2 ++ src/varinfo.jl | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 9b6525097..91b9704c7 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -55,6 +55,8 @@ function setlogp!!(vi::ThreadSafeVarInfoWithRef, logp) return ThreadSafeVarInfo(setlogp!!(vi.varinfo, logp), vi.logps) end +has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) = has_varnamevector(vi.varinfo) + function BangBang.push!!( vi::ThreadSafeVarInfo, vn::VarName, r, dist::Distribution, gidset::Set{Selector} ) diff --git a/src/varinfo.jl b/src/varinfo.jl index 9fa4cdc14..e6c2875d2 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -171,9 +171,6 @@ function has_varnamevector(vi::VarInfo) return vi.metadata isa VarNameVector || (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) end -function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) - return has_varnamevector(vi.varinfo) -end function untyped_varinfo( rng::Random.AbstractRNG, From 9ace554691294f6b5e294dfee116ad37527820b0 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:29:26 +0000 Subject: [PATCH 109/182] added back `extract_priors` impl as we do need it --- src/extract_priors.jl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index da9b176a0..ff77e1a4b 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -116,3 +116,18 @@ function extract_priors(rng::Random.AbstractRNG, model::Model) evaluate!!(model, VarInfo(), context) return context.priors end + +""" + + extract_priors(model::Model, varinfo::AbstractVarInfo) + +Extract the priors from a model. + +This is done by evaluating the model at the values present in `varinfo` +and recording the distributions that are present at each tilde statement. +""" +function extract_priors(model::Model, varinfo::AbstractVarInfo) + context = PriorExtractorContext(DefaultContext()) + evaluate!!(model, deepcopy(varinfo), context) + return context.priors +end From 67c9821f11b88aa53463536c7e49aa6d1424b58f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:34:07 +0000 Subject: [PATCH 110/182] forgot to include tests for `VarNameVector` in `runtests.jl` --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index d15b17b42..ef7672db8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -37,6 +37,7 @@ include("test_util.jl") @testset "interface" begin include("utils.jl") include("compiler.jl") + include("varnamevector.jl") include("varinfo.jl") include("simple_varinfo.jl") include("model.jl") From 32a2d3104c9d93be2a6695347167083f32c27f7e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 17:53:48 +0000 Subject: [PATCH 111/182] fix for `relax_container_types` in `test/varnamevector.jl` --- test/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 210812932..580815fc6 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -100,6 +100,7 @@ function relax_container_types(vnv::VarNameVector, vns, vals) vnv.ranges, vals_new, transforms_new, + vnv.is_transformed, vnv.num_inactive, vnv.metadata, ) From b3bb42d706b6c33df69af8bbff0400c3b8563fbf Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 26 Jan 2024 14:57:48 +0000 Subject: [PATCH 112/182] fixed `need_transforms_relaxation` --- test/varnamevector.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 580815fc6..2e8e40f46 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -36,15 +36,19 @@ function need_values_relaxation(vnv::VarNameVector, vns, vals) end function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) - if isconcretetype(eltype(vnv.transforms)) + return if isconcretetype(eltype(vnv.transforms)) # If the container is concrete, we need to make sure that the sizes match. # => If the sizes don't match, we need to relax the container type. - return any(keys(vnv)) do vn_present + any(keys(vnv)) do vn_present size(vnv[vn_present]) != size(val) end + elseif eltype(vnv.transforms) !== Any + # If it's not concrete AND it's not `Any`, then we should just make it `Any`. + true + else + # Otherwise, it's `Any`, so we don't need to relax the container type. + false end - - return false end function need_transforms_relaxation(vnv::VarNameVector, vns, vals) return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) From 25ff2b1a3242ba5b2da076600b4bdc7973aa32a7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 10:00:07 +0000 Subject: [PATCH 113/182] updated some tests to not refer directly to `FromVec` --- test/varnamevector.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 2e8e40f46..6b7acfbeb 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -161,8 +161,8 @@ end # We'll need the transformations later. # TODO: Should we test other transformations than just `FromVec`? - from_vec_left = DynamicPPL.FromVec(val_left) - from_vec_right = DynamicPPL.FromVec(val_right) + from_vec_left = DynamicPPL.from_vec_transform(val_left) + from_vec_right = DynamicPPL.from_vec_transform(val_right) to_vec_left = inverse(from_vec_left) to_vec_right = inverse(from_vec_right) From 004f038c63d0a251871607cb6306d25ef2570bcd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:19:33 +0000 Subject: [PATCH 114/182] introduce `from_internal_transform` and its siblings --- src/abstract_varinfo.jl | 106 ++++++++++++++++++++++++++++++---------- src/simple_varinfo.jl | 7 +++ src/utils.jl | 57 ++++++++++++++++++--- src/varinfo.jl | 26 +++++----- 4 files changed, 149 insertions(+), 47 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index c37bde795..e08af97e2 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -755,13 +755,13 @@ end # TODO: Clean up all this linking stuff once and for all! """ - with_logabsdet_jacobian_and_reconstruct([f, ]dist, x) + with_logabsdet_jacobian_and_reconstruct(f, dist, x) Like `Bijectors.with_logabsdet_jacobian(f, x)`, but also ensures the resulting value is reconstructed to the correct type and shape according to `dist`. """ function with_logabsdet_jacobian_and_reconstruct(f, dist, x) - x_recon = reconstruct(f, dist, x) + x_recon = reconstruct(dist, x) return with_logabsdet_jacobian(f, x_recon) end @@ -769,7 +769,6 @@ end # just use `first ∘ with_logabsdet_jacobian` to reduce the maintenance burden. # NOTE: `reconstruct` is no-op if `val` is already of correct shape. """ - reconstruct_and_link(dist, val) reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) Return linked `val` but reconstruct before linking, if necessary. @@ -780,26 +779,21 @@ by `dist`. See also: [`invlink_and_reconstruct`](@ref), [`reconstruct`](@ref). """ -reconstruct_and_link(f, dist, val) = f(reconstruct(f, dist, val)) -reconstruct_and_link(dist, val) = reconstruct_and_link(link_transform(dist), dist, val) -function reconstruct_and_link(::AbstractVarInfo, ::VarName, dist, val) - return reconstruct_and_link(dist, val) +function reconstruct_and_link(varinfo::AbstractVarInfo, vn::VarName, dist, val) + f = to_linked_internal_transform(varinfo, vn, dist) + return f(val) end """ - invlink_and_reconstruct(dist, val) invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) Return invlinked and reconstructed `val`. See also: [`reconstruct_and_link`](@ref), [`reconstruct`](@ref). """ -invlink_and_reconstruct(f, dist, val) = f(reconstruct(f, dist, val)) -function invlink_and_reconstruct(dist, val) - return invlink_and_reconstruct(invlink_transform(dist), dist, val) -end -function invlink_and_reconstruct(::AbstractVarInfo, ::VarName, dist, val) - return invlink_and_reconstruct(dist, val) +function invlink_and_reconstruct(varinfo::AbstractVarInfo, vn::VarName, dist, val) + f = from_linked_internal_transform(varinfo, vn, dist) + return f(val) end """ @@ -808,11 +802,8 @@ end Return reconstructed `val`, possibly linked if `istrans(vi, vn)` is `true`. """ function maybe_reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) - return if istrans(vi, vn) - reconstruct_and_link(vi, vn, dist, val) - else - reconstruct(dist, val) - end + f = to_maybe_linked_internal_transform(vi, vn, dist) + return f(val) end """ @@ -821,28 +812,29 @@ end Return reconstructed `val`, possibly invlinked if `istrans(vi, vn)` is `true`. """ function maybe_invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) - return if istrans(vi, vn) - invlink_and_reconstruct(vi, vn, dist, val) - else - reconstruct(dist, val) - end + f = from_maybe_linked_internal_transform(vi, vn, dist) + return f(val) end """ - invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist[, x]) + invlink_with_logpdf(varinfo::AbstractVarInfo, vn::VarName, dist[, x]) Invlink `x` and compute the logpdf under `dist` including correction from the invlink-transformation. If `x` is not provided, `getval(vi, vn)` will be used. + +!!! warning + The input value `x` should be according to the internal representation of + `varinfo`, e.g. the value returned by `getval(vi, vn)`. """ function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist) return invlink_with_logpdf(vi, vn, dist, getval(vi, vn)) end function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist, y) # NOTE: Will this cause type-instabilities or will union-splitting save us? - f = istrans(vi, vn) ? invlink_transform(dist) : identity - x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) + f = from_maybe_linked_internal_transform(vi, vn, dist) + x, logjac = with_logabsdet_jacobian(f, y) return x, logpdf(dist, x) + logjac end @@ -850,3 +842,63 @@ end # TODO: Remove when possible. increment_num_produce!(::AbstractVarInfo) = nothing setgid!(vi::AbstractVarInfo, gid::Selector, vn::VarName) = nothing + +""" + from_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the internal representation of `vn` with `dist` +in `varinfo` to a representation compatible with `dist`. +""" +function from_internal_transform end + +""" + from_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the linked internal representation of `vn` with `dist` +in `varinfo` to a representation compatible with `dist`. +""" +function from_linked_internal_transform end + +""" + from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the possibly linked internal representation of `vn` with `dist`n +in `varinfo` to a representation compatible with `dist`. +""" +function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return if istrans(varinfo, vn) + from_linked_internal_transform(varinfo, vn, dist) + else + from_internal_transform(varinfo, vn, dist) + end +end + +""" + to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a representation compatible with `dist` to the +internal representation of `vn` with `dist` in `varinfo`. +""" +function to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return inverse(from_internal_transform(varinfo, vn, dist)) +end + +""" + to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a representation compatible with `dist` to the +linked internal representation of `vn` with `dist` in `varinfo`. +""" +function to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return inverse(from_linked_internal_transform(varinfo, vn, dist)) +end + +""" + to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a representation compatible with `dist` to a +possibly linked internal representation of `vn` with `dist` in `varinfo`. +""" +function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return inverse(from_maybe_linked_internal_transform(varinfo, vn, dist)) +end diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 93c211483..5e6b99442 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -298,6 +298,7 @@ function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribu vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end + # TODO: Fix this `reconstruct`. return reconstruct(dist, vals_linked, length(vns)) end @@ -689,6 +690,12 @@ function invlink!!( return settrans!!(vi_new, NoTransformation()) end +# With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. +from_internal_transform(::SimpleVarInfo, ::VarName, dist) = identity +function from_linked_internal_transform(vi::SimpleVarInfo, vn::VarName, dist) + return invlink_transform(dist) +end + # Threadsafe stuff. # For `SimpleVarInfo` we don't really need `Ref` so let's not use it. function ThreadSafeVarInfo(vi::SimpleVarInfo) diff --git a/src/utils.jl b/src/utils.jl index f1ad6bb0e..1b5a958f8 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -234,13 +234,58 @@ end Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) -from_vec_transform(x::Real) = FromVec(()) -from_vec_transform(x::AbstractVector) = identity -from_vec_transform(x::AbstractArray) = FromVec(size(x)) +""" + from_vec_transform(x) + +Return the transformation from the vector representation of `x` to original representation. +""" +from_vec_transform(x::Union{Real,AbstractArray}) = from_vec_transform_for_size(size(x)) from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ FromVec(size(C.UL)) -# FIXME: drop the `rand` below and instead implement on a case-by-case basis. -from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) +from_vec_transform_for_size(sz::Tuple) = FromVec(sz) +from_vec_transform_for_size(::Tuple{()}) = FromVec(()) +from_vec_transform_for_size(::Tuple{<:Any}) = identity + +""" + from_vec_transform(dist::Distribution) + +Return the transformation from the vector representation of a realization from +distribution `dist` to the original representation compatible with `dist`. +""" +from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) +from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ FromVec(size(dist)) + +""" + from_linked_vec_transform(dist) + +Return the transformation from the unconstrained vector to the constrained +realization of distribution `dist`. +""" +function from_linked_vec_transform(dist::Distribution) + f_vec = from_vec_transform(dist) + f_invlink = invlink_transform(dist) + return f_invlink ∘ f_vec +end + +function from_linked_vec_transform(dist::LKJCholesky) + return inverse(Bijectors.VecCholeskyBijector(dist.uplo)) +end + +""" + to_vec_transform(x) + +Return the transformation from the original representation of `x` to the vector +representation. +""" +to_vec_transform(x) = inverse(from_vec_transform(x)) + +""" + to_linked_vec_transform(dist) + +Return the transformation from the constrained realization of distribution `dist` +to the unconstrained vector. +""" +to_linked_vec_transform(x) = inverse(from_linked_vec_transform(x)) # FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix # flattened, while using `tovec` below flattenes only the necessary entries. @@ -279,7 +324,7 @@ reconstruct(::Inverse{Bijectors.VecCorrBijector}, ::LKJ, val::AbstractVector) = function reconstruct(dist::LKJCholesky, val::AbstractVector{<:Real}) f = from_vec_transform(dist) - return reconstruct(dist, f(val)) + return f(val) end function reconstruct(dist::LKJCholesky, val::AbstractMatrix{<:Real}) return Cholesky(val, dist.uplo, 0) diff --git a/src/varinfo.jl b/src/varinfo.jl index e6c2875d2..667911525 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2278,21 +2278,19 @@ end values_from_metadata(md::VarNameVector) = pairs(md) -# HACK: Overload of `invlink_with_logpdf` so we can fix it for `VarNameVector`. -function invlink_with_logpdf(vi::VarInfo, vn::VarName, dist, y) - return _invlink_with_logpdf(getmetadata(vi, vn), vn, dist, y) +# Transforming from internal representation to distribution representation. +function from_internal_transform(vi::VarInfo, vn::VarName, dist) + return from_internal_transform(getmetadata(vi, vn), vn, dist) end +from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) +from_internal_transform(::VarNameVector, ::VarName, dist) = from_vec_transform(dist) -function _invlink_with_logpdf(md::Metadata, vn::VarName, dist, y) - # NOTE: Will this cause type-instabilities or will union-splitting save us? - f = istrans(md, vn) ? invlink_transform(dist) : identity - x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) - return x, logpdf(dist, x) + logjac +function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) + return from_linked_internal_transform(getmetadata(vi, vn), vn, dist) end - -function _invlink_with_logpdf(vnv::VarNameVector, vn::VarName, dist, y) - # Here the transformation is stored in `vnv` so we just extract and use this. - f = gettransform(vnv, vn) - x, logjac = with_logabsdet_jacobian(f, y) - return x, logpdf(dist, x) + logjac +function from_linked_internal_transform(::Metadata, ::VarName, dist) + return from_linked_vec_transform(dist) +end +function from_linked_internal_transform(::VarNameVector, ::VarName, dist) + return from_linked_vec_transform(dist) end From 38c89bd268e7d008ccf38c27d2461ed0c924b721 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:28:03 +0000 Subject: [PATCH 115/182] remove `with_logabsdet_jacobian_and_reconstruct` in favour of `with_logabsdet_jacobian` with `from_linked_internal_transform`, etc. --- src/abstract_varinfo.jl | 12 ------------ src/varinfo.jl | 27 +++++++++++++++------------ 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index e08af97e2..604177cec 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -753,18 +753,6 @@ function unflatten(sampler::AbstractSampler, varinfo::AbstractVarInfo, ::Abstrac return unflatten(varinfo, sampler, θ) end -# TODO: Clean up all this linking stuff once and for all! -""" - with_logabsdet_jacobian_and_reconstruct(f, dist, x) - -Like `Bijectors.with_logabsdet_jacobian(f, x)`, but also ensures the resulting -value is reconstructed to the correct type and shape according to `dist`. -""" -function with_logabsdet_jacobian_and_reconstruct(f, dist, x) - x_recon = reconstruct(dist, x) - return with_logabsdet_jacobian(f, x_recon) -end - # TODO: Once `(inv)link` isn't used heavily in `getindex(vi, vn)`, we can # just use `first ∘ with_logabsdet_jacobian` to reduce the maintenance burden. # NOTE: `reconstruct` is no-op if `val` is already of correct shape. diff --git a/src/varinfo.jl b/src/varinfo.jl index 667911525..c356ed270 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1086,7 +1086,7 @@ function _link!(vi::UntypedVarInfo, spl::AbstractSampler) if ~istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, link_transform(dist)) + _inner_transform!(vi, vn, dist, to_linked_internal_transform(vi, vn, dist)) settrans!!(vi, true, vn) end else @@ -1114,7 +1114,9 @@ end # Iterate over all `f_vns` and transform for vn in f_vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, link_transform(dist)) + _inner_transform!( + vi, vn, dist, to_linked_internal_transform(vi, vn, dist) + ) settrans!!(vi, true, vn) end else @@ -1185,7 +1187,7 @@ function _invlink!(vi::UntypedVarInfo, spl::AbstractSampler) if istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, invlink_transform(dist)) + _inner_transform!(vi, vn, dist, from_linked_internal_transform(vi, vn, dist)) settrans!!(vi, false, vn) end else @@ -1213,7 +1215,9 @@ end # Iterate over all `f_vns` and transform for vn in f_vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, invlink_transform(dist)) + _inner_transform!( + vi, vn, dist, from_linked_internal_transform(vi, vn, dist) + ) settrans!!(vi, false, vn) end else @@ -1232,8 +1236,7 @@ end function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, dist, f) # TODO: Use inplace versions to avoid allocations - y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, getval(vi, vn)) - yvec = vectorize(dist, y) + yvec, logjac = with_logabsdet_jacobian(f, getval(vi, vn)) # Determine the new range. start = first(getrange(vi, vn)) # NOTE: `length(yvec)` should never be longer than `getrange(vi, vn)`. @@ -1327,8 +1330,8 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar # Transform to constrained space. x = getval(metadata, vn) dist = getdist(metadata, vn) - f = link_transform(dist) - y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) + f = to_linked_internal_transform(varinfo, vn, dist) + y, logjac = with_logabsdet_jacobian(f, x) # Vectorize value. yvec = vectorize(dist, y) # Accumulate the log-abs-det jacobian correction. @@ -1380,14 +1383,14 @@ function _link_metadata!( # Otherwise, we derive the transformation from the distribution. is_transformed[getidx(metadata, vn)] = true - link_transform(getindex(dists, vn)) + to_linked_internal_transform(varinfo, vn, dists[vn]) end # Compute the transformed values. ys = map(vns, link_transforms) do vn, f # TODO: Do we need to handle scenarios where `vn` is not in `dists`? dist = dists[vn] x = getval(metadata, vn) - y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) + y, logjac = with_logabsdet_jacobian(f, x) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) # Return the transformed value. @@ -1487,8 +1490,8 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe # Transform to constrained space. y = getval(varinfo, vn) dist = getdist(varinfo, vn) - f = invlink_transform(dist) - x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) + f = from_linked_internal_transform(varinfo, vn, dist) + x, logjac = with_logabsdet_jacobian(f, y) # Vectorize value. xvec = vectorize(dist, x) # Accumulate the log-abs-det jacobian correction. From 218dc2355300e6f2e8817146789630dcb9929936 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:40:03 +0000 Subject: [PATCH 116/182] added `internal_to_linked_internal_transform` + fixed a few bugs in the linking as a resultt --- src/abstract_varinfo.jl | 12 ++++++++++++ src/varinfo.jl | 13 +++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 604177cec..0b07ac28e 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -890,3 +890,15 @@ possibly linked internal representation of `vn` with `dist` in `varinfo`. function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_maybe_linked_internal_transform(varinfo, vn, dist)) end + +""" + internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the internal representation of `vn` with `dist` +in `varinfo` to a _linked_ internal representation of `vn` with `dist` in `varinfo`. +""" +function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + f_from_internal = from_internal_transform(varinfo, vn, dist) + f_to_linked_internal = to_linked_internal_transform(varinfo, vn, dist) + return f_to_linked_internal ∘ f_from_internal +end diff --git a/src/varinfo.jl b/src/varinfo.jl index c356ed270..4d1ffb07b 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1086,7 +1086,9 @@ function _link!(vi::UntypedVarInfo, spl::AbstractSampler) if ~istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, to_linked_internal_transform(vi, vn, dist)) + _inner_transform!( + vi, vn, dist, internal_to_linked_internal_transform(vi, vn, dist) + ) settrans!!(vi, true, vn) end else @@ -1115,7 +1117,10 @@ end for vn in f_vns dist = getdist(vi, vn) _inner_transform!( - vi, vn, dist, to_linked_internal_transform(vi, vn, dist) + vi, + vn, + dist, + internal_to_linked_internal_transform(vi, vn, dist), ) settrans!!(vi, true, vn) end @@ -1330,7 +1335,7 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar # Transform to constrained space. x = getval(metadata, vn) dist = getdist(metadata, vn) - f = to_linked_internal_transform(varinfo, vn, dist) + f = internal_to_linked_internal_transform(varinfo, vn, dist) y, logjac = with_logabsdet_jacobian(f, x) # Vectorize value. yvec = vectorize(dist, y) @@ -1383,7 +1388,7 @@ function _link_metadata!( # Otherwise, we derive the transformation from the distribution. is_transformed[getidx(metadata, vn)] = true - to_linked_internal_transform(varinfo, vn, dists[vn]) + internal_to_linked_internal_transform(varinfo, vn, dists[vn]) end # Compute the transformed values. ys = map(vns, link_transforms) do vn, f From 1df4293cf99f1b80e34463276d99f5393c74469c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:40:35 +0000 Subject: [PATCH 117/182] added `linked_internal_to_internal_transform` as a complement to `interanl_to_linked_interanl_transform` --- src/abstract_varinfo.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 0b07ac28e..428a7fd00 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -902,3 +902,15 @@ function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::Var f_to_linked_internal = to_linked_internal_transform(varinfo, vn, dist) return f_to_linked_internal ∘ f_from_internal end + +""" + linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a _linked_ internal representation of `vn` with `dist` +in `varinfo` to the internal representation of `vn` with `dist` in `varinfo`. +""" +function linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + f_from_linked_internal = from_linked_internal_transform(varinfo, vn, dist) + f_to_internal = to_internal_transform(varinfo, vn, dist) + return f_to_internal ∘ f_from_linked_internal +end From f8df89672ddb4c02c939c3385fc9d7ed8f00ee51 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:57:00 +0000 Subject: [PATCH 118/182] fixed bugs in `invlink` for `VarInfo` using `linked_internal_to_internal_transform` --- src/varinfo.jl | 9 +++++++-- test/linking.jl | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 4d1ffb07b..d763c15ca 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1192,7 +1192,9 @@ function _invlink!(vi::UntypedVarInfo, spl::AbstractSampler) if istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, from_linked_internal_transform(vi, vn, dist)) + _inner_transform!( + vi, vn, dist, linked_internal_to_internal_transform(vi, vn, dist) + ) settrans!!(vi, false, vn) end else @@ -1221,7 +1223,10 @@ end for vn in f_vns dist = getdist(vi, vn) _inner_transform!( - vi, vn, dist, from_linked_internal_transform(vi, vn, dist) + vi, + vn, + dist, + linked_internal_to_internal_transform(vi, vn, dist), ) settrans!!(vi, false, vn) end diff --git a/test/linking.jl b/test/linking.jl index 9103fff67..c773947c9 100644 --- a/test/linking.jl +++ b/test/linking.jl @@ -44,7 +44,9 @@ function Distributions._logpdf(::MyMatrixDistribution, x::AbstractMatrix{<:Real} end # Skip reconstruction in the inverse-map since it's no longer needed. -DynamicPPL.reconstruct(::TrilFromVec, ::MyMatrixDistribution, x::AbstractVector{<:Real}) = x +function DynamicPPL.from_linked_vec_transform(dist::MyMatrixDistribution) + return TrilFromVec((dist.dim, dist.dim)) +end # Specify the link-transform to use. Bijectors.bijector(dist::MyMatrixDistribution) = TrilToVec((dist.dim, dist.dim)) From d62f26ad545de8387702fefdb6521ed597a37499 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:58:12 +0000 Subject: [PATCH 119/182] more work on removing calls to `reconstruct` --- src/simple_varinfo.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 5e6b99442..6f73e47ea 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -323,12 +323,14 @@ Base.getindex(svi::SimpleVarInfo, ::Colon) = values_as(svi, Vector) # we simply call `getindex` in `getindex_raw`. getindex_raw(vi::SimpleVarInfo, vn::VarName) = vi[vn] function getindex_raw(vi::SimpleVarInfo, vn::VarName, dist::Distribution) - return reconstruct(dist, getindex_raw(vi, vn)) + f = from_internal_transform(vi, vn, dist) + return f(getindex_raw(vi, vn)) end getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}) = vi[vns] function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) # `reconstruct` expects a flattened `Vector` regardless of the type of `dist`, so we `vcat` everything. vals = mapreduce(Base.Fix1(getindex_raw, vi), vcat, vns) + # TODO: Fix this `reconstruct`. return reconstruct(dist, vals, length(vns)) end From b4517d60d3f12c12b0100b8f9e9245b3f24b82bb Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:58:28 +0000 Subject: [PATCH 120/182] removed redundant comment --- src/abstract_varinfo.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 428a7fd00..ce40ae83f 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -820,7 +820,6 @@ function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist) return invlink_with_logpdf(vi, vn, dist, getval(vi, vn)) end function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist, y) - # NOTE: Will this cause type-instabilities or will union-splitting save us? f = from_maybe_linked_internal_transform(vi, vn, dist) x, logjac = with_logabsdet_jacobian(f, y) return x, logpdf(dist, x) + logjac From b7d4754ff938845a2d6911ffa69b4b2a408ab0fd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:58:37 +0000 Subject: [PATCH 121/182] added `from_linked_vec_transform` specialization for `LKJ` --- src/utils.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils.jl b/src/utils.jl index 1b5a958f8..feb612e5b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -267,9 +267,11 @@ function from_linked_vec_transform(dist::Distribution) return f_invlink ∘ f_vec end +# Specializations that circumvent the `from_vec_transform` machinery. function from_linked_vec_transform(dist::LKJCholesky) return inverse(Bijectors.VecCholeskyBijector(dist.uplo)) end +from_linked_vec_transform(dist::LKJ) = inverse(Bijectors.VecCorrBijector()) """ to_vec_transform(x) From 0244dd9f140b4beaf3f631ff4875746a03817265 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 15:04:57 +0000 Subject: [PATCH 122/182] more work on removing references to `reconstruct` --- src/varinfo.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index d763c15ca..a4bebc481 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1674,6 +1674,7 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end + # TODO: Replace when we have better dispatch for multiple vals. return reconstruct(dist, vals_linked, length(vns)) end @@ -1688,12 +1689,14 @@ function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) return getindex_raw(getmetadata(vi, vn), vn) end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) - return reconstruct(dist, getval(vi, vn)) + f = from_internal_transform(vi, vn, dist) + return f(getval(vi, vn)) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) return getindex_raw(vi, vns, getdist(vi, first(vns))) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) + # TODO: Replace when we have better dispatch for multiple vals. return reconstruct(dist, getval(vi, vns), length(vns)) end @@ -2284,7 +2287,7 @@ end function values_from_metadata(md::Metadata) return ( - vn => reconstruct(md.dists[md.idcs[vn]], md.vals[md.ranges[md.idcs[vn]]]) for + vn => from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn)) for vn in md.vns ) end From e886d07085a36554eb2b3f92efd944a1ff7ca0bc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 15:08:10 +0000 Subject: [PATCH 123/182] added `copy` in `values_from_metadata` to preserve behavior and avoid refs to internal representation --- src/varinfo.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index a4bebc481..20c493a3e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2287,7 +2287,8 @@ end function values_from_metadata(md::Metadata) return ( - vn => from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn)) for + # `copy` to avoid accidentaly mutation of internal representation. + vn => copy(from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn))) for vn in md.vns ) end From 2af660526ebf8eb8d1fea3cbd25eb7807dc7014d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 19:42:36 +0000 Subject: [PATCH 124/182] remove `reconstruct_and_link` and `invlink_and_reconstruct` --- src/abstract_varinfo.jl | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index ce40ae83f..2cdd9ad5f 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -753,37 +753,6 @@ function unflatten(sampler::AbstractSampler, varinfo::AbstractVarInfo, ::Abstrac return unflatten(varinfo, sampler, θ) end -# TODO: Once `(inv)link` isn't used heavily in `getindex(vi, vn)`, we can -# just use `first ∘ with_logabsdet_jacobian` to reduce the maintenance burden. -# NOTE: `reconstruct` is no-op if `val` is already of correct shape. -""" - reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) - -Return linked `val` but reconstruct before linking, if necessary. - -Note that unlike [`invlink_and_reconstruct`](@ref), this does not necessarily -return a reconstructed value, i.e. a value of the same type and shape as expected -by `dist`. - -See also: [`invlink_and_reconstruct`](@ref), [`reconstruct`](@ref). -""" -function reconstruct_and_link(varinfo::AbstractVarInfo, vn::VarName, dist, val) - f = to_linked_internal_transform(varinfo, vn, dist) - return f(val) -end - -""" - invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) - -Return invlinked and reconstructed `val`. - -See also: [`reconstruct_and_link`](@ref), [`reconstruct`](@ref). -""" -function invlink_and_reconstruct(varinfo::AbstractVarInfo, vn::VarName, dist, val) - f = from_linked_internal_transform(varinfo, vn, dist) - return f(val) -end - """ maybe_link_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) From a0664d715523cd5d2f13863e2ccffc97bc20725f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:23:14 +0000 Subject: [PATCH 125/182] replaced references to `link_and_reconstruct` and `invlink_and_reconstruct` --- src/context_implementations.jl | 38 ++++++++++++++++++++++------------ src/simple_varinfo.jl | 11 ++++++++-- src/varinfo.jl | 11 ++++++++-- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 2b28b44a9..8835f40e3 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -468,6 +468,19 @@ function dot_assume(rng, spl::Sampler, ::Any, ::AbstractArray{<:VarName}, ::Any, ) end +# HACK: These methods are only used in the `get_and_set_val!` methods below. +# FIXME: Remove these. +function _link_broadcast_new(vi, vn, dist, r) + b = to_linked_internal_transform(vi, dist) + return b(r) +end + +function _maybe_invlink_broadcast(vi, vn, dist) + xvec = getval(vi, vn) + b = from_maybe_linked_internal_transform(vi, vn, dist) + return b(xvec) +end + function get_and_set_val!( rng, vi::VarInfoOrThreadSafeVarInfo, @@ -483,11 +496,8 @@ function get_and_set_val!( r = init(rng, dist, spl, n) for i in 1:n vn = vns[i] - setindex!!( - vi, - vectorize(dist, maybe_reconstruct_and_link(vi, vn, dist, r[:, i])), - vn, - ) + f_link_maybe = to_maybe_linked_internal_transform(vi, vn, dist) + setindex!!(vi, f_link_maybe(r[:, i]), vn) setorder!(vi, vn, get_num_produce(vi)) end else @@ -498,7 +508,8 @@ function get_and_set_val!( for i in 1:n vn = vns[i] if istrans(vi) - push!!(vi, vn, Bijectors.link(dist, r[:, i]), dist, spl) + ri_linked = _link_broadcast_new(vi, vn, dist, r[:, i]) + push!!(vi, vn, ri_linked, dist, spl) # `push!!` sets the trans-flag to `false` by default. settrans!!(vi, true, vn) else @@ -525,17 +536,18 @@ function get_and_set_val!( for i in eachindex(vns) vn = vns[i] dist = dists isa AbstractArray ? dists[i] : dists - setindex!!( - vi, vectorize(dist, maybe_reconstruct_and_link(vi, vn, dist, r[i])), vn - ) + f_link_maybe = to_maybe_linked_internal_transform(vi, vn, dist) + setindex!!(vi, f_link_maybe(r[i]), vn) setorder!(vi, vn, get_num_produce(vi)) end else # r = reshape(vi[vec(vns)], size(vns)) # FIXME: Remove `reconstruct` in `getindex_raw(::VarInfo, ...)` # and fix the lines below. - r_raw = getindex_raw(vi, vec(vns)) - r = maybe_invlink_and_reconstruct.((vi,), vns, dists, reshape(r_raw, size(vns))) + # r_raw = getindex_raw(vi, vec(vns)) + # r = maybe_invlink_and_reconstruct.((vi,), vns, dists, reshape(r_raw, size(vns))) + rs = _maybe_invlink_broadcast.((vi,), vns, dists) + r = reshape(rs, size(vns)) end else f = (vn, dist) -> init(rng, dist, spl) @@ -546,10 +558,10 @@ function get_and_set_val!( # 2. Define an anonymous function which returns `nothing`, which # we then broadcast. This will allocate a vector of `nothing` though. if istrans(vi) - push!!.((vi,), vns, reconstruct_and_link.((vi,), vns, dists, r), dists, (spl,)) + push!!.((vi,), vns, _link_broadcast_new.((vi,), vns, dists, r), dists, (spl,)) # NOTE: Need to add the correction. # FIXME: This is not great. - acclogp_assume!!(vi, sum(logabsdetjac.(bijector.(dists), r))) + acclogp_assume!!(vi, sum(logabsdetjac.(link_transform.(dists), r))) # `push!!` sets the trans-flag to `false` by default. settrans!!.((vi,), true, vns) else diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 6f73e47ea..3048d6ef7 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -693,10 +693,17 @@ function invlink!!( end # With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. -from_internal_transform(::SimpleVarInfo, ::VarName, dist) = identity -function from_linked_internal_transform(vi::SimpleVarInfo, vn::VarName, dist) +from_internal_transform(::SimpleVarInfo, dist) = identity +function from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) + return from_internal_transform(vi, dist) +end + +function from_linked_internal_transform(vi::SimpleVarInfo, dist) return invlink_transform(dist) end +function from_linked_internal_transform(vi::SimpleVarInfo, ::VarName, dist) + return from_linked_internal_transform(vi, dist) +end # Threadsafe stuff. # For `SimpleVarInfo` we don't really need `Ref` so let's not use it. diff --git a/src/varinfo.jl b/src/varinfo.jl index 20c493a3e..a786d172b 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1876,8 +1876,8 @@ function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) end function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) - # FIXME: Include `transform` in the `push!` call below. - return push!(vnv, vn, r) + f = from_vec_transform(dist) + return push!(vnv, vn, r, f) end """ @@ -2296,13 +2296,20 @@ end values_from_metadata(md::VarNameVector) = pairs(md) # Transforming from internal representation to distribution representation. +# Without `vn` argument. +from_internal_transform(vi::VarInfo, dist) = from_vec_transform(dist) +# With `vn` argument. function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) from_internal_transform(::VarNameVector, ::VarName, dist) = from_vec_transform(dist) +# Without `vn` argument. +from_linked_internal_transform(vi::VarInfo, dist) = from_linked_vec_transform(dist) +# With `vn` argument. function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) + # Dispatch to metadata in case this alters the behavior. return from_linked_internal_transform(getmetadata(vi, vn), vn, dist) end function from_linked_internal_transform(::Metadata, ::VarName, dist) From f2d59b2d773812358dbc8713e647f80954d52303 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:30:40 +0000 Subject: [PATCH 126/182] introduced `recombine` and replaced calls to `reconstruct` with `n` samples --- src/context_implementations.jl | 9 +++++---- src/simple_varinfo.jl | 7 ++----- src/utils.jl | 20 ++++++++++++++++++++ src/varinfo.jl | 4 ++-- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 8835f40e3..47c9f187f 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -207,6 +207,7 @@ function assume(dist::Distribution, vn::VarName, vi) return r, logp, vi end +# TODO: Remove this thing. # SampleFromPrior and SampleFromUniform function assume( rng::Random.AbstractRNG, @@ -220,9 +221,8 @@ function assume( if sampler isa SampleFromUniform || is_flagged(vi, vn, "del") unset_flag!(vi, vn, "del") r = init(rng, dist, sampler) - BangBang.setindex!!( - vi, vectorize(dist, maybe_reconstruct_and_link(vi, vn, dist, r)), vn - ) + f = to_maybe_linked_internal_transform(vi, vn, dist) + BangBang.setindex!!(vi, f(r), vn) setorder!(vi, vn, get_num_produce(vi)) else # Otherwise we just extract it. @@ -231,7 +231,8 @@ function assume( else r = init(rng, dist, sampler) if istrans(vi) - push!!(vi, vn, reconstruct_and_link(dist, r), dist, sampler) + f = to_linked_internal_transform(vi, dist) + push!!(vi, vn, f(r), dist, sampler) # By default `push!!` sets the transformed flag to `false`. settrans!!(vi, true, vn) else diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 3048d6ef7..e7c3e90b0 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -298,8 +298,7 @@ function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribu vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end - # TODO: Fix this `reconstruct`. - return reconstruct(dist, vals_linked, length(vns)) + return recombine(dist, vals_linked, length(vns)) end Base.getindex(vi::SimpleVarInfo, vn::VarName) = get(vi.values, vn) @@ -328,10 +327,8 @@ function getindex_raw(vi::SimpleVarInfo, vn::VarName, dist::Distribution) end getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}) = vi[vns] function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) - # `reconstruct` expects a flattened `Vector` regardless of the type of `dist`, so we `vcat` everything. vals = mapreduce(Base.Fix1(getindex_raw, vi), vcat, vns) - # TODO: Fix this `reconstruct`. - return reconstruct(dist, vals, length(vns)) + return recombine(dist, vals, length(vns)) end # HACK: because `VarInfo` isn't ready to implement a proper `getindex_raw`. diff --git a/src/utils.jl b/src/utils.jl index feb612e5b..73c480887 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -380,6 +380,26 @@ function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector, n::In return r end +""" + recombine(dist::Distribution, vals::AbstractVector, n::Int) + +Recombine `vals`, representing a batch of samples from `dist`, so that it's a compatible with `dist`. +""" +function recombine(d::Distribution, val::AbstractVector, n::Int) + return reconstruct(size(d), val, n) +end +function recombine(::Tuple{}, val::AbstractVector, n::Int) + return copy(val) +end +function recombine(s::NTuple{1}, val::AbstractVector, n::Int) + return copy(reshape(val, s[1], n)) +end +function recombine(s::NTuple{2}, val::AbstractVector, n::Int) + tmp = reshape(val, s..., n) + orig = [tmp[:, :, i] for i in 1:n] + return orig +end + # Uniform random numbers with range 4 for robust initializations # Reference: https://mc-stan.org/docs/2_19/reference-manual/initialization.html randrealuni(rng::Random.AbstractRNG) = 4 * rand(rng) - 2 diff --git a/src/varinfo.jl b/src/varinfo.jl index a786d172b..f488bf134 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1675,7 +1675,7 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) getindex(vi, vn, dist) end # TODO: Replace when we have better dispatch for multiple vals. - return reconstruct(dist, vals_linked, length(vns)) + return recombine(dist, vals_linked, length(vns)) end getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) @@ -1697,7 +1697,7 @@ function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) # TODO: Replace when we have better dispatch for multiple vals. - return reconstruct(dist, getval(vi, vns), length(vns)) + return recombine(dist, getval(vi, vns), length(vns)) end """ From e3bfa760efa86e1495d7f0d2d479273ed6b210a5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:33:49 +0000 Subject: [PATCH 127/182] completely removed `reconstruct` --- src/utils.jl | 81 +--------------------------------------------------- 1 file changed, 1 insertion(+), 80 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 73c480887..b88a74639 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -301,92 +301,13 @@ tovec(C::Cholesky) = tovec(Matrix(C.UL)) vectorize(d, r) = vectorize(r) vectorize(r) = tovec(r) -# NOTE: -# We cannot use reconstruct{T} because val is always Vector{Real} then T will be Real. -# However here we would like the result to be specifric type, e.g. Array{Dual{4,Float64}, 2}, -# otherwise we will have error for MatrixDistribution. -# Note this is not the case for MultivariateDistribution so I guess this might be lack of -# support for some types related to matrices (like PDMat). - -""" - reconstruct([f, ]dist, val) - -Reconstruct `val` so that it's compatible with `dist`. - -If `f` is also provided, the reconstruct value will be -such that `f(reconstruct_val)` is compatible with `dist`. -""" -reconstruct(f, dist, val) = reconstruct(dist, val) - -# No-op versions. -reconstruct(::UnivariateDistribution, val::Real) = val -reconstruct(::MultivariateDistribution, val::AbstractVector{<:Real}) = copy(val) -reconstruct(::MatrixDistribution, val::AbstractMatrix{<:Real}) = copy(val) -reconstruct(::Inverse{Bijectors.VecCorrBijector}, ::LKJ, val::AbstractVector) = copy(val) - -function reconstruct(dist::LKJCholesky, val::AbstractVector{<:Real}) - f = from_vec_transform(dist) - return f(val) -end -function reconstruct(dist::LKJCholesky, val::AbstractMatrix{<:Real}) - return Cholesky(val, dist.uplo, 0) -end -reconstruct(::LKJCholesky, val::Cholesky) = val - -function reconstruct( - ::Inverse{Bijectors.VecCholeskyBijector}, ::LKJCholesky, val::AbstractVector -) - return copy(val) -end - -function reconstruct( - ::Inverse{Bijectors.PDVecBijector}, ::MatrixDistribution, val::AbstractVector -) - return copy(val) -end - -# TODO: Implement no-op `reconstruct` for general array variates. - -reconstruct(d::Distribution, val::AbstractVector) = reconstruct(size(d), val) -reconstruct(::Tuple{}, val::AbstractVector) = val[1] -reconstruct(s::NTuple{1}, val::AbstractVector) = copy(val) -reconstruct(s::Tuple, val::AbstractVector) = reshape(copy(val), s) -function reconstruct!(r, d::Distribution, val::AbstractVector) - return reconstruct!(r, d, val) -end -function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector) - r .= val - return r -end -function reconstruct(d::Distribution, val::AbstractVector, n::Int) - return reconstruct(size(d), val, n) -end -function reconstruct(::Tuple{}, val::AbstractVector, n::Int) - return copy(val) -end -function reconstruct(s::NTuple{1}, val::AbstractVector, n::Int) - return copy(reshape(val, s[1], n)) -end -function reconstruct(s::NTuple{2}, val::AbstractVector, n::Int) - tmp = reshape(val, s..., n) - orig = [tmp[:, :, i] for i in 1:n] - return orig -end -function reconstruct!(r, d::Distribution, val::AbstractVector, n::Int) - return reconstruct!(r, d, val, n) -end -function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector, n::Int) - r .= val - return r -end - """ recombine(dist::Distribution, vals::AbstractVector, n::Int) Recombine `vals`, representing a batch of samples from `dist`, so that it's a compatible with `dist`. """ function recombine(d::Distribution, val::AbstractVector, n::Int) - return reconstruct(size(d), val, n) + return recombine(size(d), val, n) end function recombine(::Tuple{}, val::AbstractVector, n::Int) return copy(val) From c0aef8179a253ef985a14946d0317bb75fe6af58 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:37:35 +0000 Subject: [PATCH 128/182] renamed `maybe_reconstruct_and_link` to `to_maybe_linked_internal` and `maybe_invlink_and_reconstruct` to `from_maybe_linked_internal` --- src/abstract_varinfo.jl | 8 ++++---- src/context_implementations.jl | 5 ----- src/simple_varinfo.jl | 10 +++++----- src/varinfo.jl | 2 +- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 2cdd9ad5f..32d6cc31b 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -754,21 +754,21 @@ function unflatten(sampler::AbstractSampler, varinfo::AbstractVarInfo, ::Abstrac end """ - maybe_link_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) + to_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) Return reconstructed `val`, possibly linked if `istrans(vi, vn)` is `true`. """ -function maybe_reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) +function to_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) f = to_maybe_linked_internal_transform(vi, vn, dist) return f(val) end """ - maybe_invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) + from_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) Return reconstructed `val`, possibly invlinked if `istrans(vi, vn)` is `true`. """ -function maybe_invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) +function from_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) f = from_maybe_linked_internal_transform(vi, vn, dist) return f(val) end diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 47c9f187f..1ec241a5d 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -542,11 +542,6 @@ function get_and_set_val!( setorder!(vi, vn, get_num_produce(vi)) end else - # r = reshape(vi[vec(vns)], size(vns)) - # FIXME: Remove `reconstruct` in `getindex_raw(::VarInfo, ...)` - # and fix the lines below. - # r_raw = getindex_raw(vi, vec(vns)) - # r = maybe_invlink_and_reconstruct.((vi,), vns, dists, reshape(r_raw, size(vns))) rs = _maybe_invlink_broadcast.((vi,), vns, dists) r = reshape(rs, size(vns)) end diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index e7c3e90b0..255d8c3b7 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -292,7 +292,7 @@ end # `NamedTuple` function Base.getindex(vi::SimpleVarInfo, vn::VarName, dist::Distribution) - return maybe_invlink_and_reconstruct(vi, vn, dist, getindex(vi, vn)) + return from_maybe_linked_internal(vi, vn, dist, getindex(vi, vn)) end function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) vals_linked = mapreduce(vcat, vns) do vn @@ -476,7 +476,7 @@ function assume( ) value = init(rng, dist, sampler) # Transform if we're working in unconstrained space. - value_raw = maybe_reconstruct_and_link(vi, vn, dist, value) + value_raw = to_maybe_linked_internal(vi, vn, dist, value) vi = BangBang.push!!(vi, vn, value_raw, dist, sampler) return value, Bijectors.logpdf_with_trans(dist, value, istrans(vi, vn)), vi end @@ -494,9 +494,9 @@ function dot_assume( # Transform if we're working in transformed space. value_raw = if dists isa Distribution - maybe_reconstruct_and_link.((vi,), vns, (dists,), value) + to_maybe_linked_internal.((vi,), vns, (dists,), value) else - maybe_reconstruct_and_link.((vi,), vns, dists, value) + to_maybe_linked_internal.((vi,), vns, dists, value) end # Update `vi` @@ -523,7 +523,7 @@ function dot_assume( # Update `vi`. for (vn, val) in zip(vns, eachcol(value)) - val_linked = maybe_reconstruct_and_link(vi, vn, dist, val) + val_linked = to_maybe_linked_internal(vi, vn, dist, val) vi = BangBang.setindex!!(vi, val_linked, vn) end diff --git a/src/varinfo.jl b/src/varinfo.jl index f488bf134..14b67a37e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1650,7 +1650,7 @@ getindex(vi::VarInfo, vn::VarName) = getindex(vi, vn, getdist(vi, vn)) function getindex(vi::VarInfo, vn::VarName, dist::Distribution) @assert haskey(vi, vn) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" val = getval(vi, vn) - return maybe_invlink_and_reconstruct(vi, vn, dist, val) + return from_maybe_linked_internal(vi, vn, dist, val) end # HACK: Allows us to also work with `VarNameVector` where `dist` is not used, # but we instead use a transformation stored with the variable. From f7c0853756c9a1bb82b02d5c50dbdf4e7f7175cc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 30 Jan 2024 10:56:58 +0000 Subject: [PATCH 129/182] added impls of `from_*_internal_transform` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 91b9704c7..160e88c11 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -244,3 +244,13 @@ end function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) return invlink_with_logpdf(vi.varinfo, vn, dist, y) end + +function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) + return from_linked_internal_transform(varinfo.varinfo, vn, dist) +end + +function from_linked_internal_transform( + varinfo::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist +) + return from_linked_internal_transform(varinfo.varinfo, vns, dist) +end From 77b835e3fcaedaa52c01866f40b04b517ec51c55 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 30 Jan 2024 11:33:59 +0000 Subject: [PATCH 130/182] removed `reconstruct` from docs and from exports --- docs/src/api.md | 1 - src/DynamicPPL.jl | 4 ---- 2 files changed, 5 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index f9db6603c..397b178ac 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -298,7 +298,6 @@ DynamicPPL.link!! DynamicPPL.invlink!! DynamicPPL.default_transformation DynamicPPL.maybe_invlink_before_eval!! -DynamicPPL.reconstruct ``` #### Utils diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 7de80015e..ec0b086f7 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -79,10 +79,6 @@ export AbstractVarInfo, # Compiler @model, # Utilities - vectorize, - reconstruct, - reconstruct!, - Sample, init, vectorize, OrderedDict, From b83c26203e2b8291769dd1ab821986896a69d06b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:40:09 +0000 Subject: [PATCH 131/182] renamed `getval` to `getindex_internal` and made `dist` an optional argument for all the transform-related methods --- src/abstract_varinfo.jl | 63 +++++++++++++++++++---- src/context_implementations.jl | 2 +- src/simple_varinfo.jl | 15 ++---- src/threadsafe.jl | 2 +- src/utils.jl | 6 ++- src/varinfo.jl | 94 +++++++++++++++++++++------------- src/varnamevector.jl | 2 +- test/varinfo.jl | 2 +- 8 files changed, 124 insertions(+), 62 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 32d6cc31b..c048014b5 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -779,14 +779,14 @@ end Invlink `x` and compute the logpdf under `dist` including correction from the invlink-transformation. -If `x` is not provided, `getval(vi, vn)` will be used. +If `x` is not provided, `getindex_internal(vi, vn)` will be used. !!! warning The input value `x` should be according to the internal representation of - `varinfo`, e.g. the value returned by `getval(vi, vn)`. + `varinfo`, e.g. the value returned by `getindex_internal(vi, vn)`. """ function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist) - return invlink_with_logpdf(vi, vn, dist, getval(vi, vn)) + return invlink_with_logpdf(vi, vn, dist, getindex_internal(vi, vn)) end function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist, y) f = from_maybe_linked_internal_transform(vi, vn, dist) @@ -800,26 +800,32 @@ increment_num_produce!(::AbstractVarInfo) = nothing setgid!(vi::AbstractVarInfo, gid::Selector, vn::VarName) = nothing """ - from_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + from_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from the internal representation of `vn` with `dist` in `varinfo` to a representation compatible with `dist`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function from_internal_transform end """ - from_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + from_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from the linked internal representation of `vn` with `dist` in `varinfo` to a representation compatible with `dist`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function from_linked_internal_transform end """ - from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from the possibly linked internal representation of `vn` with `dist`n in `varinfo` to a representation compatible with `dist`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return if istrans(varinfo, vn) @@ -828,57 +834,94 @@ function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarN from_internal_transform(varinfo, vn, dist) end end +function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return if istrans(varinfo, vn) + from_linked_internal_transform(varinfo, vn) + else + from_internal_transform(varinfo, vn) + end +end """ - to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + to_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a representation compatible with `dist` to the internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_internal_transform(varinfo, vn, dist)) end +function to_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return inverse(from_internal_transform(varinfo, vn)) +end """ - to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a representation compatible with `dist` to the linked internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_linked_internal_transform(varinfo, vn, dist)) end +function to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return inverse(from_linked_internal_transform(varinfo, vn)) +end """ - to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a representation compatible with `dist` to a possibly linked internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_maybe_linked_internal_transform(varinfo, vn, dist)) end +function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return inverse(from_maybe_linked_internal_transform(varinfo, vn)) +end """ internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) Return a transformation that transforms from the internal representation of `vn` with `dist` in `varinfo` to a _linked_ internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) f_from_internal = from_internal_transform(varinfo, vn, dist) f_to_linked_internal = to_linked_internal_transform(varinfo, vn, dist) return f_to_linked_internal ∘ f_from_internal end +function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + f_from_internal = from_internal_transform(varinfo, vn) + f_to_linked_internal = to_linked_internal_transform(varinfo, vn) + return f_to_linked_internal ∘ f_from_internal +end """ - linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a _linked_ internal representation of `vn` with `dist` in `varinfo` to the internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) f_from_linked_internal = from_linked_internal_transform(varinfo, vn, dist) f_to_internal = to_internal_transform(varinfo, vn, dist) return f_to_internal ∘ f_from_linked_internal end + +function linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + f_from_linked_internal = from_linked_internal_transform(varinfo, vn) + f_to_internal = to_internal_transform(varinfo, vn) + return f_to_internal ∘ f_from_linked_internal +end diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 1ec241a5d..6cf8bc42d 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -477,7 +477,7 @@ function _link_broadcast_new(vi, vn, dist, r) end function _maybe_invlink_broadcast(vi, vn, dist) - xvec = getval(vi, vn) + xvec = getindex_internal(vi, vn) b = from_maybe_linked_internal_transform(vi, vn, dist) return b(xvec) end diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 255d8c3b7..dc2780aae 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -332,7 +332,7 @@ function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribut end # HACK: because `VarInfo` isn't ready to implement a proper `getindex_raw`. -getval(vi::SimpleVarInfo, vn::VarName) = getindex_raw(vi, vn) +getindex_internal(vi::SimpleVarInfo, vn::VarName) = getindex_raw(vi, vn) Base.haskey(vi::SimpleVarInfo, vn::VarName) = hasvalue(vi.values, vn) @@ -690,16 +690,11 @@ function invlink!!( end # With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. -from_internal_transform(::SimpleVarInfo, dist) = identity -function from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) - return from_internal_transform(vi, dist) -end - -function from_linked_internal_transform(vi::SimpleVarInfo, dist) - return invlink_transform(dist) -end +from_internal_transform(vi::SimpleVarInfo, ::VarName) = identity +from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) = identity +from_linked_internal_transform(vi::SimpleVarInfo, ::VarName) = identity function from_linked_internal_transform(vi::SimpleVarInfo, ::VarName, dist) - return from_linked_internal_transform(vi, dist) + return invlink_transform(dist) end # Threadsafe stuff. diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 160e88c11..648a8ef1c 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -222,7 +222,7 @@ end istrans(vi::ThreadSafeVarInfo, vn::VarName) = istrans(vi.varinfo, vn) istrans(vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}) = istrans(vi.varinfo, vns) -getval(vi::ThreadSafeVarInfo, vn::VarName) = getval(vi.varinfo, vn) +getindex_internal(vi::ThreadSafeVarInfo, vn::VarName) = getindex_internal(vi.varinfo, vn) function unflatten(vi::ThreadSafeVarInfo, x::AbstractVector) return Setfield.@set vi.varinfo = unflatten(vi.varinfo, x) diff --git a/src/utils.jl b/src/utils.jl index b88a74639..f75290af2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -256,10 +256,14 @@ from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ FromVec(size(dist)) """ - from_linked_vec_transform(dist) + from_linked_vec_transform(dist::Distribution) Return the transformation from the unconstrained vector to the constrained realization of distribution `dist`. + +By default, this is just `invlink_transform(dist) ∘ from_vec_transform(dist)`. + +See also: [`DynamicPPL.invlink_transform`](@ref), [`DynamicPPL.from_vec_transform`](@ref). """ function from_linked_vec_transform(dist::Distribution) f_vec = from_vec_transform(dist) diff --git a/src/varinfo.jl b/src/varinfo.jl index 14b67a37e..a12a25cc3 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -449,8 +449,8 @@ function merge_metadata(metadata_left::Metadata, metadata_right::Metadata) push!(vns, vn) if vn in vns_left && vn in vns_right # `vals`: only valid if they're the length. - vals_left = getval(metadata_left, vn) - vals_right = getval(metadata_right, vn) + vals_left = getindex_internal(metadata_left, vn) + vals_right = getindex_internal(metadata_right, vn) @assert length(vals_left) == length(vals_right) append!(vals, vals_right) # `ranges` @@ -471,7 +471,7 @@ function merge_metadata(metadata_left::Metadata, metadata_right::Metadata) elseif vn in vns_left # Just extract the metadata from `metadata_left`. # `vals` - vals_left = getval(metadata_left, vn) + vals_left = getindex_internal(metadata_left, vn) append!(vals, vals_left) # `ranges` r = (offset + 1):(offset + length(vals_left)) @@ -489,7 +489,7 @@ function merge_metadata(metadata_left::Metadata, metadata_right::Metadata) else # Just extract the metadata from `metadata_right`. # `vals` - vals_right = getval(metadata_right, vn) + vals_right = getindex_internal(metadata_right, vn) append!(vals, vals_right) # `ranges` r = (offset + 1):(offset + length(vals_right)) @@ -513,11 +513,11 @@ end const VarView = Union{Int,UnitRange,Vector{Int}} """ - getval(vi::UntypedVarInfo, vview::Union{Int, UnitRange, Vector{Int}}) + getindex_internal(vi::UntypedVarInfo, vview::Union{Int, UnitRange, Vector{Int}}) Return a view `vi.vals[vview]`. """ -getval(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) +getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) """ setval!(vi::UntypedVarInfo, val, vview::Union{Int, UnitRange, Vector{Int}}) @@ -584,16 +584,16 @@ getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] getdist(::VarNameVector, ::VarName) = nothing """ - getval(vi::VarInfo, vn::VarName) + getindex_internal(vi::VarInfo, vn::VarName) Return the value(s) of `vn`. The values may or may not be transformed to Euclidean space. """ -getval(vi::VarInfo, vn::VarName) = getval(getmetadata(vi, vn), vn) -getval(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) +getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) # HACK: We shouldn't need this -getval(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) +getindex_internal(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) """ setval!(vi::VarInfo, val, vn::VarName) @@ -614,13 +614,14 @@ function setval!(vnv::VarNameVector, val, vn::VarName) end """ - getval(vi::VarInfo, vns::Vector{<:VarName}) + getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) Return the value(s) of `vns`. The values may or may not be transformed to Euclidean space. """ -getval(vi::VarInfo, vns::Vector{<:VarName}) = mapreduce(Base.Fix1(getval, vi), vcat, vns) +getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) = + mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) """ getall(vi::VarInfo) @@ -634,7 +635,9 @@ getall(vi::VarInfo) = getall(vi.metadata) # See for example https://github.com/JuliaLang/julia/pull/46381. getall(vi::TypedVarInfo) = reduce(vcat, map(getall, vi.metadata)) function getall(md::Metadata) - return mapreduce(Base.Fix1(getval, md), vcat, md.vns; init=similar(md.vals, 0)) + return mapreduce( + Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) + ) end getall(vnv::VarNameVector) = vnv.vals @@ -1246,7 +1249,7 @@ end function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, dist, f) # TODO: Use inplace versions to avoid allocations - yvec, logjac = with_logabsdet_jacobian(f, getval(vi, vn)) + yvec, logjac = with_logabsdet_jacobian(f, getindex_internal(vi, vn)) # Determine the new range. start = first(getrange(vi, vn)) # NOTE: `length(yvec)` should never be longer than `getrange(vi, vn)`. @@ -1338,7 +1341,7 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar end # Transform to constrained space. - x = getval(metadata, vn) + x = getindex_internal(metadata, vn) dist = getdist(metadata, vn) f = internal_to_linked_internal_transform(varinfo, vn, dist) y, logjac = with_logabsdet_jacobian(f, x) @@ -1399,7 +1402,7 @@ function _link_metadata!( ys = map(vns, link_transforms) do vn, f # TODO: Do we need to handle scenarios where `vn` is not in `dists`? dist = dists[vn] - x = getval(metadata, vn) + x = getindex_internal(metadata, vn) y, logjac = with_logabsdet_jacobian(f, x) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) @@ -1498,7 +1501,7 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe end # Transform to constrained space. - y = getval(varinfo, vn) + y = getindex_internal(varinfo, vn) dist = getdist(varinfo, vn) f = from_linked_internal_transform(varinfo, vn, dist) x, logjac = with_logabsdet_jacobian(f, y) @@ -1547,7 +1550,7 @@ function _invlink_metadata!( # Compute the transformed values. xs = map(vns) do vn f = gettransform(metadata, vn) - y = getval(metadata, vn) + y = getindex_internal(metadata, vn) # No need to use `with_reconstruct` as `f` will include this. x, logjac = with_logabsdet_jacobian(f, y) # Accumulate the log-abs-det jacobian correction. @@ -1649,7 +1652,7 @@ end getindex(vi::VarInfo, vn::VarName) = getindex(vi, vn, getdist(vi, vn)) function getindex(vi::VarInfo, vn::VarName, dist::Distribution) @assert haskey(vi, vn) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" - val = getval(vi, vn) + val = getindex_internal(vi, vn) return from_maybe_linked_internal(vi, vn, dist, val) end # HACK: Allows us to also work with `VarNameVector` where `dist` is not used, @@ -1662,12 +1665,10 @@ function getindex(vi::VarInfo, vn::VarName, ::Nothing) end function getindex(vi::VarInfo, vns::Vector{<:VarName}) - # FIXME(torfjelde): Using `getdist(vi, first(vns))` won't be correct in cases - # such as `x .~ [Normal(), Exponential()]`. - # BUT we also can't fix this here because this will lead to "incorrect" - # behavior if `vns` arose from something like `x .~ MvNormal(zeros(2), I)`, - # where by "incorrect" we mean there exists pieces of code expecting this behavior. - return getindex(vi, vns, getdist(vi, first(vns))) + vals_linked = mapreduce(vcat, vns) do vn + getindex(vi, vn) + end + return recombine(vi, vals_linked, length(vns)) end function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) @assert haskey(vi, vns[1]) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" @@ -1690,14 +1691,14 @@ function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) f = from_internal_transform(vi, vn, dist) - return f(getval(vi, vn)) + return f(getindex_internal(vi, vn)) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) return getindex_raw(vi, vns, getdist(vi, first(vns))) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) # TODO: Replace when we have better dispatch for multiple vals. - return recombine(dist, getval(vi, vns), length(vns)) + return recombine(dist, getindex_internal(vi, vns), length(vns)) end """ @@ -1707,7 +1708,7 @@ Return the current value(s) of the random variables sampled by `spl` in `vi`. The value(s) may or may not be transformed to Euclidean space. """ -getindex(vi::VarInfo, spl::Sampler) = copy(getval(vi, _getranges(vi, spl))) +getindex(vi::VarInfo, spl::Sampler) = copy(getindex_internal(vi, _getranges(vi, spl))) function getindex(vi::TypedVarInfo, spl::Sampler) # Gets the ranges as a NamedTuple ranges = _getranges(vi, spl) @@ -2288,26 +2289,45 @@ end function values_from_metadata(md::Metadata) return ( # `copy` to avoid accidentaly mutation of internal representation. - vn => copy(from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn))) for - vn in md.vns + vn => copy( + from_internal_transform(md, vn, getdist(md, vn))(getindex_internal(md, vn)) + ) for vn in md.vns ) end values_from_metadata(md::VarNameVector) = pairs(md) # Transforming from internal representation to distribution representation. -# Without `vn` argument. -from_internal_transform(vi::VarInfo, dist) = from_vec_transform(dist) -# With `vn` argument. +# Without `dist` argument: base on `dist` extracted from self. +function from_internal_transform(vi::VarInfo, vn::VarName) + return from_internal_transform(getmetadata(vi, vn), vn) +end +function from_internal_transform(md::Metadata, vn::VarName) + return from_internal_transform(md, vn, getdist(md, vn)) +end +function from_internal_transform(md::VarNameVector, vn::VarName) + return gettransform(md, vn) +end +# With both `vn` and `dist` arguments: base on provided `dist`. function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) -from_internal_transform(::VarNameVector, ::VarName, dist) = from_vec_transform(dist) +function from_internal_transform(::VarNameVector, ::VarName, dist) + return from_vec_transform(dist) +end -# Without `vn` argument. -from_linked_internal_transform(vi::VarInfo, dist) = from_linked_vec_transform(dist) -# With `vn` argument. +# Without `dist` argument: base on `dist` extracted from self. +function from_linked_internal_transform(vi::VarInfo, vn::VarName) + return from_linked_internal_transform(getmetadata(vi, vn), vn) +end +function from_linked_internal_transform(md::Metadata, vn::VarName) + return from_linked_internal_transform(md, vn, getdist(md, vn)) +end +function from_linked_internal_transform(md::VarNameVector, vn::VarName) + return gettransform(md, vn) +end +# With both `vn` and `dist` arguments: base on provided `dist`. function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) # Dispatch to metadata in case this alters the behavior. return from_linked_internal_transform(getmetadata(vi, vn), vn, dist) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 7bbb8bda8..ed919a20c 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -393,7 +393,7 @@ function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) isempty(vnv) && return vnv_new for vn in vns - push!(vnv_new, vn, getval(vnv, vn), gettransform(vnv, vn)) + push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) end return vnv_new diff --git a/test/varinfo.jl b/test/varinfo.jl index ddd1b7ea8..570f9623e 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -644,7 +644,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) # Should only get the variables subsumed by `@varname(s)`. @test varinfo[spl] == - mapreduce(Base.Fix1(DynamicPPL.getval, varinfo), vcat, vns_s) + mapreduce(Base.Fix1(DynamicPPL.getindex_internal, varinfo), vcat, vns_s) # `link` varinfo_linked = DynamicPPL.link(varinfo, spl, model) From c4faf3e47e033161f2bec9abbdd4e62784fa1e11 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:41:01 +0000 Subject: [PATCH 132/182] updated docs + added description of how internals of transforms work --- docs/make.jl | 2 +- docs/src/api.md | 3 + docs/src/internals/transformations.md | 377 ++++++++++++++++++++++++++ docs/src/internals/varinfo.md | 2 + 4 files changed, 383 insertions(+), 1 deletion(-) create mode 100644 docs/src/internals/transformations.md diff --git a/docs/make.jl b/docs/make.jl index c38853d5b..de557ef9f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,7 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], - "Internals" => ["internals/varinfo.md"], + "Internals" => ["internals/varinfo.md", "internals/transformations.md"], ], checkdocs=:exports, doctest=false, diff --git a/docs/src/api.md b/docs/src/api.md index 397b178ac..e07d762da 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -270,6 +270,7 @@ resetlogp!! keys getindex DynamicPPL.getindex_raw +DynamicPPL.getindex_internal push!! empty!! isempty @@ -297,6 +298,8 @@ DynamicPPL.invlink DynamicPPL.link!! DynamicPPL.invlink!! DynamicPPL.default_transformation +DynamicPPL.link_transform +DynamicPPL.invlink_transform DynamicPPL.maybe_invlink_before_eval!! ``` diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md new file mode 100644 index 000000000..7ee800556 --- /dev/null +++ b/docs/src/internals/transformations.md @@ -0,0 +1,377 @@ +# Transforming variables + +## Motivation + +In a probabilistic programming language (PPL) such as DynamicPPL.jl, one crucial functionality for enabling a large number of inference algorithms to be implemented, in particular gradient-based ones, is the ability to work with "unconstrained" variables. + +For example, consider the following model: + +```julia +@model function demo() + s ~ InverseGamma(2, 3) + m ~ Normal(0, √s) +end +``` + +Here we have two variables `s` and `m`, where `s` is constrained to be positive, while `m` can be any real number. + +For certain inference methods, it's necessary / much more convenient to work with an equivalent model to `demo` but where all the variables can take any real values (they're "unconstrained"). + +!!! note + We write "unconstrained" with quotes because there are many ways to transform a constrained variable to an unconstrained one, *and* DynamicPPL can work with a much broader class of bijective transformations of variables, not just ones that go to the entire real line. But for MCMC, unconstraining is the most common transformation so we'll stick with that terminology. + +For a large family of constraints encoucntered in practice, it is indeed possible to transform a (partially) contrained model to a completely unconstrained one in such a way that sampling in the unconstrained space is equivalent to sampling in the constrained space. + +In DynamicPPL.jl, this is often referred to as *linking* (a term originating in the statistics literature) and is done using transformations from [Bijectors.jl](https://github.com/TuringLang/Bijectors.jl). + +For example, the above model could be transformed into (the following psuedo-code; it's not working code): + +```julia +@model function demo() + log_s ~ log(InverseGamma(2, 3)) + s = exp(log_s) + m ~ Normal(0, √s) +end +``` + +Here `log_s` is an unconstrained variable, and `s` is a constrained variable that is a deterministic function of `log_s`. + +But to ensure that we stay consistent with what the user expects, DynamicPPL.jl does not actually transform the model as above, but can instead makes use of transformed variables internally to achieve the same effect, when desired. + +In the end, we'll end up with something that looks like this: + +```@raw html +
+ +
+``` + +Below we'll see how this is done. + +## What do we need? + +There are two aspects to transforming from the internal representation of a variable in a `varinfo` to the representation wanted in the model: +1. Different implementations of [`AbstractVarInfo`](@ref) represent realizations of a model in different ways internally, so we need to transform from this internal representation to the desired representation in the model. For example, + - [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. + - [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). +2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. + + + +## Working example + +A good and non-trivial example to keep in mind throughout is the following model: + +```@example transformations-internal +using DynamicPPL, Distributions +@model demo_lkj() = x ~ LKJCholesky(2, 1.0) +``` + +`LKJCholesky` is a `LKJ(2, 1.0)` distribution, a distribution over correlation matrices (covariance matrices but with unit diagonal), but working directly with the Cholesky factorization of the correlation matrix rather than the correlation matrix itself (this is more numerically stable and computationally efficient). + +!!! note + This is a particularly "annoying" case because the return-value is not a simple `Real` or `AbstractArray{<:Real}`, but rather a `LineraAlgebra.Cholesky` object which wraps a triangular matrix (whether it's upper- or lower-triangular depends on the instance). + +As mentioned, some implementations of `AbstractVarInfo`, e.g. [`VarInfo`](@ref), works with a "flattened" / vector representation of a variable, and so in this case we need two transformations: +1. From the `Cholesky` object to a vector representation. +2. From the `Cholesky` object to an "unconstrained" / linked vector representation. + +And similarly, we'll need the inverses of these transformations. + +## From internal representation to model representation + +To go from the internal variable representation of an `AbstractVarInfo` to the variable representation wanted in the model, e.g. from a `Vector{Float64}` to `Cholesky` in the case of [`VarInfo`](@ref) in `demo_lkj`, we have the following methods: + +```@docs +DynamicPPL.to_internal_transform +DynamicPPL.from_internal_transform +``` + +These methods allows us to extract the internal-to-model transformation function depending on the `varinfo`, the variable, and the distribution of the variable: +- `varinfo` + `vn` defines the internal representation of the variable. +- `dist` defines the representation expected within the model scope. + +!!! note + If `vn` is not present in `varinfo`, then the internal representation is fully determined by `varinfo` alone. This is used when we're about to add a new variable to the `varinfo` and need to know how to represent it internally. + +Continuing from the example above, we can inspect the internal representation of `x` in `demo_lkj` with [`VarInfo`](@ref) using [`DynamicPPL.getindex_internal`](@ref): + +```@example transformations-internal +model = demo_lkj() +varinfo = VarInfo(model) +x_internal = DynamicPPL.getindex_internal(varinfo, @varname(x)) +``` + +```@example transformations-internal +f_from_internal = DynamicPPL.from_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) +f_from_internal(x_internal) +``` + +Let's confirm that this is the same as `varinfo[@varname(x)]`: + +```@example transformations-internal +x_model = varinfo[@varname(x)] +``` + +Similarly, we can go from the model representation to the internal representation: + +```@example transformations-internal +f_to_internal = DynamicPPL.to_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) + +f_to_internal(x_model) +``` + +It's also useful to see how this is done in [`SimpleVarInfo`](@ref): + +```@example transformations-internal +simple_varinfo = SimpleVarInfo(varinfo) +DynamicPPL.getindex_internal(simple_varinfo, @varname(x)) +``` + +Here see that the internal representation is exactly the same as the model representation, and so we'd expect `from_internal_transform` to be the `identity` function: + +```@example transformations-internal +DynamicPPL.from_internal_transform( + simple_varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) +``` + +Great! + +## From *unconstrained* internal representation to model representation + +In addition to going from internal representation to model representation of a variable, we also need to be able to go from the *unconstrained* internal representation to the model representation. + +For this, we have the following methods: + +```@docs +DynamicPPL.to_linked_internal_transform +DynamicPPL.from_linked_internal_transform +``` + +These are very similar to [`DynamicPPL.to_internal_transform`](@ref) and [`DynamicPPL.from_internal_transform`](@ref), but here the internal representation is also linked / "unconstrained". + +Continuing from the example above: + +```@example transformations-internal +f_to_linked_internal = DynamicPPL.to_linked_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) + +x_linked_internal = f_to_linked_internal(x_model) +``` + +```@example transformations-internal +f_from_linked_internal = DynamicPPL.from_linked_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) + +f_from_linked_internal(x_linked_internal) +``` + +Here we see a significant difference between the linked representation and the non-linked representation: the linked representation is only of length 1, whereas the non-linked representation is of length 4. This is because we actually only need a single element to represent a 2x2 correlation matrix, as the diagonal elements are always 1 *and* it's symmetric. + +We can also inspect the transforms themselves: + +```@example transformations-internal +f_from_internal +``` + +vs. + +```@example transformations-internal +f_from_linked_internal +``` + +Here we see that `f_from_linked_internal` is a single function taking us directly from the linked representation to the model representation, whereas `f_from_internal` is a composition of a few functions: one reshaping the underlying length 4 array into 2x2 matrix, and the other converting this matrix into a `Cholesky`, as required to be compatible with `LKJCholesky(2, 1.0)`. + +## Why do we need both `to_internal_transform` and `to_linked_internal_transform`? + +One might wonder why we need both `to_internal_transform` and `to_linked_internal_transform` instead of just a single `to_internal_transform` which returns the "standard" internal representation if the variable is not linked / "unconstrained" and the linked / "unconstrained" internal representation if it is. + +That is, why can't we just do + +```@raw html +
+ +
+``` + +Unfortunately, this is not possible in general. Consider for example the following model: + +```@example transformations-internal +@model function demo_dynamic_constraint() + m ~ Normal() + x ~ truncated(Normal(), lower=m) + + return (m=m, x=x) +end +``` + +Here the variable `x` has is constrained to be on the domain `(m, Inf)`, where `m` is sampled according to a `Normal`. + +```@example transformations-internal +model = demo_dynamic_constraint() +varinfo = VarInfo(model) +varinfo[@varname(m)], varinfo[@varname(x)] +``` + +We see that the realization of `x` is indeed greater than `m`, as expected. + +But what if we [`link`](@ref) this `varinfo` so that we end up working on an "unconstrained" space, i.e. both `m` and `x` can take on any values in `(-Inf, Inf)`: + +```@example transformations-internal +varinfo_linked = link(varinfo, model) +varinfo_linked[@varname(m)], varinfo_linked[@varname(x)] +``` + +Still get the same values, as expected, since internally `varinfo` transforms from the linked internal representation to the model representation. + +But what if we change the value of `m`, to, say, a bit larger than `x`? + +```@example transformations-internal +# Update realization for `m` in `varinfo_linked`. +varinfo_linked[@varname(m)] = varinfo_linked[@varname(x)] + 1 +varinfo_linked[@varname(m)], varinfo_linked[@varname(x)] +``` + +Now we see that the constraint `m < x` is no longer satisfied! + +Hence one might expect that if we try to compute, say, the [`logjoint`](@ref) using `varinfo_linked` with this "invalid" realization, we'll get an error: + +```@example transformations-internal +logjoint(model, varinfo_linked) +``` + +But we don't! In fact, if we look at the actual value used within the model + +```@example transformations-internal +first(DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext())) +``` + +we see that we indeed satisfy the constraint `m < x`, as desired. + +!!! warning + One shouldn't be setting variables in a linked `varinfo` willy-nilly directly like this unless one knows that the value will be compatible with the constraints of the model. + +The reason for this is that internally in a model evaluation, we construct the transformation from the internal to the model representation based on the *current* realizations in the model! That is, we take the `dist` in a `x ~ dist` expression _at model evaluation time_ and use that to construct the transformation, thus allowing it to change between model evaluations without invalidating the transformation. + +But to be able to do this, we need to know whether the variable is linked / "unconstrained" or not, since the transformation is different in the two cases. Hence we need to be able to determine this at model evaluation time. Hence the the internals end up looking something like this: + +```julia +if istrans(varinfo, varname) + from_linked_internal_transform(varinfo, varname, dist) +else + from_internal_transform(varinfo, varname, dist) +end +``` + +That is, if the variable is linked / "unconstrained", we use the [`DynamicPPL.from_linked_internal_transform`](@ref), otherwise we use [`DynamicPPL.from_internal_transform`](@ref). + +And so the earlier diagram becomes: + +```@raw html +
+ +
+``` + +!!! note + If the support of `dist` was constant, this would not be necessary since we could just determine the transformation at the time of `varinfo_linked = link(varinfo, model)` and define this as the `from_internal_transform` for all subsequent evaluations. However, since the support of `dist` is *not* constant in general, we need to be able to determine the transformation at the time of the evaluation *and* thus whether we should construct the transformation from the linked internal representation or the non-linked internal representation. This is annoying, but necessary. + +This is also the reason why we have two definitions of `getindex`: +- [`getindex(::AbstractVarInfo, ::VarName, ::Distribution)`](@ref): used internally in model evaluations with the `dist` in a `x ~ dist` expression. +- [`getindex(::AbstractVarInfo, ::VarName)`](@ref): used externally by the user to get the realization of a variable. + +For `getindex` we have the following diagram: + +```@raw html +
+ +
+``` + +While if `dist` is not provided, we have: + +```@raw html +
+ +
+``` + +Notice that `dist` is not present here, but otherwise the diagrams are the same. + +!!! warning + This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occcurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + +## Other functionalities + +There are also some additional methods for transforming between representations that are all automatically implemented from [`DynamicPPL.from_internal_transform`](@ref), [`DynamicPPL.from_linked_internal_transform`](@ref) and their siblings, and thus don't need to be implemented manually. + +Convenience methods for constructing transformations: + +```@docs +DynamicPPL.from_maybe_linked_internal_transform +DynamicPPL.to_maybe_linked_internal_transform +DynamicPPL.internal_to_linked_internal_transform +DynamicPPL.linked_internal_to_internal_transform +``` + +Convenience methods for transforming between representations without having to explicitly construct the transformation: + +```@docs +DynamicPPL.to_maybe_linked_internal +DynamicPPL.from_maybe_linked_internal +``` + +# Supporting a new distribution + +To support a new distribution, one needs to implement for the desired `AbstractVarInfo` the following methods: +- [`DynamicPPL.from_internal_transform`](@ref) +- [`DynamicPPL.from_linked_internal_transform`](@ref) + +At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implemente the following methods instead: + +```@docs +DynamicPPL.from_vec_transform(::Distribution) +DynamicPPL.from_linked_vec_transform(::Distribution) +``` + +These are used internally by [`VarInfo`](@ref). + +Optionally, if `inverse` of the above is expensive to compute, one can also implement: +- [`DynamicPPL.to_internal_transform`](@ref) +- [`DynamicPPL.to_linked_internal_transform`](@ref) + +And similarly, there are corresponding to-methods for the `from_*_vec_transform` variants too + +```@docs +DynamicPPL.to_vec_transform +DynamicPPL.to_linked_vec_transform +``` + +!!! warning + Whatever the resulting transformation is, it should be invertible, i.e. implement `InverseFunctions.inverse`, and have a well-defined log-abs-det Jacobian, i.e. implement `ChangesOfVariables.with_logabsdet_jacobian`. + +# TL;DR + +- DynamicPPL.jl has three representations of a variable: the **model representation**, the **internal representation**, and the **linked internal representation**. + - The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. + - The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). + - The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). +- Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. + diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 1312f152c..8d74fd2fa 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -49,6 +49,8 @@ In addition, we want to be able to access the transformed / "unconstrained" real - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal representaton of a particular `VarName`. + Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: 1. Type-stable when possible, but functional when not. From c8d9695f1c460126b51f720c26c6f26decc10e49 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:44:42 +0000 Subject: [PATCH 133/182] added a bunch of illustrations for the transforms docs + dot files used to generated --- ...transformations-assume-without-istrans.dot | 17 +++ ...sformations-assume-without-istrans.dot.png | Bin 0 -> 25856 bytes ...sformations-assume-without-istrans.dot.svg | 88 +++++++++++++ .../assets/images/transformations-assume.dot | 22 ++++ .../images/transformations-assume.dot.png | Bin 0 -> 44454 bytes .../transformations-getindex-with-dist.dot | 20 +++ ...transformations-getindex-with-dist.dot.png | Bin 0 -> 42305 bytes .../transformations-getindex-without-dist.dot | 20 +++ ...nsformations-getindex-without-dist.dot.png | Bin 0 -> 40798 bytes docs/src/assets/images/transformations.dot | 28 ++++ .../src/assets/images/transformations.dot.png | Bin 0 -> 56255 bytes .../src/assets/images/transformations.dot.svg | 124 ++++++++++++++++++ docs/src/internals/transformations.md | 2 +- 13 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 docs/src/assets/images/transformations-assume-without-istrans.dot create mode 100644 docs/src/assets/images/transformations-assume-without-istrans.dot.png create mode 100644 docs/src/assets/images/transformations-assume-without-istrans.dot.svg create mode 100644 docs/src/assets/images/transformations-assume.dot create mode 100644 docs/src/assets/images/transformations-assume.dot.png create mode 100644 docs/src/assets/images/transformations-getindex-with-dist.dot create mode 100644 docs/src/assets/images/transformations-getindex-with-dist.dot.png create mode 100644 docs/src/assets/images/transformations-getindex-without-dist.dot create mode 100644 docs/src/assets/images/transformations-getindex-without-dist.dot.png create mode 100644 docs/src/assets/images/transformations.dot create mode 100644 docs/src/assets/images/transformations.dot.png create mode 100644 docs/src/assets/images/transformations.dot.svg diff --git a/docs/src/assets/images/transformations-assume-without-istrans.dot b/docs/src/assets/images/transformations-assume-without-istrans.dot new file mode 100644 index 000000000..6eb6865c3 --- /dev/null +++ b/docs/src/assets/images/transformations-assume-without-istrans.dot @@ -0,0 +1,17 @@ +digraph { + # `assume` block + subgraph cluster_assume { + label = "assume"; + fontname = "Courier"; + + assume [shape=box, label=< assume(varinfo, @varname(x), Normal())>, fontname="Courier"]; + without_linking_assume [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_logabsdetjac [shape=box, label="x, logjac = with_logabsdet_jacobian(f, assume_internal(varinfo, varname, dist))", fontname="Courier"]; + return_assume [shape=box, label=< return x, logpdf(dist, x) - logjac, varinfo >, style=dashed, fontname="Courier"]; + + assume -> without_linking_assume; + without_linking_assume -> with_logabsdetjac; + with_logabsdetjac -> return_assume; + } +} + diff --git a/docs/src/assets/images/transformations-assume-without-istrans.dot.png b/docs/src/assets/images/transformations-assume-without-istrans.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..f58727ad268029520317959c23ff7e687387940b GIT binary patch literal 25856 zcmce;Wn5g%n)cff65JBp(!t#V1P|^IJh;2N(-17UL$Kg(!QBZi!KHEc#+~MEo;|bo zGjryd+3z`fPV-?ktA5m~wN}+#RsZX{3H~fAhJuKX2mk<3B*cXk0Du=i0f1*Y@Xujq zf?jJg!w#?XrNo2*k54}tE%~tkz&n71uz-?l>cOJRm$x%BuaCK_yv&|w!GA-k&=q0E z?wY+w{Y&qW*`T?(fzI!PFF&&%a~|8$j4X==Fm!vwf(j;1!(b78@WW z@6yuLzBo_ac>kareWN$a>9DnsROMn~j3-I`4AuewI_>oQA721p-r(#1afavlYZ9SD$}{Ly}G7 zQWna5`cr_CP9lR=o@^S8r`rVnnqt0e8lP9A!}bV;DBM%AE<(OsI{Ss3w8b|BU>dhe zL~tkGeK+QG|*Eh#$o(F+6` z#z8k64mGM&2YYUJD9QB)?P)w6NIUDsH+_bs#_2OS<*d$*^eSU_L530NhqV~alx>Ja7X# z?7(gFgFEWWKJ*g)wV*#;dt{!-XzGvX%*>77%_o^bEU4+S%;@&|C+9|_&$n+R5nMpM z*FU;I4L?+WBF5Z~j!luE^0-?~^e=zaV!n9>(0z#-0{}F?z2FKM)plBu95)EpE+3W* zj!avUh_1e=(~3lFlv_+Xthg$WdMrmJ&#`c9FhkgOdHmQQN6L0~4(E!qB39!h29X|a zXyO~1LI;u2JKC7bjt8!mX?D)lJ4)`X&!R1<+4-3<+n+316gF^aIS>4zXZ4tDtYSD# ziB+$ndz+yg0|H=SV zu$;DeLv@eCXmumopv}EuDszeVe!lwHdYU1_<`b(}{{*=k&Pwm>{2Y@%leq>#U`17x z5xm|cyLn*VhFKx-q^DG~sO@82i#Y?(UqStxWIZlbzkQ4t&-I!T~Y*-qcw^%p6{ilZgL$fsuGn|=>yg}>o&1A+eI zn2i0K+xrp{$b*=I@Pn29OmvFJq=D!!dNItIaH%pUKl`kIj2Q1soY-9N;Q|1%pGG&X z2rDM$c;R9Oo{os)g?&YIg@4q2u~-B@6}R3{!NUCOz!tWPiQJTk2O`#yt^2GFoE*MsAL8Hr$mwMzCjFsuH^p5Mt;%*oi3|oZ;y}6V&z`0Ez(rKOsVvSTnW@jjVy6B z*E(K>Bw=`Yw(QLh7_l?RaJ6u4ZVNRTS4Sgg`4qC53Jl#JM;z56IvsBLLdwCMZILYq zXhx*|PH5V8dmG7WtGw!soK&0U+pqW*17+2#twv3r*;nN}gAJHppUpcb1;oYSfn*dE zc|B`?ME8P5n)BlC|El=?|9JxV&lSj+9|8d8k_rmr{jrp#L~ww1q%4b>GGb!l>4@hU zq8Y$IoM~7|S7&F*ABFE}XlOosn4FgfOu~QYKK_~=r_TKn;ImEK#WaojEaTPcKRXls z=Z5^>Ol9bqltC11${+CCI0hy6XL=&ae$lu;Mhz+7kk`jOoC)6ZSWuJEb@i!_@Z@tC zcz$M6E-StdO}{PL3$zr^WJ8}?XY*4=?&?bh*;2Ia1kIPO)|M=|EMWCj&EU zdew9=FGOHGSw>9f@%*a&@z?po&gd1pW~2w*9z{036S>XZQi*Qm!#X4C_N1_(Kf2&K z$4u$R?^*_oHuCZL$sDl>;%x0_@F>haeZPmamgzWc_#G95g^I&LBEOS+Q!m?_j+DdS zZO@MQC0XwGd&JnI#3?LJF>*-`;{(wcb`N&#kVbl`c`2GF9$u4b6!oqmV$Db zmrA%47i()V%L=>^<7lqh8>|Udew=K_ogV4f>PXOW=J6C=t`hg+Z_3J!!^dphr`T4T zHZ5{vRa-B7<;n+JexTzdV^(=usHdrtJ|`+6lAIbV5sN$xwz^y5T~Jo}_>H&0j+y0F zFLc!lyLG-^NGn&J!mePQ%ib|jcXyElm3Lf?FT>i%ql$Bn3W$qcRn6IPdhNdV*iD*Lqy`peiPQv_vt+K-qA_oecMD^MrbEvn7{iHd zregFRNV!mkhp#4BX^;0osjzenswCUwCuTBR9Jg#T1*6#_p3#Ue8JY+RNT8ie+0`983n@e*>VGfm`nc1H97#hKXd;@5sW zS;*UO$i2dG@jRS9dgQ+M$Xi{Cr)6ymKP0)iJ0>^LhYsfZ<6=HWe8=Q9R~6k#OVOc& z8njmrmwP&!b~Kb=Hg)aLoJ*t&iv|f(TB==a`=IGlLSzRAn(JIwBlx1ZdmewpXvZ-o zgVu*~Be9PN-{MU;++M^PVjA6W!XKfj+pUyOaRtQMV-?H|?Kt0Fdnf?IKRF`tW5gzVxnRp85i z;N6EI*jL)c-%PwWd%s{pbj>HmkkNMG@a^%Me*&?bZhDFD>ds{yHMQR3xcwl)T%f#N z%;oMf9nJD}m}8X*eCGUM5F;J8J;a#Z&bfG|3RSAACJA%&1b6v!e#e z*X!dbwJ5hWQLO&xUZ*i6P%fp($2|vVLxSy@=)iD=7r%KiU2J}d#df~%V~XWF|EL51 zNAFoh%Hv1;IauGy+^yS&I_^uOlKV#m6@uAQtBVP)RlE}5HAH#>2A8)tw-xQZZ59qv zZg8N3>+m;seZT05w3bhG;im{A`95qbCDi^b_z;Zn_;9T_4>Gv*oK;bCXU~mk)F*t- zw|s4+d1baP=3@P@y^&zwNp9`wu^{K7ac?tKrJ)JFT4=?%@zLZz zKEx&;eXdjF4Mab>TwRa$s00VSgr00>ls8GqYW6XG9>B;m>wP8jeSvgaG1Pu^QI01{ z+M>?tcjiy5*vZRtLhjSe8V5f#dSu@7;V7randX2GSI0?lB`p0Ll`e;%m9aYTKtnmv zEyG9S%H_{;a}x_=Iida4V(oe5f~a1cD^=F03Jm-0H(~iz-p$1sTup(?-h7t(rgD@F zXZp?$0qJO9_W6lmV;!kU1;NU4&D%>QY{j~c4VTyV%>3tlbwVmFOO7Y}^kv4aOab=|Q8dQMUZ7 za;hIg$^L)X9?6L{^4(Y?_STlD)-8DGNrWCpr)|kRdUg5Uc)f;=&~qFP5v^5xnTVY- z%o1{8IRLt%cx$Ihl_HNDnfzN)j`8w8odkqnLupXm^I=jKxcw&g;ggyPBbo za%ZE5bLpbyaG*2rqdPaNf7ET-@)u~=W_C;BkLxqj>Bn&NV@%KboJm53-b_BJZwzc( zpWUqmC0DLCL=<-A2yuYI(^+8as zwM_s%`(e{%|47&dC0D60HksnBaIAntAqSEs!~VqC*)s-7L3rQ=KF5#A@92Vvd{-s} zZ0Q&~>OnyN*|vqY-3ZoghFrcebQ6swWpqJ!1Mu&f@NnTht^5kI6P0XtD@$?(#zfJW zoL)6`Nivi5meK;c_Zlm<3-5vY#jDYUd@CMFVKp}`48bo~QJecC+PfT} z?x>GVti1<8HkxPJgCP|6 zs3~3S;Sk&x_eUdH56ioyhIJKJ*et~j+l0C9tl_+JO7|wqUU-?Ut@=@h;v%Ke_H?X5 zr)6R9GlbaNO5K&i0_QhJMv}pp#8aFJD@{WYg3qq4(Zl8D)(AuwwaPX&mHd%_k7=_o zkb4{I050Cb6r{=(d1RLDrw&|;E*lu*a+!3IuBQtZU507^7J5%9NFe2)*v-hqD8p z@2ynoJDj%eH-O2+C%d2s1hqq!Nar&z&Lb4 zvkp)g4}EM*3`#|G<9mPf9-!`0U}U6Aat%a#;_A{{=TbfRy=44X^yI`8lM#znshFDx zkCyeXbF}Pa{_J$nLTV#LGb;k3ObQf|bytOl51z0^#nbEJa5aJdYZSog{AEUUE4?|^ zR(JQheQR%$)#AOMXR_msF6iXVk85mQ+`Nn_s?N>rL_RqdIw^P{-5_S?Pw9?GTtQ?) zv4RfW41WcoXA$ckC6RPw6Ln5S=>ycs^$J9@3s*k$xxW-Uuk4?#(NpCjpI1z&T`&fR z4xx5ABLz+`-e-8yuzDSAu@g zNs6OSqE#SF$WUg219vc4jy=ACVU6rKbn}e3&z*L0>r`UYVxf|86Y;zIVN+PJ!F@Q~ z;!$ov&AhxJq_s_)46V;>S8*0#yEl9-S zZ_Y#cm(B62ce0n4cJOxTF@FAyfu!A`oy+waLYGY#&EQ5C#d%v&3T8nbrL8UWopU<~ zvU{s)k(T zawaOE!^wtT2!}V9Ls2ld6EMARalc8$!~HvYXCI*zSs-OHouoN8m-`$(6w0W$JRlF_ zfVcaPM)=>fR=?IBJ*aE)cpZ)eTXJmI=e#bNVA5&oybRi+_8yi@4g~w-Q zLb})O^PC{zK=1maS^h#!E-&rdwCgwE#8F&6(~*i8*6vE|Rx70iKZ_aK=I9_hck=fI z&LbTt`_rY%KVYmKp4$DmEsEz$db2^>5(Alm#s)zs>O)FW`Je>UVyYXVZDDIt>+Xt* zcQl!RP5Kv87>#3r5sbeC42{kV z10R>riWwjht})T0c#bz$5Plx#`_t6)a!ncsy`YN5HB62cqKfNu(KjmeuzC~s`F3OQ z9-ODl+FR>jb$O^u&R$aqauC!ez+(yAyed9c{QWb#GMVS~&zhFz)bB&Q z{12YMmQD-mFtk-co}{-w0X{n!z*ql<&6uex6kp)a4(1fi55LGyJN3w)yvLp*OV{Ri zv4p4`p&}M8w0h?=w`mQly5KSoiP6|Ta~eHXT@=5~jP!7yABbbH5+<28o7a@C;<$l&xNeCzbwg)Z&o0Y7&&}HA7Z#WE zFU=LCfOQwHW((!2rXl|R_B$E91oUj zS(nK7E0Jbm=Zl_FTG(aQ(?5g!fJdWJK~=d#4?LP7DI|qD+pjLfmx8Tc>x71(ey_Fa zJ`FwjA>*!myAe*q)I~!$Uz>6nV|6)fkxoB{^?^69x7w;oa5DzugkrFgyeeN^;f>4@>MhtWhb_zS^dR(NgHwy@ejE1JR2K6 zY+l-u;>6H!%{+U{6@qhUNSDW!BhA?dwZ+`3L`UIq*Mk%<=~U9#dN$&%Qg9syFq+g~ zzIu1=W?Rk=%?qd5$UO0c^aa}R;qS}>>V$@uwN;twTCGLe{OJlD0&^cM>U5<;t&eEp z{pDxp+s^G11As@*kMKYE;;D2LxFbW(41@>yv@dIvF_Scp_SCsE#O@e->a0~SyKY1F zLyQTp-`!`gS9+h#j2zQ@ehJQ5{jQ@D=FryqaqUnhx0U+~$Y>_LE-9f)5UjLttm2gz zPkw=SmIo>fH3>+NnL;4pqj?;yElvGBF(@H_QiJCy^qoKU9trRbgYDN(y{yxs1X&bb&{)M%xNa})`Y7dvTnSM~{7>x( zS#yU7IScJgxiBl9mbUA6w(hj{&da-xca9dL)Xcsl51iYMwcokd%_rlJPAeH}qdOc- zE(;C@ZuTIRI1-yM(n$s?de$ABDM$j5fCs`p{!!n*f5#7f-ENtcoehJ)(nYoaKmmFe zlf1lqW@hFz>PsITtXt)wf3;5ePt#!qi`Rh9Rj|gD5@=!WKjuwUC* zVOY8Or=ZZ&=ZS7dex>;(h6ezs_*?vMNv!{L)1QQy{}qwE^O zp-XjbA}NUo%yHVEhI@jwc0;Cw7MyPl_SGou1{__F|Ydj=(xuQB^n}D zGRaY;Z5|Cg^{YmmcK^1syt7ZDNR z>+360H!VRH1AVyThHa=gXkdbo;L~-8FfcG+Me>rU6x=Yu@o!!L()aI62+#mNrcdJg z&j_GUAl2KDuaAj?n*ss?)8VfHpQ@h(ye~oOrNms01=19;{f0t8s5KQ870$u{!21B0 zM&E8O8it!WvFpgS_zi=!ga`@X<0?@~1mJ;j_NN}(jx?gnR4`>OX{MrrElT)FrVD5J zlhJQ%3^Vl=;`N(7xK z>L@Qpp-@*-XcIlX`R}h~)b3>aQir!OStAsQ@b#G@Qw_M$ zj4MkiArb>@pCz4C+jgk143O{N3Rn<3%)fv%w)yzU?cL2`5#>%s9cytk)c95Sn`CX6 zZk5?2q}GveQDmO;EbjZJ#>8GTKE{r2DS8nksgI0@f5V%!OT!fPv6^K` z_we&dfUgjmQYfEwLL_X>G5xUUcXNJ{+%fQVUyR-jA>$|Rf1^empUo|RYKIZ7uF5R( zm1ZZc4>%m>_mCoyK)H+TowD<$_mP90>K{2>FMoK@J!l`?X)RbD^4#YiHZnQ}3N|^U zvfiEF&Y!Bz?EahUqL=r6|IQ8?o;GQ|oySXA)>cM+&2|NO4ub1q3bVRaK^g0gu%q)N zFWmMBauO-;%nY06iaN6cmRoDNc+Ww~cE{c4B4B zLeWO|u2tpd(%%Te>!tQUGcnNff8u@slOEuSl#hbEii5+b)kT?CWPYmkX_hN{?(v$Os7+IDi)L@Nq^{*_hTayvI_4x`O792d+>Q zj3A!?mK(p#uH{*gv}yMV+Lo@EeENdgBo?r(cbn@nO!ffVj75p4yE|OBKSiV_h$`?k5Ol%Ac`2abbb^l?+#9vRV4p>Z9e?M@8aDG{v=5W(9%;jx zWV5@kX{e3d@h&SX>sv4{8c$o%?{fV>+#-?r)ic+p5m;Z6pK3=)q~WSiu|e)b(XA`~ zB&V(DN#Vyy!nPB`@zHDXAG5}ipg74y@$nI#z7cZ_#YGj zthm=daN&W%^5u%x2gd;F*>Id0(HSw=XSX?0$}7k@6!ovu zdsAo4qyC}7sve)nDcS)oV#iA3_(g|D)v1zdaS=>-rKyz8ay@;gi`BE=5x1%%vH+qr zVQeU=0GUY+Ef>1YQwCGEq188@ITj{qw7N6#RZ8PSg_nkUrHVo$n=F3}zi3BhRtkat z4)iWMxEaC^>&=)W)*D4PtD+XG8iY!EtZ2C;I1$#K=$&}VEv-s~lZOo#@3SK2o22$0 zjw=i-J{FdLEj^nirq9(pMQg#Tyi9*6<)D@xj*v%{CU-CktDLE9k^_z8-cPD_Q29+8 z#%Y!v6lRGVI~KekwRqg!jA5m|rA@yX$l)X3dK=_230dx zr(G7`{+dV)P|VbRRTl9W!wCu-Q%Q95bL=zOl9zAA^tKbrs*mxHD_~xTX7@od-pnYF?isqB2F6lL!kCyG&T+2N*?3)ymu&AV_|82GP&@U*8~u>QWw&dRN$VDO(f zZv1d77BksWeZ15K&UUTpFuDI{$xx_4YK$VXe;n=s*0wMIE3Ny^jH8K2!?={>iTx5= zpbohPkWM_iFyU54?c>ePY`M0zQ-KRw8#{e-N)6Hu!S5mY_e?CUAw|k-0W^y^oszOC zDJB_V3WZcUK+Ei?XlNB>kM%+dC)KR)#$yfh?JG0r>Nia#QIAH_9?$1#zP7ZR(C3X9RoAqu(g{I@o@RyntC?|B6Q#Xavt}YwUT1MylE; z<=??1-QgMN!=L-g&lgh?1@2?SdI(e@QD-YKZs4^2jo)ARy?&1C<*b#W(+WeemU`GZ0{m+fJ=JEcS_~Ch9e}9Qw`KHZ| z*TjAxnJ?^NK9K;nY{DR&$F3F4JCHGe>ng@qmXYo^oKmBBiikaX9XYEyM#wfv_(L zMz}>BxC5yIIX{+Ulu8z(YS0E3G)1TrYoN*^g#ol9FG!N3qoQOFUkzW=WIgTu!^r(N zo;a((P2Eex+|=LA2dOk|Y6X_8Qbo43((o~_v+XTNa4)APi$;SOLIF=+8Y;&rSs9iYgnLOl|#OQ@G>z;UfnwV?)6d2 zh-y#%v812~JQ^GD-TD}H7UOa-0}e2>4D~b^%LK8WIdS%bN|QgXCW2ENEgp$S=T$)x zsTSNUjSd>qX>wC>Tv}W&1JO#-&cqCNg7za|nr8OIcJ6X8e}&gg*s9+aA?x5-H@qwD!%xyc{uXl)G< z@S*brCw1p6B+Jv-oKimUFjV;NzXfjtnu#yo{_w{Qltl80_>kttP+4_& zciSf_EqO5M^XihNu*mtW-7Nop388caT<~^LEYgxj0?au#qt?R|)@^m~J@G?XbJsYT zxo<%J$?E6OduhFSlo2&>xbK302A=YC3c~2}e5oC5<{RQZ0P}I0v34&PvCzDF?@kDDuMwC`l?w-wuO*1|a8%5--tBd$1&;M$RZ%WSNcHy!T# zu1DPA-N!~tRe5uXHx!U+Q?(Uq_MVInT(NA3}-rm(} z_XvxX^!D0)swRHWQZw1n-DzMn$a&aQ@n}pgzCu%^;!j|YsG{huFY48~go<|QGmP8` z-9TYApWo7Z3~4Pnq%l1P%Lj@zctXWb{Ry4xzBrl+e57-*baP<}Vl;^#U#c3M-Q>nb zb1VstCfB{*FF&|FW1q19VoJb+4;b=i-p#s>ib59@mzI_e|Mg-NJx~v}27?ilQX+T& zZghiz<=Kq$MMP_Z^^U{+;qX*;+sED+o23V!1!HfRt=Dw&tw}#pOu>EZg!;vGMMq4r zM~??$+DvFf1S%2|>x1YgR|e%wonJ7YX1l}iYW!}jXZn4sJ=R+N68O1HJ z&nrg2KD94V5K>IXwc@WAe}?U%S|Do9Q>eZ&vHQ7%MDG)IImZ=b@U=aj3UPD4bA@>&+6Vu>E2LMhw|0SmU|J784 zzq{52KLuBrvAb z_XnBzz`;>hrqxtwSxCw>Sh?8fkoE>C>oBhyhVxJwX%u{BbL5Tmq8R1 z6dV{reSRdl*Xl z1D3IoiiZ7$5v?@q?r#{=5paM{%{g0M4LdugXRYsG_Tiy_5Uq%G0H8DJ$|7uN0X--n zAn6M38K7Htqc8fYSFgC8N^^Ao0nI%xr~p2n#3Bgwa4FusYoGUpfm{+8Asgme$Z7q{=kYBkF&+%M znxB||R8arPnyh5^Y|srzpkD|vQFV+{5bAVU++b_ts)eOos0+=PKhXbV*q47<>NaGC z*1T`M8&h$-^fq`q)0FAfLAN~+1tKqe3*DnXEpIk@MK{Hhci)t0? z)p4k3HaAtCUvPXdaEgkdZnQ1A-~2wmu}*6exw}2HuqZ%XXJ@^`y>bY-T%xZ+AFi*5 zW=AvDo!TlDA1o%j;Bj3fPM9SWAS~|g%H*DoEZ-}344C#bs#=lDlGYi$$y($bN%Crp zH)3Rkxr$fHb#VhLP?n2;$+OH60Z5X(%iJZj56O{_x8A8B1In8D2D8!N{c?+;CeEqF zl&{X%z(#IvV`S4+{o=CCIiTzGv_@|2`~S)soxd@q9{5@er+KLhoXZp_HMhxUxiP-3 zB7(X{B;|OT-&w1)X{p~$d14uP`XVwcGrCn|{7>#{`H}F+oS(xVS~=XD6WS}c z(G;2t;$9RVoUOMsgIqib4>Xp?S|eQAXUX#?3NyZ9xO%QAOZ1yX%2~aScbDT=-ot7D z;oQ=ak8~NZ6#T{wvodzN3TU?jnqI5Mi(o1*y8ry$x7O>@v&uV;&)fOkUNWZE-RP5n zJ2^#TrH7a+te2RF{2>Uk5A{9&BNF{k9n)5uoseIfwUBR4P>@7%nL%@iEtnT29^Q}+ z2~7MuC9`VK*|mnAhae5cNx~V-xE;-m**nebX9#Y}wuRFqYDm+D*f)BzF~G<+g0Wv+5ut`IlB z0%Yo{{m!J<)MVOt=QixLZJgM1Mks6NY@IhkVVJ}qE#!UN)h%EDXZ9{ExOav8C!f9T zQvV$U9)&)EK+T&LeWmatnbx#i?l-&i@$6{$^Xh7AMmfZ*@>hsw9k~$PZ6I|OW4zxC zpEv$StN}^2a$;DAfw*mA3^T4m?W4{a%T#+Ge)wFBK6--rGxJu{n@czlMR;OrR2q5p z1`GR3FiS{#`cWe9u@ZZI4m}?at*#|vXNvh!$eKpRJ2XBR1_>&eV;=COmM%25sHeAB z-}v&A1UE?zlDJ3~0G_i~=^()?m(T;7dD@&F&;gAYsl&ZbTM2OA$G z>P}ky&Y<9M3{7l=Gdj1W(QwG$%8e1!uv&F=iTp^CBKF^!XEDX#S21UXv~Brjrd>J% z&!{=)Q$0+)-8sE`3l$*lxV_93_>$CY3)+({S&g43_uuk)O!?AEM6$WJu+x(BIlEJe zjBhPF^U}Dci;`G{61JVDdnWokmU8|$W@6O2BO~`-mV-FC)bwqBDN1~P|2EknpCQ3R z(fs=ATnBA-Yq4os4_^KcUKx--UMW=>o=-fZAZd2lqf0LTs?>X4YahbkrKYlmW%FM; zhPT`~EF)U_>#AUDVq1e5>KQfA($H^H63T-|b01on5a>?R1l+{)Mpm!^L_QI!LnJ9a zvtvJjIRb&&*{6%jp^5(`GPFZJw>x+ZDXH(M#%*wg)J$roF>r6LB_#2hFRJ0(3`>+A zD-o!!KZTu_hU2dN2uo`BN{eXcYaVSeYBn;4eQFazv0m&SUc>IC zWXmeYN^w4L*YZ}2W88`kM`dUZY~3UhNQ)`SvDJF@w28Lt3~8s+uqd+uPU)Tki^t?~ zIImW|E!9%=eBL!yvC)Bb#?C;H(-#VP9yj!#h)7{{>#n08{o9DcNc5z+$@^1oTqT@N z>=J3gJWFtNojI3tJgnVu3In`+ETuIHdpGtvB=M6DhSHUDOW5(Bcki4IeO(sSrtM~X zf_~ZHouSH^)9G1iYo!ZNUzs{ZB}dnOKT$o=G5!@%I+)<;qTHMe4s9G9l&h6zpyj)b z{ia>I&s}8n3zB36Bb-I~aSFGt>lV&o-CmlOfk`GkaN{};&3@9;yT$wNk{5~mR+O=m z>9(JRV3iN+)^vN_LlwY^_SS{Xl|otHj~0d!6h!AN<`(loegDo{m6$lJgolr0e*7(v z7%8>1GR`BKi$keD4~v7n?F)5-C!&`W)b8XMls{?==9Ur`=;xI3g}}D((-F+%`h8)j zPP&GN{4JWNkPB75vv>(F<50QXLbP8tuCx>*tIA~jy2+FCRA;BjBor&wDq^MmU^IH*XM>IUD#ft}=-7{Y!5f zt7zywdRCkI_J^1vkRVb;NW$l{DR;9_q|WQhErmz~`r;LeJhi^61xe(DF*{NEcrg zbN#?s^}0Y7)Msbxo}SGUC0}&A(2ovIM=`WD-7ySa2CfEcaQu@^qci6DTkIq#d8Z)uAg-4aG zL{q{Hy{_wY6+uw*Au+Hp=|b-;u(HDK85v1{u}*LSUUE!+ZbEUtIU$KOqQV~Ahvv}m zU1oS%&Szn9gkUPeoX3OU3W6(RR7B@@Hz2F9R)J*D$1KHF`9Xi!yiZZSM_3Gp`nXr| zIjcwwZJJoE*@O@{Wb03zr$h~(>DJ&D@0HitJj*92~CqTK9^G*;;)tYvi&OWSDOmII8MH6tdws<;S<*lSC zKik?x{p+0KbZat2XPv}(?|-BYmr)5cdY&zr%TqhNwVJwuBekT0M^^aMPd!P4AjF0r zj)mGTSo^g+>sl&851zWFU(?%rH_t~;4JJOSC7|c&7KQ5J!5qjY9mypSCd-w49D=?+ViyALiMokwOw5q1<{e(-!Qqkt)jW5 z4dSdCECUnSW%P#(wKuz@`9i}ZGUnAQ;$M(Nym*bf_Mdfm-i?kE6>SDRbE!J#V>TQp z{^--CzCm$@Iv+i1u1ymaSzZ<9yS%}&;4nPwvQK<(tG2I0--=Q8O3^AXYZHtUaLuZg z_AVgC-F4ca;q_I~asCv!D=keaESSM^)7iRsn;$a@22LOOJS1rAt^H?m;FMG@Dd75l zVosk|l_HR<&M_BJsl#5(Bp%_|GSC9jGh3ZXN>EE7se{kloYcL6Thy3j{Lc1TH8K8; z@`F$moG3N~PmF0vDC(zT%qq0y^()a`?KWBRa5%{G1l@RLhv1Z@h%fhY*$Bl6ZEUB5 zPU(*QVcy*OWAXNi$^U@Shb$JgwTX#uVjH&e(;0ITi(a_(dbena_tf@rU=!VPu<9hn z-p>>TzwvGuE^NAl=*ht=7H`vPDF^NkE+tj2cKY#XE&0{QL%ize{O>4nxp8pF`}wZ- zPYqnscTkeWI4&89xjk1eVr+gK-l39g2trET4EoNUC~ZtkFB%snVZJ$0+1ry4pY1Z< zmbOg6h3xnjaA7pH)~X;I%r+NjUM$M%*0s%Xj-6Zr?$#ww_T>tTD-{YaQ&PLK!@ivn zD>WBBWksPwk>NubAa^Lb;J2;5W{$Rnlqi^!xY5fL3Qj*7js36(p3#8imZ+sHAL22a zTy$J}sNK9;jLoS6v!_e1*QXw6O2X?A>Kt9gjN3ZR`6q$bNX(oX`c1rv`FUw->az%+ zcfFn-xglja?{eyeN(_(fo{dAg(`Q>fdyWMhBH~k73mTMCGGP%hAWB+=U)=Za!a$0% zffBVck0`<%;yM<&?lXx8V{P{H)moc2-qi2eGkndMZ#dn4S<&#wf3+BYPD6i}jHck) zuk0S?KH%EVEL$%8a%zOLUT?AS0<8G<83s^fc~P&YvRptgh`(^Ff3Y!`rHe+IYQBhA z54I|KeoXuo5uS_O%%*#Je9+TVLv{N5F@r8fe2o$o%b<*tTVgAUt*#X}2wTa56qMl#06u3i`?a;jw&A6XuS&%H6UqTa_J;zqmeHLc!_$ZZAOx^ zUOS~FA8MQmXpM`mVmw2HDf(%u&g59j>2VELi#_8pBn{#O5g~+ApWjI+<)y|}OHbup! z%pv?!%~-#h0Era_mcfka`i9yztsLF08!}EYfe@0xe7L$_;>~$!LyVqkW>t8g-GY_m z5<`$d-)YY~L(R7Z=eQ#2J?1c7`Ko0d7(G7+{(0%Anx7D+ z`~|s(eZumtv#)-I_btEjVyuS@#Z!Epj#j=9K1s8?mOE+|`~F-|R^lcH5J{2vf$(fF#O`PYN36M}aD(G&_fLwz)D`ldY_Y~+ zk}0h%H($<6ewJx7u-hjMRQTHbzN1BjBuz+Y)%!G_Q8q#EtxG^}DC5U71rjR;>%56G z-iHYu6Mek&+J-FQ;!REut57I6R3C5{JuE@|%oCvof5;+lER0dnOyx%f3V%9XcvHLf z46wy&XdGyZiS!buS;2#K+uNrV1i#cl{XC6?=ZidW#8x5BF!{ES%}`*7Bm2HZ)6$Ub z(oE~$25;7JT8&%RWb2YFO>RpNk7JN8Kp=fufSx$NH zn?--@dWWl8IM`Pjd4p5h&gvZYbz9&<^5~b3FX?Zj7^=hIN4EI+^P5#!BND_jhR?n@ zdVya){>i3XGtjg$L=S1>+6>Z4JTlrpZ@6L<6_eGnOR2xwTUWeAK~D5AnK=ztJs%N#$78wnjhTgabmpd5}>D>H^yJG)DU3!hgBo*tT z;|L~n+nh@o<<(0X5*kx^x4+_{eHsWk7_M}FH;OZevV?;siP}y2xR!qy@1(8aQeNrz z4DuYyBf9nb`kX4QVLyr54Vi1}YJA>+edD@JsU7Ye0*;lbhzq;wqm_rpY)Hwjh7lf(bFgPRx5e0=! z$n;650pyB^i3L4?JY0xLCR!ft?mdkArLlqVG?+z2=7rPK{@-$os!>ER1?Z-hTjNGiPSc+H=mHbLM&WJi%zviqlquF~VLgPIDmQe%4)7 z5K;L1)pWB>yEhC#@1+`lVDeX`7yl$2=eYg@eAdzxbG-f|A@#q!ROsa&FDWT`VU-oI z!Q#Lw&CUC+NawSR$HcuTzq3Huhf`N;>V-x0$W$)b~fuji=9w2WI0I{S+L()?^Q}{*&1KkKrkd;oqauEwB*0~sIC2bG{wG4*~btqW44b_DH zRXT77bDGazyQ5MzI)}0<8a~f;zv_rWPnW{aa#AG#paAP}?g0=?cQ*qgqX;dKPKCtK zPNz?B9V^E*`MAXiVQ*Jy^$A`NONcAKEadCRO=<6&eU!4Hh zrDex=ku0E~9pn6Kboz29B%!{&yV=3F9 zz^xZPJ|;~!==4SYG%AD;Tdu}**i~;Wn47O5f(Nv)B_wV0<{OYaEO`|3j6)k)N zBvnA<`gIkuD&4P(25*{slNfKl)iZs?k_+&Fk5d5w9Zev@1WCszc{&xjnaKZ+ET)+F z`1sgZ<0htJwm$)PQ&dPu=-xeJ@~s@}f5qWjyC*_Ms;X~>On~^Q=Vwuq1omGbeU}ZJ z042K|-+$Ex0-9-aeVvh!(cu5tRzbvu%y3W&M9tw}EQV4l0TrT%VEga4HG1Uk-x{Uz z;5&3!!z~VsrJSXc@*($yWcXL}ARgS*{NL+*oXOgCjGO87Dfiy8Ic7GbEb8g&9=wkK zEN8}S9c*A~MTtc~@{?x^HuaQGDoh@v* zb!2Triugb30ydUgrfYi+)$3Yh#G7gB&cw9UFuxueD$EZmft{$C1W)vVp(|Hk8oiG@ z%NOy&^#EQC>L_Iv_CV%6J@4OrnbduPM-p%v=RF|Ah3Kwv2T2#iB@cC6UO}VlEt2l( zw8;wOKXm{jxH3mnQ<0Gwbt^IKhm%3)n9GdX*c8;Bwd;Hi*wlwd?Q|HHa_#NBlApgC zjJVlygUXTUaeMjCS=%Am8KJ?L@60Xchl|ce zQ3hoP{o|W6>mSP1Uv4ZH4C_#TCsOn55Ec&)euJ#q_>X@8zq~+kWD>5EgXT2;u9!-> zv`Ts(dzO_DzqB^cQwB|R?Ox#{Prlb04qG`O)T9c#XL5YDj4gmchhPSn`;_LxhXf1u zo5b<^<*y&uD7fPyp&3Wo`$42VCF*vW*&PncvM~?b3@Y*vGIwM>>MM9q_MKkO)W@g9 zxM+ZL^FsRorcn`F!`}=;m+Tjv(JklNHViMV6zB0Vj||@0#AWD3Zg0R!!ttZK{fobU zEnC{)9hbQAk0i8`I^7mNdha_vl(nC`G^%);+k?$i(^GGKVXL&&D@%Er`kAGzd1w1p zIx@y)(*~9X4c`Aspz4jjkiB+G(X^FR2Ej}Qi@Pz_EN>Zyg$rmUS!7>vx#2OO5?`L> ztwft}6zTM`s+x{HV>OeC97|#ToO>bV`0>xAn^W&Dsp?u@bN*11dTZWF&0UEX6gK%S z)m&z+)Y%(aVORvl$`-lD*18D@^X#pMhc0tsH(tE-h2y@#ySi6>ZG8GQ8N9Z-g}s{& z^6a1-pku1|Y zCAGCO#*`yW*X`)Kh2n}C`=kM|1m0!62py_0zleLCNgWbD=1URxt4VRjH9kZwAnP-}#&_a(%hAf!b}c!lgI+rMa1 z+pIL{n32J`q&%Bi2G%}1;A9t;zHu-|oJP3wVA}xX&~PUh*u?if=q|0>3pv+13KHqY zQ_ma8&?LXiN2Ejz)UZrX@)qg^b_$6SH?4At1R}n2gLVokZMdHFtnjS`?ka*Vt8c(; z{EEmH4K69d%&DQQ(CPFBg>CDEX9K|_1+3L>dhMPtxVMYR)BS!8HaEqEESJsZ!`bcu z3v|fQBrEf{;yZhdk&rZOtpCd@5ry3}%(9L0?zcjl3mTo`OJE&9lEzk{bvj!jA|y;v zx@=}-1{o&L6EsvoqUvRd9Uta@Z?IV~vZ49ml0Ll;T|NhvHK)PMH;^5*Hc!o%OPtjS zg2bo1g#C(uB2mDdIdo*6GUEc$_r>UB9Ftv)#ZOuam;%Q|uuRV)z~O3&9PPT!)^+Ma zkt?FVzHGljNu4OLMWQ^1N#+uWG@2H?!YDAk&0=S5P+N6UDfk8SNsTeeckgGVau|C} z2z2*UIHwXNqsDf-`Ukt}BhN3tzYh)vY3l8`6|w2YW!osD{I<3TgTrd3|Crp~v9P6= z1}5_a)(aV__*0_CivAQQE8)75;#xrnA@ymjoI;Vt54I4_X~c~3aVPDgyOO;9ud0-i zWn~M+FBL(91San50aNYFSexOIMVWx%9!{6J*{`1h!Z|n~tNV6K%#1R85n@VqlOrJ_ zo&#^i2!89ZeRL$qYl0WdZLcii3Oew5QnAjdD^yq@m^FT<+Dn>}7HRLq)6b$hjz;s! z&giEfZt_0~v(}O(MiaMnz75tRy|!}PsN{<=qJ3j~Y1|nC@$_y(&%tP?f;B6IHfEUsLI$^NkHp>6?j1Ec91G(lkYA?u zBhs7r^9E`}`x;WW?lIZ_YHRPmM2h_i$5XTqX7RT>{a*x@8h|oM-2^p9O^-p9=@Bel**LW+pKbVvztDqd$#zjcf-A?qlmZRLUG=ho#o9~q0o*bZd|HlHBVf$;~_MJBtgVFtt zAZgmBjvfr$LJO{uTb@hH+m7%DiF z%0{ckm$&&|UAvPU!+-P7QT&vdzBkgxsU1rDofu+jsa4;4JEZT3MCF$W4kM(`XyD1) zoD-TUolZ3(bf&}{5kQt9IdA{6_Fh<-HW=gVx;EC ztgX<0bht6KgUC-_jqMNf+I2uS9;1XbGW8DGIOQn82Wik_FwE}nTefg6VZafo(Em9q z(B&S_1MSMGL!?o|d3afhrOlzF!AB1)+v%o9O+`f7F@ZlBd#<$dVrNeGWdL543prey zFz+eY%!pX(t#ERxyZbnhC_zOhc$m$%PK%N`!m4DCt(nWt%q{U9+WJE9JBCmADwA4B zPF+$^qIIm!bBD7XyV$v(K8i2^1sBY?Ku|fwh6~#$dR__4-Q~miz{>Dj8jJZzLPB&1k<&;kK>Z( zeUhqY`oAI@@Z#uz!RzcN`JwTcH!$G7nUdQ(I>fA+>grH*HKM6E;Zka`n(qo#RW?gC zkenVOFU=}y#9mTo9q9RUPY~WS`3u)9D;OMO1s%If%#yiKdtyY6nn$C}x1ID%AB%VP z%JUUD6a*tr}|lXGn|BdU9VvWmaS zxdFqk;S|mSb$#r*u6Z=@&J&G0>n&Jhn$7F=i%2fM=%f6R-{#aoROr@^kIFNwMxt_+ z{2y(59wg)T8>@q( zl))}ceZG&sEW~<#n!px2!R^j6kM4YKM_kia_i2;xYsakEuKYdXf{N9`0Ii3sOvfF86k(t zbLe$qo%PITXDG;$0Via!*)01gFw@`?kTHW}ABX<#>?2&DVBj5EcvRKRD zZKGuFmC{&@(vBk1zb^>Pj^O>44t+~VM(WAyW>4HfPcjaPHE;u0!Zk6AsZ zjI{Zw(|5nk0omI2@^eCln-aEhS6L(UXnQC#EZy2~N_IJXRd4nnCx6gcKnq?{+!g%xU}Eh{l-r2)Doav8LJjjd6v=)70o5&eZb8K&Qj$E9*`3hi{b%#Gr|C z@Cl$&k8V%NBYr|`dmA>9(ECKTX3!Gyu1xBf)kNc2K9j{YsGAd|It>!{w3!Ai72Mt+ z?9&b~Qiru~A|6d`tkXiO$=-&6fkR|KBFt+DeQ!BAQ+{-msVDDP8^c+l>2!g!cB9*; zR*-a*)u^VSZY}aae^7a9iQNfLYK7rE2z;L(SHkf+~9#unem1Qz` zZlpwZy-8`{qmSbx_E*crb5uK0}ppR%rF?FqXr8~Qu^r7DJ}^z`z+ zqDW!2ew|ajRfSf{EiMN+KjAP^#)5a4sGk0dB*53!#Y6&7@V zm+e7;d`*6@MARu$RIS50v+~i`#azDYqW2Hy?V=6yH<<9?Z)z3gY$7L5T~vLjp&?&< zmx6u=yg3^xo70Gyhz}kySl@Ft6OR6sv1txa|*VQv7U$QPIG?nN@*FM#?cka9E+Hs zYmo_)2bH3~f@@U^mV$Rnmip`&2J9in={_s*q*VwqaAH7D%TugtwJ~|STS0Ory^(;l z83GZ6MZaB-ePEC_981PWVaFCZPWy&UB-ENL)Ava@9A9kkjSUs*l08g9jJctXGNQu_ zGu?GQP4$3{3TUN@!q=}L-YaMaky=?SR~7ajz7_4;7*LfP+6#w_4lUmLigcYsiuL;}uoZNdCoi6p z=vT_dlxchkAG@Sb_(d1H5YCl%oi@D6q*TK+1ukG=XBUZ1(|Yyhg&}tWZ%?C@a}neR zhi~b;^(eT^4_a!F=cx-c&Od1=C>;@^V3Elj^ySW^&Y> zDHjd^CNz`9dndaJO0Bv8P~S_0V{g5dMt7%?aY6+pph=#b338P^8|*|toS(6nLNS%Z ztZGt1)1*b+ksfMce^qH#jWxp?|U^=4na1=se>1jay$slF@CS^Ljro;&QnbFglgW~EkozkiTF`{!a759?<+iIkQQe~*e5TsuE| zq#~Nnvy$Vf+B^=T-eSY8$k1!=lol4WtCi_i1fKuUIGg?h7gx+BbwSU)5bJ+_$}Y@D zw#)w;tX%Wl-S!vP0bMW#JWi*T$8UoZB$@yA#-_GbA2GnAz1-)F#3bT*zNI{ibl`0B z-L^9Oq9jYXiji9_imv{(7Rn*0h5qn;Bt5CBc9yfuNnD!s1w1^OMf>AMukswIwK8S>nAj+X^C$>z6-%g<( z42>-)*&8EIRH7|Az2gYfBVM9k+D6CR>sp)goJWkiMmq|9Q0KY)lw5qX+kK}U2Kigy0fA+&(A#f+}6K+VzIJbN!RrMd427by$Ms+x>pTmYQsYaiaIh5e{X zH}a%DOn_}3x&?rjstn0J#*W)6>H=EPv9YrXQgn2`R0O-rf?;p%07SGIg-WSB0gwRP z+Cn+kT3i9Zk@*BL;QwL=Zi*@?W%X4)c<={%%&7tp*PlQ23(8&v{+mdMmP~sBAd!#F z01G29m2(|{2*=Vyy*3#LnpYG5t_`4|{r^4Se{J8fv9SsLbdqUE#EQ;X3=mJx5us1+ z5$J;w@Gg2m6T?|M@xDPhM_T)^!8d@g$5}oADoq+cuc + + + + + +%3 + + + +tilde_node + +x ~ Normal() + + + +base_node + + varname = +@varname +(x) +dist = Normal() +x, varinfo = ... + + + +tilde_node->base_node + + +   +@model + + + +assume + +assume(varname, dist, varinfo) + + + +base_node->assume + + +  tilde-pipeline + + + +without_linking + +f = from_internal_transform(varinfo, varname, dist) + + + +assume->without_linking + + + + + +with_logabsdetjac + +x, logjac = with_logabsdet_jacobian(f, getindex_internal(varinfo, varname, dist)) + + + +without_linking->with_logabsdetjac + + + + + +return + + +return + x, logpdf(dist, x) - logjac, varinfo + + + +with_logabsdetjac->return + + + + + diff --git a/docs/src/assets/images/transformations-assume.dot b/docs/src/assets/images/transformations-assume.dot new file mode 100644 index 000000000..f1952b63f --- /dev/null +++ b/docs/src/assets/images/transformations-assume.dot @@ -0,0 +1,22 @@ +digraph { + # `assume` block + subgraph cluster_assume { + label = "assume"; + fontname = "Courier"; + + assume [shape=box, label=< assume(varinfo, @varname(x), Normal())>, fontname="Courier"]; + iflinked_assume [label=< if istrans(varinfo, varname) >, fontname="Courier"]; + without_linking_assume [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_linking_assume [shape=box, label="f = from_linked_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_logabsdetjac [shape=box, label="x, logjac = with_logabsdet_jacobian(f, assume_internal(varinfo, varname, dist))", fontname="Courier"]; + return_assume [shape=box, label=< return x, logpdf(dist, x) - logjac, varinfo >, style=dashed, fontname="Courier"]; + + assume -> iflinked_assume; + iflinked_assume -> without_linking_assume [label=< false>, fontname="Courier"]; + iflinked_assume -> with_linking_assume [label=< true>, fontname="Courier"]; + without_linking_assume -> with_logabsdetjac; + with_linking_assume -> with_logabsdetjac; + with_logabsdetjac -> return_assume; + } +} + diff --git a/docs/src/assets/images/transformations-assume.dot.png b/docs/src/assets/images/transformations-assume.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..b8b0ec7348e96e6b611dd5b34a30130e649acb79 GIT binary patch literal 44454 zcmc$`1yEc|xGp+z0tpf9Fl_C;EJ z482}-?5iAB5o+&l)U;v_OK)Fa5ACRhzGhFE+~HRd7VO}|M%lwvsr?(i^F$bN8h+I7 z588lB$EKdi-Q6}0oGY`%cGYr>B|pez^+Ci2?vt9|_V5yj-mzgG;s ziRZR2%%C??J|y>g{P=NOM@QwU{J-ZQ2_GFC9Kw)fAIf3YW6_?FZgjL zp66z)gN#||`oi)zk^?D8h{&3G-s)Pn>3AkFGN9=I=V|%jlLq6)ZfF zi^9FJtZ5W0%)VDQn6t6Q;lJ!OeMxfMAS=#kZtbSR-klfF#_6D}Eg~`J{AjN|Gw2_b z6hyP4Ub9tUFlF|<5+ZouL})g&-T?tyI2Niaj8hYBt5Jh_GS#g ziHz(qv~szM_v;jsjPI4lv~uciv6U5~y;wnqF5Gi|3WM9%MJhj`s=m2c57xN8nFwNT z@|j+4<^hX^-+~L?;bYv&>{+lkCZS2(M8jw0$E3w>L^ms2?;;fYP!8rvuumYl&YTem`jOR44Z(vR(Y7{#oy47~>H%__?3`BL4*n=AdK5ZlP)G7t0c z_uq7rGoG_MMANgMCv6BnQ;D(kMLk;Dr|Y91LjMy2yI3npQmxflspGBjLKiS~_Whu0 zeLbktwRnsCQ$ zqtsMW&365hamx(nJEv%@c-SlK?NQssXTkx7t|(Gif=JLw<#eGGsgPPqoB(tIC0ZO3 ztN?9&X|HTnJb}D%2)N03LIk#3+OHo6ZL0dZe0^D>js13iQ?2BR}{BqLS_zqc$TpLqVHax1Rf5sml z7alVDMUr4+U)`H_^t);Ri_N20e^|#!QaVC;lOG_AlLwn%jEeH}uj3_}6BPP#xnrfv z+`^N6@wfA4G8-+1*mVo2{6vG#^xkW%>T3o4M&{ zr(nvHKTb`b%61&yD!-r92>bQZp}X{YFx!vVOsrm2sz7c$lIJU5gTyTQq%G47{zz&ZmfG#indALrUzpR z>^s3qLN*`qE`SfSKGVzG&$DR)&a(kE1St_0T~|$n_ZHcD^9zREG$v?*^!_i7PKq&3 zs!8(-?>YzJR}Qzn4b|5;$$g#qLF(AUgP;1?OLiHT8_b?5ct=+(>g#wF6SSlbCQg@f zXlGY7d}(Gbu{?TndRzxPxZ61)sb@1nI0T2EsarS%6R2ot6kfon;^p&!9}$gEk;aPA z>A%2uA|Cc%ZIJ&DpZQZ~7kHZTm@miGf@2r#6#jCa28y_<9Uh) zGZnp46<|r;Bcy!>`3F11h8mMx{AXYwtKA1+aLgGQ8N|kk=D9a#Z=8*9#THU&;xVt~3?RPMsmvS?7+NJ#C)z(S9WW@P^xtXN z49$JH^Q>TW`%d)x9y5Om^zxvl{DHB2rC$E;R7&$;v;mrO<~3*1Hrhnk@%xGg6H~~! zG-56_W1mfZtAeRw>EYFx6Vstj&5VIDzy%ZYaU)Aq<6;SZ{1Dd#U0Z)Qw;U&-?yTTY z(q6rXR?+Ym_FLnVvdZ1doAJT>*GEZHOT@&tO)v{hI_J#RKFnRk`UQ9u$FJQR-F83Y z&tt#P1%7TXRBr0XKUH<9b8?a0G3`D#MD+C>EKYJ+TJK8J{gVH5?}1P)-rnaV;@S-H zcajdXDHW{Q5YT zJRVDTIc)WA3S|A+yqRd)lv?Z6$UJ}cCzHNPztRosi?vebE9v`v_#XXsa;Ff7TM`A8 zQLI9i8~4wwG}AI@K^fW0g)v~Io&hW6ArUkXsNR${9Jo^XZYwdh$g$JQD@Wd>&3tGdg z1*%MfDo*}_xYcpNWrBLzlJ^w*4gZ{L%XT(DIdYsKNcn#YK-iG~iH*PGmYn$dx#KbN zj@fd~MV%GLJZ@q^m3(aZ!NVt}0o^4T<%xSAS8uXJlz#k9vN9e$cYn3M@t(D8pHFuk zZ9_Dgj38R?kCP3l$%=J7aYgG|DhfOSdvl?^tu0k`h`VV%a-8O)G*<7@o(^)YX|K|Q zvTUafhsnA58>27Z?D1tEXzFh5!;gRTE6&;=RoB{V5F3^9xvujY$Q-DKO_Hs0j{c}~ zjPXyc=+*Y2g=gJwwb~5$c_m+*yE#EWVG8`rWcYCBJ}c39-+mtg_j`G@IjHDfTfSlk zy=#hEADiIygj+8MTiHs;jHFsD3klvaUiqGsTwt@1Z1>@Zoakh>M@8~iS84YrrQz;6 z9i6P?(aVorS+mq3%TJ7CM;}Iy&Bvhz-M=+oNNMpuj%S7Kq4~}5Zv@Z1GIJjG=iY!{ zUDiF)N!9km*$=o}y0x%3j^&EGXS~5yzrSeF5D1cO8*qn@*>o?}@LsA#b*Xcn-QNk@m zTwVBYqr@uu(5=uB<&W&O4En<@BQkHGLG($?M-E$l)-}cg5=F|@<~={x+z3*;XZa}% zkKyK%e0|7I?BNTy!(#5RneXe946|HU#mfipHgxAUo}CrJ(-MdoY)dz0itNnz>s%J4 z?ptHF>lkP0DGM@%f+A(E%ReW~L=wcZFBfdekwKBD#O4SpOuND#6{$?npK`?hf{toj zA4!GypbN-I++J>fF$xw-x)^7@k4zE0;^eU$_|xKR;MO}ze|NN(?=||S^Ht<2^RUVD zTFTK;wDkaC$qM!~q?ENuNi}tQrVS13lHw|qQvG51W2?$`B>qwL=P|;g&eu*n#?H{S zl3p_7lHRZMM1n}tdho*)@0FSOg4{-DnzMz3IcH{H_c_&}UZQ;kY?>2FnrgSKYrYZdR zc=3sK=Ce~Cj=?x#mY7*sd<9AC4!`@|rN8T)o~6@Eyl?0I17134d^@8Rd(IE12ErB` zwnn@9n-Z@U>;Bwc4_17uGBSoOyc@qmHTyH07s_qJ3>hAx(iu@g^ zOrCwP-R_7GuKO_K%33EtM+E*ZvSzA{e5=x(^)4x@?a z6*F(I-)fazm@8R94kPAtZThM(!cs7xhTE2wKDwgI0-faTtRr>1qVV-s3esufjFlk` zQ`C(QOx*gA%>_SOHd8iZZi{YouS))+JqimV{SxN3EH|bp@7f}Kd$kr_^t4P;^Mm|o zxsTNRb&XAiv4jTIP-O!-Ju^vMpg5oWt$T_gVdm!{xDf zjwIFLWJXr`BjFQ5U$mL(-G~*KPy4`$`*h4r_TFZ`$@J+E3Acg|G2H5O!}C1>RInvt zM7SZEOYsv8RK4j+yCrdJK%Hf6GB>{QM!RJJeA&!Zjj@XToty67$f^2iy1LKV`7|ky zJn>e4E>p1~33KAM0&XYcnK;t9ujqB?vM_%=`sD~l481$G@xEP0V_x4ca9!$!@e`X) zRA*-eKNF5|xvH&SVmA9x`ugPh@$9d?VPCw&pRgbE>m@gKCkIj^KdK~8VvV-lw@gJI zv9p2z>IVp=J%lcGB8#F5?vDe0k}1lb^q54TIk#8;f1j7}UOOo28LS#BE!eRIqL!C7 zkY=81HXa7UFcn@Bza#g`F5H`nN&g;)%c+!Yz-^dQI;|75r^2Q51&O4M^|0dLp=xo? z?gsxa4VDov1#ql^7gLuIHz>(grmW ziPx*Tne$gAobv_q(=k-$b`5Q&LB0|A+>Zct$RZSk%KZpj z%J6HW=@eG_!)@?lsbD@SAtZ}@{e$6A=5S9B z-pg4pn$(|}<+K_8dt1i$+23n43SXV8XD*~_xfZ0_>k-!!xHPgyWd{VS6}+F&ct@m1 zAit21%?ek`Ga8%BnPB006r;zLUX+o%H6t}4piRZM}&iLuHx&MVvRclr%87H6TGFj{ z_q<)+fGC|jKY=;bSLDyt2k8_(^=EFDr!a2ZEouUe5+~QPyorO`%T#gVGIFE(Yn%3 ztkUhh#Z9iSD0D8-Gv?rD@yWI=#S)#f6{^y zsU$z12yXho7S)@CNBw)x2#f&v%DHr-|VyN=j#9lhE z^pl&Zc@ywUq)qmhhi076u{RXUMXUtSBC@;)41woRuctRCrM#Jq7%dHlJDDauwGl(^ zRgSlf^K;zgf)+{ZTc*YWKZ!mVU1mOoc&sOPf=r07mW8##i|r*G0l);;5@NGUCiYCC z_c}eHN^woX2)q9E*2|@Gc@cjh1?GOcWAaIQe%c8uS-g!reEpL~Xw7DVd^8yG>;OZd3U!$2_4y?mY1(E%Dp*TA3oY*T$mmP22pu7rH+UcFA}| zDRQimXaVkYwI-D>x`15OK~$dKR_hOK-w=(-fq5s(+Mb1FD+Rv1-pBl-zKj71 zy13m&|H$fQ_qH-WLXuv`Rx3&uS!tB>*OvZZ(T7uzDnc7lMiwWpTXs3x9KxkJvg;U=g%nGJovBJ%i+qlx+mkJ@@CzhtD`p)?i`MNS$?+8 z^)(u94PLRgs#S6I$(SIG>yql-)@ibsJ)dnF$hu_%n&TS<&0n-ah@Hxf4@$_1T65V;``yyJXAkrikPqx@n z4o%ZsexWqg2)*XrmT`jlSAX^)m_{lkht<}fC443g#mQv%XWrKR?WUykmnSw_jqwwd z*l+Cj-Ab8?))YZ<^1ZC$%zLZ}YEA#vz9_%93ANp3;7M zg~hm#7)}6x-=*A#t#lUa)NJ3AjT8SKs_a#u!atg}d#O0;<1q8nQ4JGC5!QTU;g0QG zQM~Y5MVm|4!Q-Rj*{OFdIj%03i%N9E?$k*3y!Z0&u3|3L9AWOwV+IasEV9>i74ZrL zaXVd^l>Vs>j}$XlT&>q4nVgSS28C3NneH9jl@zV{BzHN!P0))}q`IL*N}4d_Dy9ts z|E36m{;+f2GXrl`sVRKlg9Pp;1ddY&)e;UN{i(!F1gf3m%}7X z`FOYcAO*M&ZIHW10AUVq!^IShG~;KS-GtJ6l~U_Tt>A_cXPWXq)bAUTR2e?1*&a-6 z@VEKW{UWUWcHl*i$>-l-%Okcx`HIi3DO6pPkkdf11|Digl+yB~adHJUx z1>SX=)cbKN>z`Fiq+9jQj0vQgEmHR6w4^gC3W@?J0STJAyVB%wD2*3$u3L zv+G(lr;d>8p+ptCJy{2B;%U8a#LrG_QT1B^SODMAWY!)2T%hvR zV7q&`*B;-=*>9ZltCGEW3L8Zt&iL3IfB;`0BVS&X6@!R=&&_=?KIdH!{m!f+eq)Wo zyF|=ks|*1{_K~cG_Tn#zg9V98tX-cdb!A;jraf?9B)?Sn^xRhh^@?hQ zyREfPrfdD!p|Udav(LFQW7dZC%V(VI4VuFl&xyadxcGBhjZdK65or)4?QyNR50?>` z7x}qZ3+vAxjpe3adTxbZc=(^tIk!Ir9KifIiWrsrxvT3=BBS&8bmQuv>AH0Q@LQDSYddr zle&m`kLU{lt8F6)wLJ*&9{>gucWxv^SWG3{pYM2y>L|82|XVB&4V%A1(;@ac;$2Sqj*R|Nr-BWr2K79K4v+ple2orZ)xXW-0m*;s zkKcc(KY(~thDStTV`A#;sz4w-^8<;zCcD%CTcQRyMLG)q^0UoN05DHJ z`HPcdLu}G0jW-4oFexNh>;%Dpk`opQ{#zO6|6acPKld*8p)~7%)&l(3)A7r%>kuUf z33KOrvotg`I|H0fr-sVPG3QH{M=J#d1+xtr;PjBhUoP*(&QQu{)k@PPtD3&gY;g_W zS300^Y#0J##uZ=R$1>{pFz!49S3mVhSqC3J*ccd}kfixi74qb%v6=hz%UWAouUr~S zc90Mk{bfDADUpznsMp#U9aCk+f5R@LxlC)r$nd|QEhQVM;@?a7?>^u=f%n%XYFSZS<8fJrmbjqHef6tu% z^Y0m_v+cF5tsK{EIcf@MtZv⁣?5tPl$>LLkjQ;qd{yJ=jZ1~M~vpK;8e2fUrj9! z`}q?oLR1h5%p0?QAA_AO;#|VN=P>QpvlO^~Mas#|9nny)udGxc4`Uof z_D-#>jg3*r8^^}Ro?&G|$x-#KMKRSdGBR3;*%(Z+u;jH4`P#N}WJ4F&yXN1%{FJ#} zyX1WAdx{h#I#RR|Q82|Tzi1;}}$OP1nL=yQ_H%uE7Q;kw&(;@Fs& zl4(;{He&F4NC;!}I#!%KudtAgo<6Eux1ypVA|gUGqclHXJV#DJUVeORtZVJ~_}D5! zR9-`4;IRrhKC>P*4NZ)L8vE;=)9SXbM8r^Uft0|&K)dHMIda_nwCNz=c=bFW!+A0K z<6CrOZ`+N0r4sG^xf)v?OV4=NFMogJ3YGb9Gmbbby_i z*<@&Z`aW*%lg;fI6p2v{*T&YiLLg-b)gU`3C#SHmFh8G`p57p6(l9G8FK={Y1djk_ z^~sj2A@O`jYY85yq+d^dI(DgFB1 z7+obkAz_CvTr^|m?FXEIn0-BuNNv=%F~V*bDN~F%cXTS4z3w!@DhpWQP4S;sZQ3Pr z)D^|W#g&zn<>lqzpt$%O`S!qYVlY^X!&SsuB$Dgt>DAQKTwPsFPftI2^2E{6(c9bm zrGV}HF(7KeDn&*{ipOtG6yP(bK*wgD?~T_$xS0L{vV%nX;lr7`NZ1O8KM)#ZWdDEw zW+tZcgSUG*$6n|qN^0!>swyff%F5HTv#P49+#DQAs;aY7Q=8VI{Gfk?nr}*BAq#%` z^hrWO!p+SM$iw`v{=P8{nt;|N#I-XsD&R&XrRtg*#(5j?E}+d2@wn23zZ|MzOzS_R z%Heftzg3l!lT%Xa;t>a{H8?of-#?z1IiOz_bJ?H%-Pze0A0MClZzGR4DVhZq)dN_c zZ;vxFGJxuGa&m&vgM*o^n~MXJo4Lmi|7mDwNJvPKqu!e?+xUs`4oFiI8yj1766VKQ z?G7IYCnq-9%)e90!p7qZ=lIBF)<+_A>iMzu-01^gR*HmJG+5_0Wcd69y|aCK|w(? z63l9OZ;~IqLI!F)w6M@<0c#9QlFyOr<9rKV95iry-z>bv8|BTLH^4xMGn$x~2>)%k zxVRL5o1)@mp?V!yiHZZDg&(;QQ*Zc)DTbDn^=!RA&dAoVt|&u>YIJmMWd-xqtCmyO znz}kLWh)PRAqmfYTT7}C&Ej}^WJG3<16O}}qCoW-DnVCYpYDDmQ17|p^VU7lHl49S?X&9);N83#x8{%w3b0l6>pKetmQI{0j++4=eJy*M~I9i5#ocBe-> zJ3A*QCue78+uGW`jssDeN9fQta1^yRTO-~mxS^nhg;^}rQ-yUa(MHC_#T6AD0M{R* zS(})cSpAIy@*0cT*tP37iLJpxlRG~rXACE5^zYxr=JQ;823RLss~a2a-AO*Re=93=1A*e1T6J%AWm;03OO1pqFG-+q7Y>SQAU=kW{k9$2lJiyf1W}^HW#YStK zj<(6Ie4-OOqjVOeNc(Qt*J5Cvzrj=Uk!VEyek*+KN#XU|Hp-m34?{JoFl4xmdrZd^ z_ST$31Ki^KR4_(TJ_n<@nUpXQL`Pqr&2*|*tJQ}QZFF=r2HkC`#aq+uN{}uJNG0$4 z_wPA4INrZ+>)XwlKLECvI$=YR9P%;D*x2~~dZ!2&3i#DVlAxd<@Fai!{IT8`AY?O* z2F|?1WKxCThKz>dgX8T<=~#R((}?ZnkbDZ8!kog{z|rTbaHGod${&(1YkBT>hzp|| z>S;#C4ezcc=f=kUo&`meISp_-9QL9=)1n|w`u0UGYREpbuH*J}Z6Z6xf|HkZZx=Z9 z^>rLX#FdX0!T>>RfAwPh%msq@PTDWPfuU#joMgUjpuXa zqNeTvD>7B#ED+1wbU?^sVX#=A@Xv|e(>wNz;(H}4+x5V%EAB@ddj!OBMd=Z~_qe^` z1hvc8X0E;@y=Z-<-=xuwJl*@uZ(Cxy>!QSwPs>?XPM~|WH^Ghu>Mfo#zS!cbgk4#b z{;!00Lok%*UTn3JeUFD9c$E&Q0CKeWF$w?LSmqx5eczynvHrYkS0C{}#1H_g2l*<# zcmE2x=OuXmFtMv;+~PD2Ml>MB<5aSJtW8WeY2W?-tl8L7?&jKyDRbZG#xv@Q{;1W! z#&PDRh?;fNQs(dEJ=%zZRs4axGjdy2#Fee=Ds!SHCbu`hwHiyXgkXU$Gr70&7dAt{ zvJ-u6`bcH#d_$7ZW4kIw>_frG{d;=)DVK`;{(+B%Q9dIpgY&!8tIk=`R&(pO<>sM^ zxPI1$rEQ_{)fZ1^0-%JCaN9PFX4Pb^9A(G6{+Z8*K6zLYxG@x?N!TqH>ci1UA7qym zrma($hsS$SUQ%2f&1N?ln{sp40h0!6a)(V-Jc4Z6Y(_>iXtj7f z$5c6e1#*fsIqD|(y)y{RQlu^?R4vFTDDdlq%p_c|t9bu8*nfSxAN1*9-$84>;UTmk zOa9GdR^!FtbS?bl)pC-mF|Q)YjkP|m89k@PlzSOoy_9UJd*r*iPyDstb|;uJf8@YW zjgRR}Otb8RXtwk9G}GU-6Q8%!Ti6tgQVbH`iKX~t>Xj`0@F~jTkQlYwV2;bHFYk(L zJ&L^_pzL)=BbKdCB-RYwk?(eoxF;gdynM&DI&LAX|I&04&P|dxT3yw63Ti|3JLH*E zlhZ_*PE%KRNpOH0~GSF_ddovrL9vc7%WWoJvXGBci&%E zB$>q|(f6zU`UH!_OG+(i5qEbQO|{4QNRJFUTFzw;+a+pw`m)_Nu7ta4_$E^yR!)ay zv6zeH3X_%$Szcb2&r{&!=CuF*e(?f8KdRdHU_PYxe3yvL?jmnE%9z9SQZL@*{W} z)-VaW5qRkOZ4?`hiD7KyUJ|*HadhqRZDTdTjPY2{N)8#59#m^P;xm6g%6De1p3*rO zKUtPhY~c(M`P&~u{TXOY=5lXLG#A;F!ntf$deY``_8Mc=NunY$D1-*L==CrJkYI)G zZK2TTBFcJ95uzF)aRgPx{QM@9MMv%M^YL6+?fap=fF?!isHi!A$5G#``Wl^1>cdlcenWYqU4sjC7I@lG&t0Fk=rQGJ+lym zrPOB@a=PRt3+3eZG*mhdOPxHMO>N|RXHneEBT0-F#B_Ch<|AqI5Ty23gYNfXp`ph+ zMNQ=I--BS(e5&}*RK!EbChd!aT~IUmCz01#TwMHzr>Cr}Y@m9ST}q$E`6o=?g0_h;hqZRa4}wMLQH5%D=0h@wjzDk0&>d+`iS4j|^#EW+qj1qD=xzS`AKb z0W>^A;>r#k4l|$iRj(t0bKYe~$t#kbO^KW3WX7pn==0^8crM=k(T}_PmvPlR>W zX<>SOrUF#<;e)2{?##U(T#Oe(?UyTw>4-7ON&Hb}b@qRQJbi1>M1(AER{V~oE{1DN zR}aU8!}ZyD)V-?0(7_d7eZ@@~^YK_)J+^yU2exUFk8vU4snV1GLlXXJl?MjWmDf-O zt5ChNrZWR0qjIB*BmDl3LLz<>IDBkuy%vI6 zPzzFB*>Z7mlhl3M&f+eu)R9ZrikKlOf1Qp!*G>(Utdq5hT(>@(tYp7MCLJKtUtiXq z?~O|JmwRgDB`eViAZe8-c}a<9akQr*Nt>{E0a%75#fbHD+OT0cG+&OTwmCLmK?sSA z1%nTT{l#l;yWQipUjD1qu$dX5DDuY;kI0XQPd&00K+H}q9^I^8C3r!&>HBKt;tk_g z;03>G-Mb^9UE1M@1OEFnjRt($5+z#yRC!}tW{eQ+66L@al>b#3et@Ltgu2Lo5V@nP zrBpj5A!+ZieXBaL6_RdUCCLq%!@CoA73500`%Tfvg9dPD#|z4$w$(b+?5=j@pDb~5 zOjl0dw0FHcTjhSyS>voo(rYs}kt$?hZbF;$Wmt&+$cJ`9HB-x3@26CjbBO;V>FZQ`)vxnsr z$HxMuOi{$BGV05nXCode2aq!B8S9a~mG5*@PKuV!k?Hk6AW&zvKDMt?9s8bVK=yyJ zq4*l>0XZU3xbHrH$P2z}G@G@!vAX(9+NxfmlQ5ea`m}S0ooUKHd_T}>?`Fkl4O!3j z+#7n(cc)p~v3e!_xqlnGV4CyOSLtxe0Znwx1pC zbX!;{Ck&qLOg<+8NK8i|5~O+bB`8x)bOQC|3m`WA=h3rY>KLA3>b$(aYWyzTU%_+9 zdg`Jy-Tsa94v6O${6iL?j<1_Rg!XG*MFJtDy0Tb;j2DuiTtFq#b-zlF+0&Kqy$kAK zUk?12rUkY5wM5LKtVJ?^)j^k*Li%B|^PQ>JZpX!uxzFE?3fXhj7rxce-#G;U`-KyK z=5CVuS*)@6H2*y#ArsHXk ziO31n{~#i54NP8_r`R|i%S+*CaYkZJnbI_qG zO5)S}vk#Jn9p)nxYJPrrg8jl9cJp`uMI$iKLmOxEMm9*YbsUJqOo|%V+-@%`OeVKo zk7>#yS0gHXQ*i}-E*+-;=^7$GdB3s&cj38j)i|@j5sC>`CA>K+|ALf37`^Tooxus2Gu=xoYoksoU!NK5~Ra#nFpgJzZBa3J}l{jZbt@H48+6WnV>YW>*;VHK05$`0r*7x`t@sET;BzYAb_V0CFX{XG!(=y;W(@~CCc~!#akU_g+31hk*xs&nhu;&s-!k&?5#@k7 z6{yXO1uDehDZl~uh6Jve?41-Ngb$; z{Z7k~mg8L!Uqx~!GlSE$2xU=`YDv^D^-4oSlY^dx@A8U@7PFORt6d@UwYC6AzgZ5T z2YEC%2;6Z{__09v@zf{w(Z>y&y_pJ-=<0O_5dzoh`Sc|oyZIc@O91oyQm2P_1f>FG zl#kC1*64b4bYxZIa=JMT7y<%rJKFsLB*;*`!m2i49X9*0klR5d3ep4D#pZuPLtj}~ z97v}Ld3$-ic*FGN{kBVk0!Z-7v3ki6gPC_ZB6&Fj`4unlwN$U$7QN;E@MT>SQAkROehuO5WHM8UY|=nM{riw_)mHfyw0#+G74WNh90~l;@FnG{n{zC*9z~k6V5PCb`hk%fq zCPI{kCJ$Ugt5R-wdAQWr*oftAvh@H`PW*!PnTLUjkx|Zh)fRAE0C08W(MaNRy}i40 z22{$<&d$mTz#Q~Rr9}18Wwx@xU6p}Q<3NUw*fe|WCsP3 zAPcY%QBhHVj{qDMfIuJ+HeaY$mX-Z19iEqGGT-_GV8|S@Pa(;Ke`~e+rLfk3gV^6E z!o%z7lg^O?(2*E;0B8~t3sn}HqN4ud5`; zEv>ltH+qYHSweh#t?drZ>(`e6W;zE*Aiw|tur?(RsHz48xVs4;YX_iej*g7JoS^&< zYG7c%%pVft4)9cf0ZBnjYWr;vdia z{Cr>wK<|M2#Sj3D2Oti7eT4y{NiLb#N6>3+ZQZ)@6v#6QVq4A`rN8!%Q$Qo`4A9fk zjtL!s#h9(KAm+4sPeUU{7sXsuT2W#4y92yG;10~p%)aMnc?_fg8Txf>{H+%#R;Gz&Pvc>jAI{A|f0tL^BlgZ`THRDuBfSqz(84Z~z*3 zY0ywmP}Vh1BOdV701gA(o|>AvysW(rs}}~tiF4F(a&~_9mSxG8|p z0f5fHM-;sA7bcrnEd!ifa`Ib12Z*cyMnRmT$>qul@EUGzeAc2hk!=5%z$+>%%^61L z<_N9`5}|-W1hWTt7SLv-)c^7qN^)}P2qv_G9s}9C@1_gj|J~hGZ{L1&KJ1;Cm;he^ z==@XlG05485xc*HJJ+lT2@oxos`Z8AhXT(x2P#TR4j23C`Vt5#MfEH4tiGN_FE)P~o0!D#s*#5oWf5n6N+l1L*#CBJO+k?b@J}Pj)1lI;D)xy9z|n1Z zXi5Vh?ExlQEd9H%?(zmly}iyc1W@lwxv*Z8**SZuP8w=(F5V1G`HEM-5HJnksW;|i7 zOleog|FafgiQ$0=X~^QlCZVvNUWRB!Zb3mrw{D*eu(se!e*T!dJ{0=0Oqf-J58Q02 zBupIvJi`QUo6Z7~kSnM$dG?oqhb{s#zhYYTlsYyO|3ZXGxuL#3{>^hWdHMEqrx1IhJ)P7%i8qki% zVmNfT)Y`3+ng%?qnAp3()ORFb=%N5O%rJVrdOMJ^U-BrEqUMhaFl2kuFWBPt_V*?5 z2;0Hhf>MYYw>z$=-m{Yv6jW3q7Q^2;IYukL%+d>h)NdvuR6n^DXoo<`@8^eBHMgLn zl9CeV^WBhcU7&5i3%x)=85QbeOP91SQ3zwkA0k2>5=jdBZU!;eUti(co9zK!3h=rY zd$YLs_%~;hnkNqRQ5sH0Gv!9W@hw}`nAL%bD-VfBz_>!VNCBb^Ac3i=slC}Mz^0)Q za2y8EH%2q*iHZp~Reharyrwv)I>>+jsvA@nWJxAzsHxREU+jNSt^9@0EF+3XL?k4F0t4qu^~9W2_XGM`tG*5zka-rvbPCI7Mc=JwVC+LWD4o zUpdv)mm5j$s4rdsI{OX~5xy_pghod@0GNQ)WCDz|bE?8Ka@c22EQ$_9VgOoC0fPqC zSkv`P-rT&f!~`&7-&hbuCEp$c0R(H5l9B?9|3Sk=Vnf4Sqw5W*SpZ%U{q^ggVO~jg zh$x{<9}3uAZEfvu-xmF`X$S}jA3b_BnXfD>C#SnR12(3Yvm?q-4Ee2Uc-%A5YNF#M<ky_i>06A_g=mOOU zCx1ZY8pM!!Ml8IprpEQ8pB)?(snxv7!H<=@P|z)EIL9*=1nQaa3Q;b?lLKZJj1f!) zuucO>{7E%m96<>Noiyt8zYoMj?Dp_b7|}?b`}0Q&gfd>gL7<+IoD7Mg^riSS=TFPc zjbB9yM)I!`sAP7Fg#&ND^lEL$2iSXSno{5=JE}fgq;wGy-^DK!O51 zACR?|ATS1W6;wn-OQ0Gs?LZ_FFp~x?Txghp@16610k(3JR_B7<_nXaXsV*xWfq-%R z{{4Flk0TR!*F8yRnoA?`T%E}in3^(c^d zz@!4=bWcVXk?Aq%{scM?#2h3=e{5}lEJiRO{*Z>i_E+h|kdTxFFOSiPh5ct)2{5JC6Kk^-=2 zuQAAh&FELw(pm!652Pai#>p=z@ci*3l2)0??bgArdt`H@%sCWp&vK`rQ4_lE_!v5k zv)yNKOLMkj+Y7(tRsRfjT3I0@-I^4}6^GnB`1ZO$wQ7RyX;22xrsi-Vl1+qYVor;FUKiyCrvoOsxfwZ zPZAsNjzi0rFzliEl!0(h7mE&ppBIcdSOVd^!eq`LSb3?=KPh7nUyj#^&vW2TK)AD<1}H%ZxVo{U(!i zBhb0C6y`NI*>too+$18wFI86V!H9qXrT}2l!ENl4|pFj~)z?uX@IyIP>g?_Bhila9B=R zZ|=jZH3PWscKrH$slO}25-kRuQnCX7zJ9}4yrkzlUg}EvCQIF0`>NW(n6u9w73WDq z@(+^NjGF6T3kyO#GAWput@~qH@9*zHc40DENC;F*MrH&Ew_<_Hc$YFe%^I#`Lj|86 ztf}VWq_=lGHx;-8V$Qn-gU%qFlDnt1tM_nMN5d6ZFcB7Y1oc{&a18|-Srf6Va8+Yz z=HbHj;JHiV#X!VvDN{yme)ggE2<*Vs?beV)gZj{YO5En*@DZek&2Tc(@25XkN#LK4W+K#WUASC~T8Greq!_L3W0 z6iL1Gq_;p5c5!{(8FD{vu7r~Ba(n$%$Y%dQ>wq$IRT71Mm-~9Dk-JpP(+3T^%#l{z z_Qu?b&**H~gMQuV!e%v!kE+^@3>r(4f2pbxQh)bF4d>`J&*hznMk5_5^!t5`Cxv3K z`R!|%&LOQL%AWUvzN7E;iJ}0M2XDpGuRK_YhL-?Z*!Bs0r&`^F<{L z75yL7y=7F@-Pbm{l^aAUgGN9F>6DTd6p)hcl*Rl~jBGUS=q=_a_mA+0W9~Pe^!DIf2vy~|teM2@+%;Qwy4*idZ$z3Q8Ur02f*Htk z6m+nPW%XpA#)f$pB)Vvut?+xjXTAT*jQAW|hXCQnECU{BR}{C{5m-Cd)e!}4?R;4U zNOP<&F1Fk2yW*~|$0qC5v=mwx zGvsbiFuXlJKiYI(fJ!wqulhJ4X+W3wv5hp~mw?^R$w@(=wlFX(FC+87e}bHw79^1% z5umQFJ{!qo7r4=H9vV9pDC(qwKn$S{4>tZWly88N9W-VZE{DQX?&O+SP zy4uZ9>~uYGmgqPjGOystlUqpS9KD_{a#oryPlO|TmE_`3QFDDQls2bhc-We0gY)IC z>&x{5I#EkAoco+QTE}wHStf;ah;U`CU?x63VD2KvGTXs99>`ImEkrX48~!<#Q&A_3 zFJ`{b&sU~d&{_Ak@CKrZ6lKx>&Ol}eAea$22f;f94;D@lU7w?Qa26cMhj$h~rib1h z%k?d)#8}b&JvdRH5-ky_isUBPMt`N+uHrm-jfnR#4)G$V-_1wXO*uIH=Z?Sgt*XYY z)Pgq#f9gCpuH&33a;+URe8^#T6%3>IRtphCB(SK2&vplXMl+tBo&tNtq$g=aZ(O3T!sUz9$FFJV~#or#>BTt{2G z>y$wJdZ)ORH0N;7lnkRHM}3(N&V2t+&&T$NwR)#N@1zQ%rq54yDfCPlOdD}ilu^1cNv7WJQRedBnB>|}rvyisd6m zf7{KGh^rG5)`xWq!WBykjen6-vz~m_<5;iZh0=*HXWyLJDzFxr=ES+-CdU-Topw+c zP1y`_i>8dU4wYJFe;qX(#;7D1IwePNWP*vOiG*a@#daqv1T9QVT}UL?g9oBCkvYmV z)$}qN&cPn0zYlHpaUW2~^U~BFFS||Od^=Wz!jXEn}ggLgp+wd@gxRM zH9LEuKT{fVdsX2enF&R|17TQ3mCM=~_^&|39D`UDPVr~r;#uJ6gMX<}^23S{0}asw z$!|mObbwr|R;Ya)3o9f%9I%2~@KoWQ@YAJds;`f7Vu?Sph*g724E(az2+Gu`sLzS7 z9LdQMapd4JFD)$rlM^Vh7bnvfkliaA{RR2P%csFcLp==|#1hj_p#-9> zCjHMVP5EZA58#O*2{+o{U8~OIvsOL~X#%Pb&p#`gySwuaU%85S@)YGIe*!)`Bv|i~ zkSxv2JaXTjgUlD~&ijyUtbGv&5*~QzlgMnXUj50V0|_V(;ciVzv<->V`+ z12-)yDr#|Y5t0qCrGanC3h9uZBwnEYX}6&xx?=(ZpFTsiS2pyhTq6?%eSKxWVO|j& zIxb~kSt$VJdGu%-vi)QD^Ks5-e&&NU_!?jbV_0-N&W@c6Ga#rtmgkf$ovyip^%_=& z4rG0S6bTMGa6Sho^+ZcJ5o~wBGX?(^LTkti?CkI3UcdhC?c2@W-8nuMkmD9!*)f=a zil%()SW$pu=$`H&M(hw!cm8x?TKrD^!1}m&1C~el9h(W1fp4RF4z9!X!j(K z*L%7aqWyy~`@0F##pwtS2AO~Gbi`ro5O{bI52a(YArOFr3XT?_cdkJ)fSdmq2)n37 z{%N%N2N@!PRwvj2h}`Y%&x_UT4BK!au!Qe^QDHk-qvA+kvbt)8nqV^<@CE!#GGTHc z5sexUd8GpB58&Rsq<;h&-OG=92TKiTlYmCue;Nd|$b9YUM^sc)kS;nuJpyLCw!}4r z{?M`p>ZFHsZ4_$O61bt$O{;YH!EF(gr4xrRaEz+?i(nZ5!@G0m4iS^O-Ok_corPWr zF|q2ZDq^k|5W+~*;%iu0twLV`-jTp`hiH9bVggQoqet<;l|K@6(E{b@L|fHMp*%Hta(h$e0QvKxpAB|3aa z^=KLyeU{4XpCSYDm19+O50%2wkuLmg>sRyoP6@ zos)P|QB$+Vc3SApE68796ua;IJ*K2lV~CER6y85Lm@Y96=}3pSQ4~Y~v;oaSXx;-( zECe>#E%BmKqk^g_fVB{PT4#oS#|tgxw~4tMAnVa^Fy$F^_5xwXZIy-~jQWYC%Kvv0>ar zTO@XyfiydSpcW$=@SJRdXemzM6Wck!Kg<|H*|%Ex!A-wx;;6B?Y<{;;Cj7GLKSLeT zm+jvi&E=2(LofZY_CqT&`q2G(BW}SnFAA~i^|8rnC*ZxDONy_$df}g3wyyga8H}Us-5*P^gIRv_-JG>}P-{*j z`y%>ywqK9(h*kTxew)tCFhiP$*G>G>PP7gi>b)37ceaSyq-4m{(Bhtc=YB)n#Fo$= zUtFBfZj&I{+l)XYMAL8~eml(Qd@MV7et+Zjm%yb=R9!{2#zjW6ckFC;B-61tgTLEW zaee1O`=?$_R{=_CHJo+pV-*InqnDaFIQ?lT8q)omxc#E)nyGc}uYxfqsoU5s?Gn7X z<#Pp?5huQ0)kkU7(>$F2_Fe>fzw_9cJEHyzIR=6l!MTaml}`OlUb7$J_Z^K_XLnAL zw|I%mA};Zv+RM17*nR17Vv!l8pM@{t$4wj#i9HKN%>+v6B+dAkj=K|nJ@v7gY*pqO zP8;mJOIpGE1JQRHCx+^nCIts6!mdwOg}&1pieYR`Og&b7LG(oXCfDxE=(ZR#XY#oCKKkC67gC^Zv^Fj3G2kc+ck8emMQn}I@2Oyq}P#1bjgGS z-RhCT&)HJp?wtcGYvcJ5;~!dDMV;l8mg`02#3Nc>f0oI zA~&1#Z5@gyWP`?xFh91~+*VAeSjk|!uqkwT@WFb#@a*dJ@z&;9$wGsFP6u)zo%ldB zf+o8oG=x`o`R2_<+oK6r+Qw;(dy6%TyN=Eo)jP?JaT||y5_+>fjo>?HXqJDID!C=2 zC?B?_!8DHMcwo!qy#LNtk75WpI+h#V_FIixo25_{4|~o3y@w~^__`eC{eYA^t)BgA zpNlWz=JWg>bCd{iJhE67$-+2Zdc8qgc1P2*@wVrIMybIvzYOzqyQlvZ1Zq*}1Gdq` zVZsTdgIkxd^Tby<=ct8P_u>5~Z$GtNTZrmnAKY7j(llR&9B1d5Twdv5Zwvl$i87n> z_4O<$ecx|QMS7uC7z71Ehh^>lD)gT58uM?KIzEKD$=kxFBb)hJ{BNOncG@eUjG)R{ zwbP)3ZG6HL$pKF0h}3b-11v%+Un#ZW2`rI_x0S8DD!Nb7m~;;}>jJV?++9MZ&kHi< z$BN`l*W#xqOU86D`MXD>Rpzso8LtHq0~KBR=?|`D$v&U`LMvHhvcmMa$yfBE$(1(* zWj-UTGy}S97#($q33XNO>!%CSAKwOY6Db$uYo_O0IqX%^wbs)aPS0&u%9f`+GaM51 zD8yhenDeaj&X=N8)Hx|heZTNXZ9mG}dPr+9%nFIY*wM-+Cc;%=TFm=T@$z}t4yXCM zB9(bod9MRWr8B0tj~{34j@k@PQxJQ=loYAdn7Ua0Y~EPWvO0*kdHc?tfwyPFvz~S) zM&2f$&1=q;#0~Ktl`Sv(o?yC%@9Z;xBuqf){DZG$gZIVjYtDj!#T#suUgE@vui%Ve zAaLkATAL7lKv5jTe+<(xl)SPsueT+bbAp z=umP1fQ;u~lF zB-|s)+&J}~z7weJZ!{&#?fH$S-NRMs=zv0lW=+5b`D@(W!PtC#4eOfEyBmw2`RCZ0 zn6AxH7d*CtFP~EUX%8 z*pUkT6hFbsw=7*xw^7dHwvyz2Z+;^jNrYW^Iy9X(jHz0@UYw)YPP5P6J>`@zAt*QEd@5U zZRW8Tjum!{xQ>*0QEj0A*S>^VkM1%-GOJ&l)5{C;>bZro{DtZH=??D*GP9(;y{?3 z`S=zcboefo;i~rix|)Q$N+AXlSo^Yt%A^*u*H66N>rEJyr;km(e%a2oZi&_@x=0#C z-bftFZ~Wq{#Z{=nQ$bDEc8HD~Nokp&P&-rL1H z)WOdA)~%YdswkR*<(jxJ89s3?c~Po5$N91Irmt=$WI11?X_^}4T+itjuwcFRmpCeC zS~7@2&He$JN_v00z&U|^sl-as^44)vs;h}nK>&$0&M4z8syAs?+ z_39=?gal@0{XMZQROLFftkP_zXM`%vpXh=GYGuCvM;A~YBcAbV@mx_ucP*`|26bE? z6_@ML5{%)5j(KgHys)oM$TF`vZBJak%T+ga5Mca%kBF<+xp_EyeOl=B`-)+W*8C}X zz52|+sIBCT;ShO}SV*obE9T45xo4^|UcoV)^X0Ev9aBk#`uTjPE4CsVtBRgyU)~81u^8r_w#wH+v1IXse1BU6Gx9A1^!2{-l$qOLy=(n%@m#gt8A8 z24Yno(S(bEC^z@GW2=zVt{rI7g_%?0&&_Y1#}+>B_; zpwrA$RkQpOfUCo=wQWUf?W4`yh9PSiT=9ZXu+n~QzUfs@qc?v<1&($NgV+1m((?Bb znDmVv0=%_<5z^EZ_nb8BB?I5+Id*>iJiyCA}qq6|8CB z!xLE-X`;n}bW&9!zN-hg*6s^DJ33!XZgw!1Y#kMz^4J-! zCCw9=4H}Jqa8+=VJozr&{E?0p%UpZvO7y;=iHsA&JH5Nd-(o(K@jjGN(U>|LjP&%; zfVY%*-qH4KdDxH9}_f`3sr?=g)2Q5a8Q!~(TT!av&k0^yc7bsB?*+K*`Igv;A11O35*2U5Nt%?k-WNsFin{H zF)lMAd$>7J{379+Z@qAdBjbnHVJCj>jXLzD>$RIZqd^|Qx#*Mk`yKa6n+3hUlQ_v_mc;ixb=LS=9O@xM9&2)f#Wr8s^>23BrEMZq$Wm}2+VhX1& zMYPM6gQhUtP4`yBJ2`r+&aBqIq}c2~@PDmya_;EUsA+>VHTjCyNn|Y3m!48g&gbz- z-F<6rw0}&PLeSHK|L6lji0k~=KdMV!^EFrrv1|#RNrMZ7+}zU znA(?d!c+FijeNmpe;+Hoc|cEYZHc<#tIL7sY?bTpCWa32_;Iw(l$NX$#tYsf*ISL9 z?H?!}Z;^ytk1%vjd4%oz{?-Xg=($VXS-11fgG%ASuZ#R$%&E8j^#S_8L;o5GgUxns zMkf;6o-A41`5nWZS4#_%nVfGwrmptS$TzuTEqihe9OXR8X{pA9wT*cz=KOsbtX}Ko zpn>f4*k0nllRdgtFXu;onQlh8HcWfl&G%^zw+^IEE*9Zg=`n3tn^NJc^La*i+gdiu z-Oe5>{HbwVG*Q3ReM`4M498P6CuNdT`@M5X>i&0soX%!656|imb5EtW+1++Df9?Ytb;Xmva9mx}WB%Z2>Z48!wnAjX!Lr)jF7j*jisQN7); zFrkJ<>T)G&AmCAx+}E}9Fgm)#`+9drYWbR{JAWx!4S%y;@3pt2;Ums4EnYm&WjxuR z&a4Z3s`pG$VV3<5#MRl@-k1V zuR4spOOk{B%eE4hC+{#A4y^W^*U=A?_xTpeHZ(*m_PrdOFPcmL2`mA~^TZd(C z-c(CrF6= za<><@X&8PvRf6gLq6R23VCcK&>P)V7XMF0iKs0&(!~DU(^oz01N`u+Epu*~51H8A< z^77(To9>oKOKUlw9ps;E?KjQ0QihwAf@)2Q;&L+A zU~3?BXtKRuJL*P9m+=i1KMc#HY2Zh>;~)|oB(FaCJ{qp>sZ<=ZLWhtMSv=W#T@q0p zeihNgqz@FCH}ER9t-k?uk1IkWoy8JJ@$nH)uzrc+9x@y8kl~>Nih)bP1>8VXV|#$Y zUM^cGCJMy4Y*eWH^kr*$d&!7G{ro@mQva8YHjv)K7bDr{het{Hw?YnEVFv0;dx#xo z&VVul%Yk?k?)`D7DN;=(Wyoia4dd$ZM$z*6KEkJ8)&`RH@QJ}uc&o)xi&}41t zUF}l_QOdtbP*f=m$iG8QaUH1n^78LXOF4$qt{@h_-81ydrAx>fwJk_ilg?E9j&TLi z^ii{*!D&emh>!*bGh4H5NDF5uXmRzj1>urgynOl1*$)Q$gEot>)1o|JsM^{SkY5MX z3kZ`?9{~f8BGmN$l}|?~+6xSP6Bk1+(y~SUFfGr{}XUQxLdXrI)!Fee2a@Oa&ytaL-ggjww_r0pDUD7{^jsY&2C96$=Yffl5s>yF>nm{Yo<7tN09gSF z+UWZS<_41bEp2TmEFJJkpkV@N1C$mL4wKO5)H8g12!zdV1xJM6*krY%geRE*J-uX_ zyxEP}aC^uEC%qrtyX9gdG^+I-K1 zgJ?2BAyrL|DFW^ep!I^$ggF&(+iYrWH33P|?RQ2n#=v5ogMsoBhl{EJ9Es{M0*c4; zODzbgu#dpUOjzRR8<02Vi~-r5i-@M8vHK>-mc+%ymt(F`^1I|GCzB6<0EjPBI))}% z3WzIZ9sNKH_A@d^AhP_S>raxMIs+}+^2a2rT--N76J<7y0niGMn1Z&%4TLH$ABqof&rC=ILUI>3a;TmsbQ=45n(V$n@bZkGZrU@Mmzb-=Fr2*9r;<&t*2 z2UpCsYlwlqhteVM-T~{qhFfC{q=%P-pN~Ry0eVy{0pT9V8=k~|v<57_3R3qN3d|as z0O?ZSe)sGFiI7*-6TflbKD53@Q2>$mQ5(dBnU;2;$sY|W#>Fx*Q&SrO z?v2_zZvb+>&84p!3RB~kW1cDo0|s(Cpj;d<9RpB*@cc8xHwt?D6!f1&ti%g zD-aMfz*hcRK^o%4BMc%wg{UusT2^9TFUz!1gi#bSlHY0K7rFoI(9y?67bo066!fxdGPJ&qTUX zzzt{`FJGpyJRFZ>H(Xs?tA;^|yRQKoddClC`MP6R9wtg<)_5J2$k3pOuwP%IMYI9T zgoBBRiH&_1xXhpq;ZsC>QD)Xi$jF$5@+X&*nECCJ4#>pAa{%?JVCaF`+1=i5=iorf zZtx6>I>8G!yBH8}lo^om9HyTYbGS7UfHVe&&a($1sE0H`L;y>@OnWR_eRg&h^mF(( zUT0^U>g#`c_F(JafGDH|7C_&F#gCbwP$M8De1LlD(o0O}J!reYYMSd|%1KHVmzDk4 z{*4e1;Z!-2R7wJ%}b08f_!BtIfTuij%1HE%jP0E04rcn*UI zDdehf{7>cs52g9NkISA@lX6)M;t8L!0e=F-$aY}<5OG8(A0dGQVDGo6hFPxZz105@ z_90X-0p%emHzfImDM8*WPdCS>=to1v#ifE$hV=u3W%*-f7^U#k<>3quCi4enmkF~$ z+W_2jI7hDGQPlZfG=zn9gN)?i?G6}FuwU&ktAC$Eu_z&LO@`?8v2sQ#DzuJ%z%Cx4 zmNTLYFwx7+cV8vunqHONOF(38+&7C2+r~ym?aAp8Z6JDq6Gxj%V+@#0!KlUbi51li z&9dCiz*JsG8W2jbvu^>HY<}kmw5ELxIN?!Nef==xu-%_$D&m03BoI$14m1g1=TY_> zkwA_51izc2P@S$3)r9X;!xn`t2WtSh2B5Lb0f`561gH`;m!%w9VN*+i z+{{d~vl*PCWq(k1;ESAvMeaT~SXDb)TU!`VIMhG{39bN}UqVMe6got0;U_wmvq>Bm zfoi>ApK@978bB-y$}N#V3|I+mCxv};%F;1IFaS_8xIm*6L-HKF7~)IkO%+~VDBV#3 zMln-5)@6Cn9LyN-c)(q;LD|UxcwT0NgI9?Rf!q97G)OZ){9VVqo_I zwRHEa;`G!D;84RK*JWcx(?+Tt7l5y=ZllMck?`-XK$v~T=n7#6Rtw_$fM7@i2&q8@ z2ww_LL1}rp@eYpn@h-^bCumc>44{nUs#%(36+pdZ<^w?BhbOpWX}RK2F$h*HA_6vt z`DD!=+->&(>@M89hjMP&SV8ZHBH6rXX4S5`cl#YkcFQ9-Jz}o)_Vn0{S1{lI0a_eT zOT*&G3M1|60j9mK`#b?K=M{5+2oK+sYyC+&W*>ApAi)FfsXd&){{H^(gim2#BiQnx z2kzon&;>trO--=p`I_a+;93Dx4P+A(-+3s1X}q!sWU^*vi@?W(<2Lt9MO5^vm;$G) zUZYRFT9Gbnn;|pKq#t-mz){+$+vy1<;rRLUClG|5+&hGdGft`Ccl|UffvpH?Bv>;r z>^%uwjP&$N;8>!lP0y);d4%xeMKRbBY+N9q+L{sj_u%}ZoQo0}+6Lm0^9?pP8-N1> z2TYE#MJ4bB&W`uSYn&~?9j^|%D+84!t>Mgp2D|wn0;(^<;6Da;7inY;&d1LvdQgVU zvjMro?EWU|Hcn15Q`D`(`CT7U{``v7w-GWkv+Hcaoz?T>%^CI-|%ya#;fGW`DvOd9U z_xlO(5DL+*AWGjSCQ1)DbxRV7DU6+3z`+X*C|W1N5Osj6>YuAgJqdNdU?L++w3(~{ znPQlDhGI_HMS*Ib^99tsFcU_aaRONl#X=whI-qG!(jUGXW=m{}SJvx+DglAEw!{^L z$saK7P_@v`&JM&fxrgsdW%ZJ_N8D?E`#v4MaNbPLo6#OwhO$U?)ujvBqo z_WxBd3I-nYel8Z}9=;;Mp^fcbdYtf7{b;2~k^qucV9)0(wwM^pDg#2Hc>6*ewqlZ2e{Y6h1Z>35oh;A4!;EY_=#J_{2q1_b}M^4X$_9C49eSYl@ zy0^01Ofxm^n%hdpzPHwJddh=f(?d-Kk(7%}>1U?EbgU}e&qq%c$f6kb+Afz5;mR=4^NqB;-N*Xr_S5OIO>fU5<&uS2*6?UfJ2b=& zN)T}WS)b=T0n{^F9-qdpU_0;K5=ehRH(1>m#_`S8UV{1^OCKF>;~$gd@1gH3OJAGa z2pjrGHk@VR06B)`qBM2=;LDzb)W>}a`nmd>&TRF{P#A4{naCSm94pIxb`@c8_4sZ^ zTtX5(XAMgqYXSk)f$aZ6iFmnIIYw1xE%+$zP2lhoFONGIQ@<`K0dMq9trZ2~XNi?)awXttp>`N*v z=GhOUabhW~yw1<)_xv<{j1KKcI4-m2hm`Us*R{h3w((qM*=r;;U;&?(`j2 z>odEd&6IUq)fgA3L@4WiG0ah&_4t6UmudOISK>otn5Iu?&-A0wplOexbpug+&jsG? zlr`hM;U-SkhCLz$e6i84L!}pUCA_!Hh zcUoS(Gu`x;Hj9OdnvN~g8?pAu8qpAFKpo9djKqXIj)ByS~5dveAK35G5 zd@iMsT;D*=enw^Qc%@>`tb$JiD~+ebLmB1XZxm7=&Kv)n+Osq@Tv>0lo2Jr=M4NPK0(@xXT;gdABMxU(NB zrl>-NvZ+OSug?lDG4_wtIC!$zAoR1$y#E2`hT(EtJ!u8o_?u0qL1|eH=Mp;L^Se1;?wiWE4@qeXT2Ap+N2K(r=HS_oQ&3%IM?R9 znBJ4(>0HRT7mz}!qx@(iY5Vli24CuWhGuVAEQgnfr1mSSUXPzcrwNqw;dOs{h|o_z z=A@aspH2+bU^0uJM->+8HkYF}*4LJm@%s|DKi_w_EHib*3uLo? zDxTL@)1w#u@yf57t_RS#sBtkDCtzVUUBMFXSuJkZ=z9U z>fSKTQ{!SRqq&v;;sTiO=LbZgaP((X7boME$*-dmgIh0R@?QSZSKr0bDJ!SUU>HQs zp@=Y8hZMGyc@870{|hAqdvDFea3w&tQC|R`Cx-q;Zt>c;XM!QuXGz{{J=eOpCElo{ zxRf$lK#$c}WQ#2K4JA`1x^w60ZqgoNu;`L%9hBA7ypY`8_-ZWaP=AP)HUwX7xX0bm zb##-pvf!?P1p7tt%l|!BqYxBK6yman4b)0%mZVpxr^y;)^-u>G? zhgxN|(|9>+r&Jc>zO53zcN;$H-c!rL7t_co5jzuG;`g0Cc;-17?^EKrwL)ea#gBI; z=d``nW;`F9t&#I)s9x{~lDeo<1DAMAj#}9FSTQVQb(!C^$7H7lk7O|D!81<#**iEo z>_$Zm4|{jcjL>BVoVNCJ3?9T-d2okZ5h#|(Gr3!ZP9Cdorjve6{gpqjmxz(%kI-`7bxG0I zNbo#q_SZ|l$|p#6Xtr*BYPKfqPD}5rvd#UNWxn^FQbJuG_f|=;LW9V(vGEvI5Mhe}nn_vnLj&6v9f1We3zr9HtJALY zK3p2lss(e-wvpiB-7p4Xfh1- zzv&_y{+cf_={F$Cx^sf3jwjbA(Xf{DirHg1ZIr%Cj4qNqO|BiSs?q<&I^ulM$|ftq z9)%W?$|lmAO9_jOt@6!r1lw`)N2Z!1x9d?k^cREWo}pxjxQF#wOF+)WkQkZR$}&$F z!oRC`7@*$w^3aS@TI7vJKX#zb--7g>6Fd~*aLWz%+c>3~?p0R){czgUl9W2G)GRJJ zZpnN)k%&xF#*32Y+{3Tc_AKn@dne9(-imc6zA4M1<8l!@Y$qDySHI4H-p3rJ6`@P5$lJ#_`HdeTYO}?BBX^QdCozZjKaAZJ?g3^xb)z1RtLsn15(fL@QJ& ztg;b}R6ZNjl63cpuk^beak}-bRCv&+fQ&i)$Hzs@_448GRVY*bvysS^Kt6`bw|@=2 zphumjHTd>nVO9R*OxH}|)t5yCv6l5G`5lvwe|@~Yd&-u%J)2&|t71SGic_Bbs$S?U zdwDvqFn^U9|B$K3p|>O@edx^xQZPDS=_Ql-67MP*7h5@G_RCvJjcjzTFx228X^Amf zlm~$*Mo9LxdUoxxZE8RkSHw^l;nrB4i@eA{Lg+=z`9nM!exRJTNy{*+9aSTW76hZ#ldoOzS+|LKAfCQmRO` zlHTL{zk^>5ccg~cJL=3h!#&*{?s19fGSEl8&d}v8Z7TtWpR!$i*_aD;i1(H?H|FCHO z@u5J*CHsQci}m8$8fRlNuii7)4(Z2nG!=vtW4;Q!=?9qu^f&VSpPrDi{NXC{P8BNb z&5iqS-b>JkGuj}HA*#yjAJ@g<%7VhQ?Xlgpg>Q0QZtv4*ShwGOn)aL)OOC6xaO=&; z2-@>3mKe5S#UU3gOAOq<^;C8!(21bfCnr^IJ{gfHgy!{6B7=dvuuQFrR33}>Q0F!W z>O#tz3;6<$lEGJTGn-S3;+@Jw1!XM?wvya@&(~rf@$ib7$0ua{Ft|l@7U9miY^z|! ztuH=0qN(&sUX^I(c@qOs92$h}a^S#T#Y zBNCntq*cYgx6={1vlnQ1!a@*#EH4eWX|uPDZyQ=RzF;ULBQ8Ih>rR?+-~zLH^nqv6 z(n&GbUqPVLQR$L8(}ibs-4kc8G&)t-Tk?bR;uDMM3nt3QSi9f3GyCzSyr;22*k#u0 z`sHaT_RM}{I^Ei93p8(UL(52KV`PAbs=vVSb-mOY-;+(eqKGA%=pENxgLiVKjSt^N zq^?n`)EB0X`)U@b4!p`jYLY_q#(2-T`lZtPV8_j{ia*XG#Va?M0*!wCpKO`C*t;XE z=s@(caljXAcdX5)XZ7{j7TI!xOIc^PFw@E^*Xbz4-fCvBzx>YtyIW6=4FoVD}zt%##f4V<_43E_ z8_N5xJd3B}l{;qRO__Pk*8qaYa)0%HJFhLWl<_I7>M;!&smPfKVwQ!xl>BSBYs~FJ zo;34)=TYna0}C%mPd9)2hnV(gX9xm*W~tsWyG0kQnu8RdAjm+{grM$NPWQ1}*H$%V zIZuk(IT-Y+3}25PzM$`Psc(H%zki-By!!lDjj#8(`GE4=M@bg*Px;Cs%e@w&UY&xw zD$&YeB7JF=rqU^EpSvS01)2-!$M~ur-A}j`GKL6?$zfpt$I39kNc#6w8nawv@~EG% zHPeu%f@z3B@fBlZLoss-;h#h0CELCL&VRMxSsnj;eGs|2cLP=4^*CaA&&}aGzEWbC8Fq?* zk>rnq5syldQ6mN3iF_D@sN;!>Q_bu)C!XuGprYzuOWndgKGs9ATB?Txt>QtxzL~Ae zWBqaZ(!uFWPhGj0CNX+DI`g(LZU0HqRPcmF@bVffUF}-kG z>DJq@$j1mHGm*eVTKsq=60^J_%;IJ{-m?10Si|LKYo+i=jsyky&!JHbnc#@ieaHU5 zY=-hvNzIvr>Vd{u-8~b($;BoA9w(Ds#z;bIUS}S?G{u;|pUKCo4wLl?zpBfvYNRA; znx|MLwa=_}JCe*zM}MaAs-KN&`*B^jC*vVn|BUK}fZ95n*>kyi3YS@aK3@5sJGB+s z?H|a@Ne%19jFdEhg)rA9?7P?bYW(&5&-7Y}#_&_l;?QZxp%!|HV$zQtwZ7ey<0hBv zW@1odOwCFiuisHF6jxHzmbxQK_vVk`wYNw$UoXee^+DPj#^$ZlD|&kPkxHZKt8HCz zi^{hfAMXY-OY%5KMdg;%o#fCSoYpyc{HB#u+3+VW`hwGkioj$gxrLs`yb|b|yba!WhPuv~jXP7D_{!eOvcK6q zx2*6QHZ$ce++$|Mk{ed$gIH)yh_|19SaZXFILP9V?sGWs9M)n!jZGI)Rjec2oGV85 zKGgKTd+5uPM~&G(w}I=L<1k-z>|o$*;@1CW%ObM4gn{tejf8Cjoe+HjE8N`brqDe< zRwsA!loa(-gXo&9!}Z8)!Nw$Y>jMAx%hd_XEEe9x1-~94(G4XZar6HDX)r!FxFmc{ zefzHIMh@dG0!N2(49bkSkc46}cAHNf5s53R#v;^d!+ExHC3y+jVH;aEio6A~5kC$z z-E!+@EYT-pQdIi$$Bib5EhDXskprt5=s7w<7K6Pc<<#g#q5iUseYrctV!|Qi=QL^_ zX8!zo`j)2MrC4K8N?fWgsE`*UFT~>*eKzedzBL7|V?>!-j5Q4$Ryv4#cRG@{Ff&^I zMk)JvC=mI0$IlY;e7VwrZ2J8*iJ0@L)A^r=96~aL;b$H@{ZlbxzO$ZPN6y~`DpxF< zsayr}9lj~MA59oP{Nvf<&Gjj5_zClw+WOStU(BMRbPK~P{~Mx{ij_W>T}#8+bl2X- z#F(Ph^e^BE`JTIed6@PrpM!lg?EJWCzvcF{3%96$X6eDmK?ThzwY|rAgu9IrPm&Nx zz149ah4os?1G3>lv@DiMV)DI!j&17TV_Y&bSvT@FIWLhXm_?ISO;e%|7I}a4`kx^^ zLA#(Sd#rklwDM&&wM_e!;=rkEU6R8D4E2NU;@GNI z{97$MX?Lu=yG`=FZhSO^IsY#IF?Py!&p|C-;ayFx?qv2$b6(6sEZu)oq~!a5)2+Cm#u;G5dn7wW|HCb}7f;bzpG8(0k{gh&2dwOq_ritxI#JW@$?aw-zR+Ci%(rkg27! z-6BfvY3rtwZEZ^UdJEYjjPJ;;lFuh4pmYu%b!3v)R}p|ywNDjkbUl?Hi%9wXv}JRB-)6C z)S5Xsyl&0-Pw~C9w<}K)2nU>}{$7*C)ePU3PTjv<8DH_Rv@r0>EJpcva$B}4K!LOl z>#-1M76c{S4PU-j`mPGQGH3kwIq5psC6?;VmT9<$450Uqw(PaH_1+oitWvdX9YLC4 z`HP?29lvF1%P{0Xj$U@p%%RT#(_6Sfe&61Z8`8b_vMa@^PySmfYGzLt91J(K;o!{k z-9#nM&4LUJ4Zk-x?L?AD~R_S4JS~N4|<1s6X#Qa z5wza`SA@ro`SIhp73c)irx`B1C3=bN-viJf5H7*ONNVDhsq;6i8U_wUEC^slpIccm z4f~@$LkmUmxNQF~kkwGVJ#Vb%kBC@0zIg1jzWS~`V5CHyG}FvF^ZG4KP3e#a?_TxI z3>W39N=ZjE6s(VF=6n>FqqksWDBDlRFQ?vDr8bI}F`X#aouMOX$FJhAJ}8B}tBa(SwGTq|ur z+rhj)R%x5M2URIa1>EdFZGTsV0)e2r4Ylk1917qzBS71_4`zUd=6Eg)H)oah2}0ECP-)|@fC5tp(qf(ucr8g+FW(?vZh-`v zT~g$fMR^V2iNJGlwNe2{kPuZiG3hB3Z`|0Xf*x0*>M}ll14}uY;Uq*WQ*jR{jK!!& zBfABUW`_$*AK()Jsf#o$3pNBm$Kg<5A2EWeZ|XPjHaNHh$^eFB0%RPHx_fp+YIYGA zpsE0eqU%`!>{e7{`)wF1aLmlkn#-Md=GTu}4J|p9FeS=lL9O*S!%scOV!!Vrz3>e1!@juX*E)ija&<%WM{&Y-}K9zbyOAdxQcJr5dWu)Zsc3%M5q?#ocT+;~YD` zg`i^w7#;=l-?-u6bl;DvK(A5v<7a?pd23)JuPe2*w`g17HksQE;@uQ=S{?cI>mhXT z1U2?{G^9AwlQ0A%>;GN>>=$r({i)p}AcX*FH@xsF!b%KA8sQiD4oaMZy55NZq0jh# zwI#A&a*{%U=es&6o1%{@%bR_Rf}+84m_Ldz?eb%&sV_czrFkg?d-+|C8}%` z0}LF@&y!-~b;@aVMq{Ouy0VAge-S`QsP{#^zGL^QU$tb-`?h(Q4|u+0J`>!th^96t z3!hBX*c{-g#4XA$;&3u_C^mVi$fd zh2SY><}2xMBY=((zlBh17C$lT=rfgxMhtVduw{m$6T6u7Xt9g*$K^kZIM;IsHDbEk zgxr)PD_KKHIf=gH6NW<7emMwG9dGv0}2k(zk%r}?l%*ix@m zP)dd0qqf6gEwSfCpC^Ix%nPmE7<9yeVD=;{!tMf&Ca5Fp=tN+r>#*H;VS-s{LOym> zg@3+BtM{l!)4pK2R6))?%HDcDxX`$<{Y`(RH`m6OmS34oB>45lR}hCMm`&!yVXc6n zD<;G_FEkTa>y7U}{Ww4Kfy$7!Xi4Ds&D$ld~?5HJiBx3Z#egLX^n;ynREytqjYp^KX%i63!O#nE`ofYpQ?l_zEO?DySD)U3kgYUaK$7wp&z9 z>D1vk2($bgO)WXU(;tI<)EclJlz;yRx_G)KYhn1&LuB48$2S&qpLeyrqL>3?{?RHI zI4@Yds;rjZ&Cj0vGhDP@IINbxrQd0)n^JtFy?@hwWjgiR%0tPI6aPnT=NZ<-w(jwu zTM!$H2atYHkxi2=NGBkuAc7#B03jmM5+H)~7PpG1^e7-mQ96OpA@qnK9U>hm0i=c! zAe0b7k~4Vj`EZ|mKin^OKF#w!)7F}qHM4$e{ons(tL(@fMM9vHvHl#ZOQ}M*`=D=) z#Ff$ptshE=z2wLxt!0W+xj&NiD()hF)j7`lHJVRo&ZDmM8F7C}t;&67&f$kezFmD_ zl30d^Mn*||YQQEPcTj~~TU}WBWnZEFkPr{sXtGp`kTO)))BN18p(1siAcI92b1LOa z>s`9&WE*)COWyb*?~@vOV|`;7NrjV_7VLMM;1=jLgxhBs-oLB=u0>` z4)#&({EAI2aZV0}(f}L3%&acTwB!X%O4M+l{2D4tBQUN!W;H#!p!3EykI*;yaJPUp zymg+W?uavbW1!@;A(Gpkp{Y(UM7U(E?mJ(+T_9R2Q^~Fj^c5+)3L9a*aN~C}ug?i)J@b4{G+e`_B z8(DAu$S8K=V%$voNpEpEuFMJw>t5W7tve-lXC?hDR2`3#TTRlyA&8M+v<`!^@v%F1 zT4Fb)lJZ8MS0|M|1+ex;&TeYRRIe?!&g8giv00MwxtvAAFB5q1DZ^&o&IuGTt=MBK z*W&L>XK{79Eqs0F{3n>scuxGaPcaWro?~5=d!pCV zheoEVp=8sz267{Al$AwU*9;siSS97oZ+kt*>Vz#(p;AaKQ zyKn*T(!0Ug^1l_{N>xq+8grwXH0wUlX66>~_D&$EP>n^5B&dV1BGDeXEOAEClu;!uW||C9-3I z2N{)T273+kP|THt>R|)YtJk@1(;a%|ThfW{gj$cL1#9;n?Ly;{{BxewJjpHIK}5Yz zE78!f8J%)TMq9G*vxsVnDW{1~9*@q+?t(GVqW)E*Ybl1KtSI4udoG&QJ|t$*cgTFA z#ZcF==k$}!f?q>>QZVqHF*-D4HPE$Ez$3cOH^P6*H>n4|QU4UWFuYM~>99Kmu0q?m zoOB%s0P?J@94vs74Cn%#-w+*PW)^wPAM-Huy8Oyw^1Avc0+R?xA(F#JNefvN&QDd# z!_Vh3_SXAlL)Okw->9Cpce8%2jP4q%cNkQhrL9R!H;v4)$u$;d!e(Y53b6dF`?TG> z^-i4*O!B}S5G;Oq*CWm(w zdW$EFOOzUXs?<0~&Ymr6UY`GbzlvL6u&>naq*YCR8@U84F*1>=>^qFPEP|^ZePG{s3V)L;}M( zxYFgzT$nEn{0p*oKh`>`p9om-u|KOLagM7Wa?vQYay^E67h8k|aw;!*z3rR^<^r4* z)xV={{uq?{`_39#+{nJmbGwNTTd-6Dd4l-W(Alk{+xogeK#&4AoUn4Fml*VX7%Ub~ z@Y}pz`P$vo0)JnX-i|K(g4XTePbTdp{4?-X18#p^uY^I_=EQ&5#>>tac4)={No_JWW%Uf-dd6>> z(A(Bqn9A^h4<(Mc^m{&E8@t`tx!Cy{&wJazQy5?-@QJ^!dOTkoS{NIX^lJlR177j$ z(aZBW9rLlqwOe%SZNZ`)p}p&-%XI7PAuMk&9I-iQ93bVl{sv=YjepcqrD&jZjZ^GS z8`9vh@f-FFp|&J<8H&<_TZE3!r@mubO72Fkuza~Y3xgCJc7T>ry@VdmdiYGsOqvTbWd<9l$l>^G%WntHi$yU-Ra9K&xQBO z_-IvZ3Y5DkV(jR<=r~{?iTuweYPDW-UhidOmT*}0k9H$CMKn;!@jjQk%MYiA#3iTI zzbAht4&RmSB$2{}#{Ue5Li1O)FcO@R9e@0t6#@#XX4l){1f;EI; zel-6KA9?t0UT~sKH)dj%cW*dz`T2XZ11?2g@A`H0prDVAQGEg$HD27rOt45swXOI1 ziq7qXdU43V>T55lv&j#E)7|cNHPvW&PDjy3n!tM7a?;&Dr^E1Q1+UedQ2KT2VNIqc z{0j!sZFR*{c%L!0ZqgtbZrIE&0AmSAttLcTQ5d{A%BMSXoGbDU8vdfB$t9BaTU1KF zF$<8P!SVz){)GGD`rf%YkMg91tM@p%7b2}>(9P{77>4?(^y8L!ovQL-f`hj_P+#s- z^VQ#9G|p(cZWZF(qi2FW-XQI2q4Orn5x>?mEIY*H(tLEjKs{_SbDmL~gPF=w3%-i9 zD_^wfl8dxzMTo2ln!r4s#n)cTHZ2t54vVybfSzgr&(%`_-K-m8V%jvCtF2NADc;L# z@vueEXxgx+)TW0&Nsd+FxHQZ8EJmnH7;Ub`W!34e>ENgnq)VJaCRp7SO-uFJ-~VM+ zK(a&(Rc4MvKMEg?JmEUZ`cDcJa1KX|<0CGMV`YS88o7A4{I(d_!65^KQ$mQhw$O`hz7 zp6uM-D#k)(+Fcq{9&i0E?Y3AcOk>#;?7namtS__ zs_&Y$mF(4$y)+R8>zLk4N`4Fds0WKfr-K*I2=hFvy7c!w`onEKyCN&5MmlmYM?P}i zNe(QcI9o5PCwI-ox<~n97_8QPRIyM|eEM$vN|!~$)otQJ4yQdiPU$TYUP3zC46wjuZsXe*%wm=_y;hk|S`~6$l>r#z?`H=+WQ^A8Nbsk&e5}8TW z{OL=ui7h}3xagT!7TV0^fmD7&a%*_Z+`y(ym((#VecUs|>1|Q=<6&)`g_fV-9U7>b z>B1W(h?42_FSKv<$oWq?7Y5EfIJfn%_Jq-RQVMujVLQm&Oi&O|RTi)cQSl2a06bnQBk)`!#gl$BW1?_@r?Ke@liLvIh>5v?=^E| z2Fvk1(*HQzf7haxg|VegOTu|24TAYJAHoDxXisuwx+&4&Q>QEAq%`)LzeH&$lwT{3 z>I!|PTs7Tot3VkxolI~FM#U-o(j7KK2t-e~R3mwF?$wGUL@x9URjXMfXlo^V<5McP za_UkhldKSMs^}EH+Gb)Wt%cW`M6wU4&h;rpUCZdqVCO z7jbvgRoHaq9S41X7-iSPj|BaB%iE2w-^ps;~IjWsI;9d3UosvRz2wmGTH@H@T;EX0#)lAIkw(0=mcP4s`y$)$q|f*%B3EUQUu!CMwM_#} zb#Oqt#1f)2cDT-M7JrsKq|7`nmI*$1*`ZNR&QmaL`LU^YpCmH%{qxh>U?n--M|UxTUue z7Y3?f*0MBb?-ZA${08wc)9g8(-@PTV&b>*IYcev1Cjk(~Q^)}#-$Q2u$u2iHpX%{> z6FN0iz65qtuT!l9#>Q}UbJR(Nuu!Z+J*iIFPc#f4u6uHOZ#l@g2XolhJl&>f9%k-3 zu(Bk$V?QdkH=mg<@{UlX()HNMX)UPHtNv$IuIR*HLo9ohTNx#K3y)O~!U~;{ut0~w zX_#j5SF^pYpNYP)Z-MNogL$z~dKrKe-^E96gv*(OZs}VSJP&BEuDk98&&G7J>vzLK z{aSl`Bd!zWU2^R;FR2I3Pbz#{dK>dS@Ur#WKp$bn23oIZlXXCDDj2V3%U>izJ6PdA zO*r5ZQvV7u!$pI;Qi43Eip2Hg6@5jEvhKm2?Eoj=rc#$wWIz(XY7iYD(85cpnO$h^ zsIW3hbz$SISspe;Z4J+N)pu89P_L@KWw4hg*KLQQCl@k7t1wp5QeN$RcTRHF^8*aB z*rK3mEm=npo+z#{4)d$I9Aw^h(5z{Uq8snU^CVb}vVjv+TW6MKS499F_eaGZ!GB}m z8S8Y1L6qShD}-r`X9bFVAI9lNG-VM(D_mqF(mb&H`#bM*Niiw)EpDG08nxCv^sG!S zSMUC*KTu=P`T3gzm5`o?F2MU+bTCeM!%i0KobA{vzz@7^gIHi|G0AmS^)uyHb6m*T3l}e zp4KXLr+~;;&7;T_4UG^$_;SQ#0W|*Go(NbuaRjP?<>C1w^bImqBSni}=AUw;Wi10C0pGypFHuN(@A<9JRnmfG1EPkR*}&M6k9 zoAr1X^i^0Nef3qT4Hm+Z+?jgeY8-q7q^W+vpg$b}vYfibK68^C_W4I~%bVYa0VsDa z81w-J)YH*r{VvSW%6}`}|1pqVMm-Lgt(-f3+AQ-QNBe-fYJt4M9PI4lyBFv$`~fK^ zbp3bfYm)u|f`ZZFkVGEL?-PUmYCEcr|4~8z-+>>j<9@+rIB?c!V4MI1(!XnTr&#CF Gi+=&+)F*8K literal 0 HcmV?d00001 diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot b/docs/src/assets/images/transformations-getindex-with-dist.dot new file mode 100644 index 000000000..8c6826e3f --- /dev/null +++ b/docs/src/assets/images/transformations-getindex-with-dist.dot @@ -0,0 +1,20 @@ +digraph { + # `getindex` block + subgraph cluster_getindex { + label = "getindex"; + fontname = "Courier"; + + getindex [shape=box, label=< x = getindex(varinfo, @varname(x), Normal()) >, fontname="Courier"]; + iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; + without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; + + getindex -> iflinked_getindex; + iflinked_getindex -> without_linking_getindex [label=< false>, fontname="Courier"]; + iflinked_getindex -> with_linking_getindex [label=< true>, fontname="Courier"]; + without_linking_getindex -> return_getindex; + with_linking_getindex -> return_getindex; + } +} + diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot.png b/docs/src/assets/images/transformations-getindex-with-dist.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..621b10a51498ec87050960725b6806bab2913416 GIT binary patch literal 42305 zcmd432UwF^_$L_kDvAOZrAU`19i%q_l_tFgBs7s8dhZ|>kgoJ1ARPh;y@sM9(rXBz zg(5xl4gtb^+&i=LpJ#VxXZG3M+0Vx(C;9T7^Pab#-}{CLZB1noVp?Jl2t=atQb894 z`fCCNx`w=c3;2Yc{P6=D+r)>M9ltiSvyrq+Y>-V>n#FR)ir?=i8Yl{Y0N{EnRPBrWcL@5P z&A?3e?|6>+h1kEN@A4lU{~f&zzYT2RpNovV>e{~}Mi#3Z|Bjw2bY1NYxO_i-Ci!=? z^Y{P!gjN1~^C}6C?Jw|$OJJ&h=lSliJdkO;Jm^0f?*x2~KV8X|Ilba7aD6W&Ws)r` zZ^ifIZSNGbXRvu~PNMS?$^N~&Q&?Qg8S_x^XWsT}_5YbI72m6(a&SylYz->JeJXtu z{4@GeU;K<_kBO=vI`Z@mz`0(ktIa)Ae3=2CIwv_L!<$6R@ZSZN<^GDEx(U3#Wt$Df zKOHU@Y%wJT-X`AR`)~O7wl|r_89vj93{N+>tN-kkyytN0x2 zSAVKruj51WzL8ISjqPKD2j{}i;fEs70rE^^1>xY5Etf4G)knk-Wp78sfMD4f~#$EK+;yL4c@)eKC%*FW02A*6!Sk_H zj;+S5waG)x65Gt0-pHaE#I?{qMFSs<&J3V9Y&+#lV4m5=>%q*FCWFDF^>XpLf)(TX z&?2IgAZ)bup|4-E8P*fd$RuZAI(4o^R$rrrc-7yu!XHGBdJ%FN40A5+pLM+k`o0uV z`UnJarE%vv+Fuc|_h@>@vW@BM?@UZiD$E;ZD#Z*aLbfG*jPxR+vPWA`M?H3ekc5;i z1Cu8C*ip+#sco9Wd(!zd4`zm%cvX^>{2iT=)lrFQQfl(&Y;B?0Tt7rSR?;LR{n}C- zx3-Va{V1(o>vSPu>a|$yh3iY+Q1elYyG*k}8;2DaPjG$imOVStGR0^w@7#Ydu)ugSFGnjyo9;>w&X z=DLZwOg8+ZZrR@c?nj<7oZ6I_liluiC9P9w9y*a*9*27AN9uB- zcdC0CXH4YHeP7CK@H5PuAxGG{qcJ_(c}Z(1T{6ueQ+Z?Mu0h89Ce!U%-kIadC=NQ5x{dn z&`(#02X2>_(6+{>Wx+i4;@4B(kgmK8-5f1R;+rjcFXhPqZ-3Kn0*w)yr1^0_x{sY=3m`2?^F{vn{?y%sAjLXS$yeuVzn%> z)SP-Jv7nB2v{tpmxW>Ye;K>VrI@Yf~+GDG%=Ds9i$|w#Ki|fe;;R+O z|6Nh=rzcj!uy?jmS#K*yU%39Ob{MNXhnNQDy;ev8>hwjWHuee52CR0b^dL+QPU8@) zXOb>2NwX_ev(LeIzQYwmztcQM;kdb^2S)!SkrW$1Mfbb)G{2%rBuAKCG<6L&s_AWo zDBbnWGDv1MRnC)7MFMZo^_^)ZU+lBQksnNI|~lp#5)$APVP~!G5A7)* zuR>b(u=V~yjy|F#IDxj$_^Y}3(gUl1-lfX>ux|617n6`j$f>Oh_Fs=GqFBp5z9|d! zU1<~DL`Y2v`$cc1G4M#mUI)nhVv!|&-MO2DPE0*s#Aj>To#$`BeTZ9M9{AL5oMvrk z5z00Nd_EukWCDWv_iPh={Y0hchi-)TxvHVbK=7XvO3L?(o6c1(;vcTdD{)`UUq2b+ z)bMpoCulPqZcRnLy$bj2ziU~qSd*qIz0)$2*{r=i6Efh~t}IH_R>a|t_k&vjUpm>( zVZU-(>L2aL17=q*&RlfEmB@*ANVIEzV@3WjdDKFeGojzX_kr*J=f$g*W-Hu@;{v7l-2{`KLyzyPUe7Z@|xiX@}ZN2COobbT<>h^LVW|o z9{?m%bJpu)aMUMq)@IXtAkYqnsOjvt0>xLpP3~u#}UVOFqWivO9;3(_Q=PI$zdR3?VsJ2m-JR-FH(c% zwU(Q-8Bz*onvFU^Z+qSXKE~Bgg-Ovnx1!2#{6`Oy_AtBsGqPm@Neaj(efs4?;gt z;j+ui^dXXo<-G2}**W%dh5l<8p6t6AH+vl1eN+(iJ>V)m=8WBb7L>KBJGOVGzoL6) zh*BSk7oR~be>OfGN)1c8%>DKb$Kupqj~DFD6(b8NZ(q|)E9B}#7>*7@g4^aLQ%GG- zjC?dE%QS+$ye?bBaqF0{)za|EQU|=O%;}5(#B{=#g)pK2eb+*trkLor*l3QHIhwkX za#W;FBKh^lTUkTRAp#>MYL8QDx%WC4+*1qHSAtuDVpb6m{}(XR1Q19pn#bC zCbe@J*M7K$Z_W7iu z=M=B^b?gq)X=Um%!d_X5cy-uIHk%Hqz3iZ8^7f*YQRNdd#4W5zdkst5Ldr+FzQB`? zup<2t%ph*Re<(2dhk~z$@u0VR7LV-@A2=9P|Vppe)x&NDNbE#>WHZ0zVAEm#71`(SpsPJu3kP_zO4}Pmn5!f?L)uJh zh_in%QHXlB#P#^0tN}X2bxHD?VsdLsDU5d|dRK2$;3QI_d7Sh&pR*! zv_rn#7ns=qkJg)3{0WvujBn`qhrCJjZ?4f;AuG4Wt@bQ6ZGE}vAM9|H=8p8-`_6{9uYc8J0%5#AnQ9JgspSrrt-9tMl3t#jq%!-c4l#iO z5D&8D;Bw(3!x%piRUcjz?U9_>!=6Q#(aYU(PgE3-h`ptI=ta&4>n_PM>c_T#M@lXzxtUXg8;|?YURu1P#mBja zOEaF7a*H!JW68({;A3OrB_XVO(#uNrXNpY6gD@(~OPE8g0ZJD645SYWYW=IzW9&MX z(Gg9#AUBdo&Zo~_(~4I&$ver5I(%6_Nlc6=-HHF$xThl*){V_?*9LXZ}>E6fVToO%|N3%sD>OneNSn z`2>}gjU-FTKm?{e^9Zs*F-Mu2hDsyM@QmQ{TXIJ&@x(I|jj;~HCk~#QSQqHb;F+h8 zsG5mbpdB?Z zOCh>WEy;4rUj5RQCO)zDRh(H!y%|2RL~Kh2w#XYkvOs3v>yy12k%3$<-(QHlou}Rr za$Mp35FbzFC|UL^F##XcZ=3#P*CCVN(jT1L#2NE72ygYuha||YE*LMPSz|{%n7D9$ z0+U1_E^cVnnRG06kWlKSnE9=@dC+{-SIkQYM780-wp@`)28xoxug%6;wy>s5gdv9Ts>Pds1RvNBRcGf9 z);-po;>qi<(=cl?c3UHr)Qz)WZ?+v3-uP^z?#p`ocx2>d9K1i?K3z!9*8WrWqWF$5 zd#cUWQ(&6!*V_pe82A#3k$z~b(WZHgo zv#Q0qEf_haTmok|*tdlbVEl&~c~m{>n)GeE9v;L46LH5cGg<7?Y{C)b`> zr)zd-HfnmanpjfXW=$~d#dRk)FS1|kBlW6A8S=$bHuVB+lH)DM93~r*_tav>=mrOd z=82JUVqQD-5*^`V+AEMSwiLpvMuP{fTHrGa@zo4NqSZr(paWF$Uy>Sn{4~(X%1~Om zHAt-_zj@cD6l3Xzx_vTe7B=CTd00ksi80|TTZ7O9 z@tWItANA_iT20(Rw-9JGvVAiQnq?F`n6~nb*QW+1FzO65{t7mOjOiDXPvE|_ZD!4n z^$iTGl3g?ruOJ;oIxwE;F(a+nQr=M1 zk?tkUPY)$9$;rAlIy$YuewI@9vw7O_>0QRb!YXJqy57w`^3?qURLTd6?w*IxrWCdd zE5iyM+Qb}t96BJsJycba@?-b8G+O)`W3PeCW^HE3|5b4=d~YEljRj ziJ4Vyc!jJ^pDXN>w=|tL#j*R^!9m7Szghf3lWZRLEH{0EejR(LVu1(-_w>hBIQH-NDFA1Gh~-$ zkJQ-WOH@)#k9L-6@jvBp7Cvh`GqWDqPi%iXEFBN`9N3o-6}2LA;H-Orpc{d-@I#n(V}Jf@obXPq`Z`{H?M!ONgOp2 zj5oh2>-oK)-OXP2P#azS(*2kHR-`X@ow1~?mRR>&KjL@hq(fkUoX4Kktd%1 z0nuWO+uIr!t&&Paea{ZJ9=aVh)QpYIhGCMdx1C#_tkE5Y1S0&kUKI;wZww{z4;cRZD47|n@-7|HFs zVpWYgjn;Zx(}m8@7mf8(ts7soEBRnN$au%f3RGyHu<7>@6svQORQt2XHZupuzF4kE z)B0{cfFx=E+O&$z+YziN)9_p`b|&V{yNSeq z-1H9_Chws4QMHg=swQ3$Vo+7_>E`%e_)OlNoegRZi$gd*^d2yC(u)eq**o3r%O@|M zVPFy2EM>)8#K{|)TPJ)iI&4p-DDl6G9h%$t-7VQ__~5Y%YV3gJ=i{Q%9XBNJ58QB0 z`gtT@a}Mqhws==91|$crRJHMr71H49;tZlWzbF%8qwy|1zsL1xwx$F{nxbyl^hQ(4 zy6L{iMSr=7Y5ahvp$L0gAzM2lL8VZ?{xd@^TW4pQ7jJJPdT`b-z9jk}I+6)h{-o|D z*2_m(!{0gGW{U|Cb~-9@4~a=hrGy!ZLG7cJ(bLJFtYwQ5 z^&{3#%0@;rFzXLNbCgM&N5mj)ovL8WNI^iNwdz1>5~NDBv5%TSDfQS`B~4Q@v(PHt zT`IFtAVrPe&{?g?gw)zRv6GLs>TL3^CyB7tLn#z&P)1E?pe;k%c&3@g+N6GDw9@76 zdCxcjhP@DNWu?=pBfVN5mfuE{6-p{uh@V0UDqWaxO3)4SM=5CN1ifZR$a}G1y zkq{}ur(xL1G$LPPI%BGYuk;8S5C(?(d!xe#vANa>VGf&%*nZ=*% zl}3CkXvLX()zQ{Z_F|daKQB$M2~Txdr^l-U5l)CbrME9AtPrjXpw1dyWpZnSwrbkBgm<3wJqB@DerY?Z_{APoUa(K?M(eE_w~Ve!|cYh zNYqli@=9Vnw+2JDPP+eoFvAT{b0~1Y=O?nsHINln9Yow2sfe>(oqyc6o+sEqE( z+r;V(L`^h**EJ@=K?+9sCM$SCq>>F7*9|76>c+qq5UNH^dUV7a?;CmeM=_0p`rT@_ z_PgC8@N`M@$}#??pv5HjdJDk^H6bC*eRJ3RakYX?O)4L*J2j>}oIo1bJ5|cP{t{Op zH2dyWvGWD2YH?!>A9FJcmfEPYUT>v#Bn309z7DBWiMP=6{TBDtK224OUCp;4{V&PG zsUP_skljVQ!5{g7@RsOrQaH@d!ULJ`13hSLougFQgJkX|S)r4Sj_fozYf+UI9zB*C zN-|oQUVa|vzW&qgspgbkq~x=E^2sl*FX-;)-|MLS3l=QOWE8l*+Gwb>80LM`e0!#F zWs1U$%jD-mp5ux^Q7n@TbJ2W})FCWj^UZXzs6q$>J)*^29R&-7-0S5b-Be+d<1MFPskSiAoKHB_HM445a1=P%|dD#0k=l?MxF=e?)%T zNze@$4Usj;OHHJg6xMRsuIi>0-(%Q3)vI5qD79>pjnsZerG64MW9s%yWrDPT;Z!kO z*0qgr5w3R4aJD$>V+~FxXW(}DqL%lw=e1r@*CcVjEUYBk8FPH5m-u-z9`yyj(5YqLKe_2o zNMM*Wurq&^v~$9_-YBXT;IpL{3r}L1q6i7&%?>K&%xT(n$m@t}Td@U|$H`6Pp8UZlBp=YP8tnP=V5@*C)SasX$seJ5t&!ASbFLDUNBS;= z=_6hIEI4jGs{6(wa@i=x-=^R`Pybs%Dm2|$E@4LYnSDkP)YEF~JsH#+F9&xHC`Ht% z*&_RCU{Z^GK@y0F`-aU3X@8Rv+UMRA*I_V=^{u9*R*a}mZ}&d(`Gi!l`S$I##d`mI zOOn&GheG#CadHhK7mF&#G4TIG^q*xZE>B{kL_l1M&`cdp@Z2L>tzt?HF)&kxbYI_(D3kA=n{t; zXx*@2$WESf#@_{uaRNh|OwQ;5d<^>Rot8Tv8Kq&^Q0e5_F8_1Yi=cY5K$nPHvVw*( zOy=Km%K0%U4In#o$I8639a#4<1NC@1&S^nw~YA*I*d zJrKz3WORJGr#I#iS&m?khG+z7;-*& zj0}|WMA)J+i~f)dUVDf#Hh)_1@;L8N(CE%Uh{L3*SB>r7L#(iL-QLp+-fyL5ulD3x zWDrPN!`&2fyBl6~$2rSw)A9tN#r2hdI+FJVv0>`QmgD|H2s+LW5;mt3xG7HCgY~q6wv2ER@GDBY8%nR)d{omXqP;1#iKnw z!pY9!bb7UA1mQS5Cs{tc)Ty%D=yX$Fv8rw)Gh~zh^4(OU`dQ#9_qt?9+gTLk$wja! zCO7zVHgsZNW+MCCXv%!+`$A=}Vmwuh17a%qKxcGY*wdl7zSrYU_XKQ(ReG&p0SM(s zjZH0uwh>ADi@`FYex;g9W^k4Iv7X~!n)->660%jbTf1YpQ=Jq~}$X-9D z7Z$3J&Vu54T|Q&kd+kGCB|Ss}Pj-ke`9KYcNu5D7SG^fY>ha4m?oo`HO$t-7g*{&q z5DUh6BJ-8@czLf;qzljWI`;>#j4a=mU%1TIF9Zn1PvJBI*AVY}58BO;O&Y!`?Y+$O zyAgI})P(XF=;o`UHuZ_Jo6-+FS+FXZ;Nm?w*}fCS^u3Pf=(cs5n3iWJx-(~g6YEt; z@oTgs*%*Po7lZ@#wmIt>D;`drY%*AZG_~z=w$n-R^4#U>ik}bc+A-e}ry>tyO#ZIA_Ux21?o7yk8fbZ8xC{$A_?{^1|9$G_Rcp}O z`?;A?S2a4IiuJ#)m*~HiwyX}l0kRr%b90+b|GJ{;84Xawea0^M=GX133L9`~EB#ZP z1FCnJo%!|_pg5PLKXV#7h5+s*(fzM2z+KVz{|i3;Ki1@hyxLM1B?AN8X{v51!)NaM zHI)&L?7(=J;Hf$%bgcs>%HXPBiz4o-f8bmEr>H2D%<(iU6BCmWZzBUK(2m0!8=FFRcQz?=Sx2c$js z;$gx+(^gD4(n`ALZ+Hh(_D0i+!p5zEDO`8|>2K3#$y18a_4fAuvMo(diVZ&7q^?u| zZgbwbYP@)>ny(zk2-C?^`m(L8ps+gWP~P&1{W`F;3Sb%_kUk4ll%nPDXs#ORVyrVJ z(`RNam=m}S;wD23UA?5`HbvXm+B%FD>zHv~^#k&M_;;&oLpkqh#hn9Z&To;>ataBJ z+`XFi`^SH$eF4n*rQgOwYiiRQ>xD5je`3iWB^ZLKs67}AAb$W8r)!C_R*%^E2?FQV_zW47w zp#X*MjV1sWX$9YWsE@j8K6>-2#T59;+do!TR-D4ZdWH7ff`XM@ers!MJv}|#o>{$1 zy1KgYXVTSGRf$O_mik?s;OzP$>bsH&-Xgo9O7RNN)# zfoJ(Ww3CV&zAMF-8I-zti64-O74;P$Wx zO=jaWu2T=QDAvlBU1>3Va>5hQ?9Z)kCbQICn)lT2Fc={pPD*K1c=u+W^%p}eW@hH9 z2}fcQ60h+0eqI!`wA>l}g`YqF?bAR_LBR%$0d7n?!;ppgSAWHG0$OBfk8mzQsQO;h1Jiz;QAg7!E=AvU7#7I|aQOZ~pa5 z;fhuH`3>*e6B8)_R;|wcc;9iWnflMlN)aF;pVegQxlCyW3X1&CpFh7SdjIYn1u(9t zsCe;S53nx#E4MtS#UdkP8W9n(We7;Kb?xVJ_Sq$WO6kz{;qkE{z)Rp$fEXMp1u)Pr zH){z*-JBhCP}Hvge@6GPEBQ52wAVhpcJ10$MU83_H}$5^K4f+2qYogdo`C@aRn^G& z`1t7`dyOc+A0Cd5j;(9Wukq7VZ&#{b@e<#?tM}?v>d*FeN=C-5EZJl5`1#Jxj+&6f zEzq292jvqG$n0}`d~O1V7H+XSK`IDeqnRnSMA$CTD;mhiPE9Q?D=XV??R2iE_P3nL zS4r4+p57%LQb90mOKPZl6l0!V;(ELiB&miM3GMm7|-sobp z56P&p8_w-A;6TH`th_howjxc7G&6OB_(es*MOxY3L}x3v-2L`}P9%VznrkP!3jl{b zTL;5Gek?J7Lttht{;ePc;G$qg)rDs*ICN>s3hp|9E-(XnV^Sw1B;>g^pE#MNr0y@o55fV!7zOtU*85O1_8$(u9rGs5CDIZuiipxpE+YTe4OZ@!#+xYmaD~*@4 zt)9>a#B#+Kz;xWR&uB%Vi&+R@@-wbhpwG?ustF{84G2hpS+9K$y%8A`16Eg$e)HxH z1sz=}kX)<}<;b62uep-9Gqwq4-qwm}-cvnXLn(W$kM(2H7v7Nek%A~Yoa;-yHz#4H z$U1;z@J7|{?ygkuc>_}Ve7-&0>tx5G1JERZ!%~4KuC2*#fF(eOzJ&tJT9GYUZUM;3 z2X@}u*;$lp2n2}oM*#26?E%B>`2D-dI5+cbveqF5U?tm9tWPp#qzqUqGCH~sL!4)i zDkQ#hXLoNB^lVA5K$Q|)qN@n_FkbhxvNB!fH)`}4tjV>|~CsNKpyCjJm( z4&)a6>>M0w@wm~F1=11x!CJz$QgpH%NE?Vl8wF}9{3|UfU%!6cCgZ$XZVQX7wHsyu zG+aNP;{N?yKt)RcQ9-dywZLKpVosAgPi}#p6}$j8+_3spv&b7z?A28}!2C3j-kT#w zn^V{Yygp#aPLnk{ov{o6u@c8ISB9UWMhfC|0en8ddVMRaMZax!HnuA=PYEF55W@LB zp#>0e5~2ym?X@;?B?YT#anM`Ct4x4{?PtAAgEZT2rWXqb`+uH-UH&A?K z05UJbtF=s%(1NSg)S0b;jaH9GpIT=nD`mV}i>t&vlMPbw21!|18wp#=?EtCdx^ag%X!VtV@Pp zFv+0}`uP}>Ssf~Wk3B4tWGGzyPIg=pTQuu4@p3paoW*P5Pb;K(Qy#0G7Qk%yiUA2U zmht$nu4x^#AF-+qcy||dpdf7*lwONHyITA}V;LHZEfgCIsZ&}zf^=kk z!pI14a41WAZww8}U6uhn3%WSO_;1!?tkMf$wDoSd8)erUm)$4ny`C?7kgu(i$H zMDbnpX(Ymb z$A~0NO7$|(y5Lq@Napnx(|mWUS_t54qyd%HJe~c+-PNl8J$VXX z!?qyAj}-gIUui{6YL)nhD9M6KFIy%Jz0U17J?y*3@EnlRPGyoq0%F_KrKPttSOYa0 z%OG77c%qX5l(=B*}xUxYOKRx$>kSOA)rPM3K9rCP5+Y#DU2 z%zue&od3phWh35|paO2Qj&^QPzaiGGzqf|!G59=ryr%NqTcxTLwWeVmGjKkuyfq5q553Ekn_r0WQ=9~8RF|e6!1UB`k;|ME=KT_{o8->!!sJ6VPu~w zV+WfNXRot0srHng*lv{5?%Qn%)%VFThp!F!TxP8kOa=|>9JJB`*8crf$D4;j1I>YF z>gX@ItpY*i%QE6Gu;pbXf18`%#6Sx*Z!iueE`=w~o7LV(AH|9vpS@GE4=zk>Jkk7! z4Ed)^Ix+8=Bo&sNJIdTV$Wug~i0VR5o@JfuX?|1s;lrZi;Mf#Dn?_cLqV-1Uvl6|1 zDdNLwo(ibE;!&Hji$ILdk>qS@^_o#_)x;>iQA6smm#C>UfE9e_wl=yDvlc3F%vsI5 zo*bv9Ox-^WJ8(T@7&0ZSI8E02msNttY_#lxj!v^>{%c!Yfm!{Cb9MZfn|+FVnI^R z<&mP`!`E`BH_<~$GM#IOOmFVer`Q@9&*l?i)HhS<3N|Gbc|FsB!yyh;6gLyFEyM8C zS35(@Y2@v-8zNpf{i~pLadwQ^Tk35wRR=WS<63Bre0Vw#^bZ$#0K8xvi0^%*3{t7_ zeKRcq8Bx>%Vjk}9&8B6lUG{{7rNHVl`Raa$fo6Y=Q*5+HW}B5}pSQ;Wt8EI;32t{! z%%i@#Y^h76jmEG;giX)Z>}ZB5HT>?GtYLHGDx#?{&duBCxYy+fKK z&{j&E(RL`Mjd!R!Fa3izw5=`uDVO`=)s_8S$27BqOF^wBbyVM12hx3>^8gYDnf+Kl zj=pA%)$x<)i4mx%adgo4=E`*T%c>)>WjQgMq+slx$`f1CJ3=J0bbWDg)Vgkk?b9df zYA76PDA3i~hEVVBol)*xm6lAq(AUA`m+@_J-2mof5GlpqFV<^XS1grm@3dRf_~T3c z&h~PrxMFJZi@I|1Y4q0yW_SgONi4UrX(BY6*PcdbB_r=0ug&WuSJZajk*FiQPqqb` zA?uf2StpW>^(++n+S&K<5;J`uP{!|QnCdeiZ9^3-r6Nu}qthX>f$am@`%gc3sCY=w zFaP{_w;#wK)8hL8SCd|9SbpSP2?9~t4Sba>3>QpOQ4<4w_XAF*BK`+S!-}KT3Nif(u|_)j%P^}M?IJ0? zw0I^MkRky`%hhmi)h6fDx=v@OnJi|tm}RJ0LB-87sLvT^ZS5~8p$vTrX_xd-v>z}? z!QG?DlYMTxo(q&m_M8m_d7NyBey9t&XM4W8ljC7m^1 zGr~9;@0Y@ln{&L3K03)G5sqjDgHBDSNwd}KDKe#K*i~TZh@`v4t9I{59YIW5n%Moa~ z^cDbaXl=;54vDwo7uvop8tL4Nw5Cd?BMPDG><N0u+>I#pNm*#FjS9H^n2UmovBTWN7fH(K61!=+NGHAs4nm7Mc^7cxjE z6WtSx`EJ}N5H_hR_tOh^N7FJwY3q)b+v=#Nhdufj1eDUn97?Ox3eDh##ae@!(5c)sgd;{CbKIaizcl=Mcv7K07FYP4WyX6WQ29oA(2LUh zk#L@D5O0s8$bZ@V0^K|Pg`73qA<@rQB6tVjZoSDjjGrGgc&OD^Wzt*wt#nkR~5Wk)$ z^-WMif2OqIRJ{wP*>C^$IKLkix_ONcE?r~ZJIho~mrlgVkN41cIIbFllqs<;>Y1;b z(pB?vp!6#FQj)aYQXs0rjKl9x zE1X-e?HvnR+1PMbcY*7Rw>}Hv7*fBn3#V|-Bvy?Ljy22i0*+&|sVCDi(Hg5-)!&nw zKUGH2*X*)e*rQ9$KG9kr@ZLgFc<{{^#dr(By_wI<-D~bDg?gGrTxo@0HEp+gm*RVq zraHt^5a={frMSP`zkh{GCfx&0Bi}bu%!75T ztn$;voN$}Db1Z`A4y6|)tOxa-go0z+^5JriZQZ$LpToFI0>AB#kN;c8NBjrjFh$v9 z7U$Xizcy!?1cZIxX8AisBb@K&JTu<1Ryjg|Apt-4pSP&iI*$Rz^IjHAL>~86Q4H?A z$ntM(c7e=-4h~cebUCmCf?PcU`l_+cy$N1C&KY3hYcS(B7_TIB+gY zbBpf;nyJ}jqN=7L*LF8e3k<$1zBn$SB`ns-g-W3ByKSKGAv+PDN7$dI?Z{v3O7nvR zq-cyaeA%{(<8X55sdBEPiga8Tz(imt^;S7g{FV^8m$usj3%M|i+}jYFhhY{(&lR6d zyWW`oUaPIfjBz9Sg*CQMSIkf|YHKJkt#vl+!Tqe&pT5sa-KVe4qh0-rZ)n<@`sTF> z8k>P_GVANsbgxrsPMXcjxzZ#(mL;crTKB*1f88E8av!IrZ;y4Fst7WdQV;8YXWkik z_zq$smYI-yS^UK=#&N^w=;;_!Tg0^PO=8`U&SiFig^+svCjn5AFk_k1fuE+R*WCA? zTjAocF>d8S@)J~$s2{Y0H)G!i{kX=vA3`_v2Jy!^^t!d;@t0KB>u0Ty(6xnGW%VS( zs-!v9pI(VH$o6VOu!;msMMsAmVcr%ykSV<`MZv-{BCK9TPX1+bv_wzGz#xoWEm@ya zJa>{JTRN2dIdYDn=I)Hk?>SfUZw<8J-Q_a=TQPa#6AzbCYPC586<9u5J*W^he707Z zGVge>D!KM|p!>zJsL2aK-H@qRvgMc$;R&MZY`U*%nLahnn!G4Uqqb4^2-)M;G>$!b z_g{Jeew#fM%@uXsCTbpQ<5Q1O_&b8l)o;mGU+;^-jk_lV%g0g?Bjks_pLkXXh5Z_3 zl*tOXTXd`WuaKf!o;tW{$BJK1W`BI<8kqmJ{ZVvaX2eIVP=adV)s9f0&){|0>cVgC z4U@)VF?E&6uVR9zFX^qxz$0eOg1L7c)$=P{89$twcjk39%gsPU>A+wMm_Au`6-$I0 z<$7DNXCog@(^&A+Yw@KAfue(ylAV}0M-^Lk4$L1b(-@@ueJw}+)P8#ELly~3XY5e? zDscf)GF5)9piH12`k-3yvsa+)UXtoZ01U)Q(OgOW&*KGpY zYilK=>0YVYE}h*W-88x7$E5t~q1x-oNs>IX*;rQdE{oaEMi|?PM_~&eK=;IZe`pcC z(XmYL_vEy}d%gX93{n)ce4^%j`K$IN#8@oo(OTq3q~+h0LUR^td7}>t>lo&^LP+RB zWTTIX(!slrZ}OkJT_b*&v)@cThk8Y=MblL`1iANyLw&-Rn4{g0#kGNs9&Y_PeIn=m z87lQ0=f5F&&K#za9cXNH{`}lm?XP-!!fL<85RO+`nht=Te*+mT0NIc#D=V9ZD<~-V zE3t{p>E4u+lLL{tBx)!q^mleAa0mjYkz#%BGP4%i%acyIM^yXG_e6af$K}e23K{7m zJ~>Q1nSfw+MOdv@c3W3KWlh55#sW%j!rtfD(vlc)X|^tf&R=f69pq>D%|czbS*ydH zbh4b`QsKU&`jYK_AaHJdC}i~s(ayq>t3*Xk9{Tp}+gsp+9pj`=pR#sF^|a=JBT%Yb zh^$%gsdYnB6AK&LUpH^wY$A<$8C53ey;i!@Hx)HlKl7qJ-0Yh>Y)Y9tNHLPR*WgF? z9kI;&Ticys@cMb7ONKs+cW|YSj*i=8WZ_7pG?jo^G*C$}X$?%fBgVo4qiQpX_*ASa zCMNc(+J@xM!h#?GN;h8|O}Tn||5g)d`2ZZpce0#Dc~z!UUiU|By8fA6-_grLt8&;1FDs~~!?^FzAb0(8303I+0z(@)Jto_7q&q_V@$+el8 znI}N0&g&B(DOx@5q)!nMKY++xX*(T9# z_3}1X{DJ*f*&te=?9iFSWB4ecdv0?xhLXqN^`Bo6tCO{*2zkI{Z*;QJk&}zQtarv% zc&_R4nl@1=Dk@$v7XazWA3S&updop{dVm;#WsP~HGbR@B#HyjB$yM(<&f zK4d_V;P-ahB|@`oX?gj@xb$g+a*_NnRVxGNq&l~S!(0kyP*LW#}I z%}?<7>mUl^n}EwMB2-p;tY`-A`~Zr7hK7dsXhm)Vu(KjiMw;8&8W7S0yv$2?ccJ_D z@9(T+hg`8-A_y2DDlk4EYNu&Vc}h16&7aZ zYXDgFDK1X=#f$6i?(QH^Y;5c*7AX#tItp5$>9Y&~14jSZ(_?E}+i(EieX}uMiCfv4 zZ7V12>Ju(MeY$OBYkLDkPftHLKYvYJT-}5P2a7?6>gdAgoOQHWHkWRx-nHx20+sw(3h8g+1lEAu8rK}=jV5Ib9)W| z^Ya@U(E!5o4G?TaRn#>2~7miQLwws0#x zA;Ilv!-$Q8gH>GIPzGLG>s64SQC+?N;Ev>j2ML8HA}lN|U;tL&a!XE3{E(QKsH##+ z1pFC{qWa28o%%{=XXl4}Rc0vBK@nn~3U|_&;Z_Oa#<;%iprbM~^JOwIY5s@tD zdn|p{gBf=dkYS`1ac1Y>(0kNI6gNUbO3J{_jsqa~TK9EY(2L=izttQ}Oxf+#-Cw>u zn9TF^@s>0>(mr2B=;d7KZ>rbt`1{?4ZU3nU9N#A zV2I?qiK@iGN_7H0f#n<9^`5SbuU%a&pdF6uQ=*T|&EI5ZW)9qW{`~pS>hqu=blCe) z(5OOUt67$d0Hwh3_V%XXya5k5V}?9netMUGWnF#!sE8eekN`7aPc%+EGdPu%l~oq= zhf5|TCW`XVy_UbEr>7T12k4T9M(Vg|^*g5mzMwr~JdJ~=xw*ObRaF{i6Yx9i#QVxh^)ov=JG_Sv zA8KoBZ-mSay(1+fV-ypkJ2*Ii{gLqVyNivD9h;F6v$A5l(})AVr==Mz?)+W+dUmp} zVcyi*A7f{0YiwhKLq$cklMOv6Sc6{fd>0*QsqO3u9e9*&VnorTygEBOFGD{}0*b0j zJEzGnoHecyKs&Nfo>pCkgRJ%YcS=o7%~-iyQ0>6*a4b+2V|#o20(EQ~8=I?CR9Z*L zQjAZlsswUWZ*+hAW@2G+iI|vJTSuoEHt9r2<-@OErXXz+1?s=RtFhj=Az@;|01vOM zs~eG;N)3lFyet5AvZ3M5pFe-Z37=V;u_~!2v1ELWA0#6q0}0DdNG)O0v4FMM*eqYy zw}FQqQb$yT(K@8dzfVZr^Il12ChFF$Tb*COs$3XU(UtF}-zo#S0m$Gj0a9FQ_A ziEH|&JGV5_e*DniS%}m4)|j84uX>Pm?@c~GWvN{5J$w6-!}aoV-aoUm7f^ry{#Aw@ zUK9n38K7FAzJ43!c(iNx>h)`hT-7)4-xH$vzZ-RxUST)Nn4ReF>cYMfE|tdo1omZH z*Ry=At4l78zoN48Syfflp@TvV0TfcJIFKBPNTsUswKl8Leu7b&dV1dtrTm0514x-* z*FSf3{2n#(Z-dy1Ax?G!OmHSRCXMUXU`f(FN=(rn5Msb-s7K6iVh|6P6iQ`-*h(?> z6PIcw;<~!Hyi36S*4x{$vpfpM9haP({5h>CCHf`Y0MJY3E8!wenre(ud$e#wj0Pr% zGKRT;&Rnjpt`<;?kBtogBVYEtDcXO@N@4~1wxRLDW zxo9wce36zG1tRz?IGBNp3x9Q@rUS||?~R+$T-JVk`0~r+&TDnOd$E+E+8iM@H9`#^ zKbHBOcu@;GqD)LoE?>SJP+ZKJr;ri?WwVySLAB90S71jqTHb)>gvrh)$X$!36~*@V z_ZJoxHeMXab$s$f(#3@z++7(25!V&o$N6y!2n1(jWO#{bSKCs+<6gRX^JZ;*z1iyb zs#@>8l9E)r4yds3gmb8D6w;=y+y}&PAkT8MnQLonVbRg;u(dk)ayyI=iq6l60?Qlo_3Newe`oiB9FE6jLD}`zA^;Cu zXs#oYNq8=qaU%&yLDde^v=B_*)77OEv>gioSNSqXH5<%k3uH1*vt6VRre6X`+5Z0T zn3+F+Fd(vg78;uU{=M?32^@wBU;Fvi2^4mGmDpPu4}uta1_^nbm7hF$Vi5Y^+pDA` za=U>+l0$t8k61tZpEoI3kJGEBnLRx{9GKTW&Mhp2WoFW+iuuz*`1wlMiSg6%--k#n1v$%Q z92`jriKqXh0{!38ta2Kthd}rY@asG_OoY9k_x6(_$YpqWc%JHWh3a#4 zK<})~d(Zw^NJtB462x?XMf(gHnDAdtIIdx=QpCvx8HaX2 zZLKhv5hiYKLhu~-MoSEaXLG>%{gc1#tLM%|{ehR42pmCQsgW22oW`)P+6^8&5KJ{Y zD_awFeSH9kEJuLQK)CWNQOgi?MPG8Mt4w*uT(cIC2 z2}04?(<2Qh0LZbKHN;N_TM$JhR8F0+ zzyn4|`@Qi6Q!}$Tutcx1Ch-eLx)>2bH2fMHYcMAfHDkPRcpzJw$d5rx0B!GBCBm@Ksks194 z50!sXPke4q^C=)ol~h(HOifE;PYlY-%`Kx&$R`>f8#9Im8*+U>&up~d+%!HrJy{rh zOMom6(2Gz#Ms5dTHskr?l3)LI?gQ{pmjBtQu=AhGNb%rDgC|IwwS`6AGIw}#F|DW< zy|D0AByN|N$8Bh6Kq?Xtxk+**G4b-gn9YIZej_|T?r-94&dT5oFfh;4S0wX>#~_4d zLV5uDmPK|SB8rI#DzG+)3TbO!jfjjSA|ym1U%Gg42GYNO@j7@tSQe-=3bCo8?Ggd)AJV9L8?}0VmX;q-126Zrs2E zA#Q~{6>_FrE|zbdoqeva>a<+2LqmI7&ZZ*nhnJW^a^;2lFXqc@psoZM?80=T7|3uL zln6lT1g&r7|2p=}nTzaaGrnbUR!4d>Z7SqqJ^#lDJk)dJMn^od)TxjpMc)YEn7ikh z6#^w6&7}Bru+m_^_7}2ae*Ux4Qqs~8cB!AsYromKJ03uXl2X|7=g;)H5Jf;l0;0hp z8BBvuKDQu=rradHm@qddfkiw^d>{t%4KV@|VE}9< z3}Ao;p|Y@LF&pZ95cA{1^%?Hmp@ue$jG`iby6>)z58#S6wzh&GR?GHJUn;#Ub!sM7 zp6D;;zEgZqd$=&2xc((iyg68{wsMc;#TC`qlvBWhbK3s!&bIj}wN3d*>#V}HZL#)) zxeyoa369R<#ehUnTd(bVdreLYJWZdRG4ChFcw=Te+V@iAOxMP!zG}&mq1)c?;NPYg z3%DAF8wN>YMh4yS@iF3ppq}zO#o?0#oW9rY3KoP8-5FwZG$Jm2uK7_)uH@uKx*_r8 zU8jCrbB$yC3ym?Sgsm;>cHJIDmchH?PbZoqofh8qT&$;2{ zqh*nm6xx<8xQQ7ycS<}+=5~B^l56Q?+wJf8u|vQ6_y^|P?9KP0V2>y_yusJyhXcGC zsSkn+DN)By27oOW=CE|2f?4joqw|BSzwl4Py~D>@IW@9YB(C!+(XjQ=8wevVZKb%LR;+eMO|&HRdmMFg>;U5chT?P$7YSB zrQg%XRc!o(-ALVgas9dsXA~rqfq{XJGc6Yp4-YW${a=wc`vA^^L@{uN!NI|G_-aa* zS?^YnVrs6m7PIfEsP3aa%$sdYj5@K7RBjr3!^v0u^RNn;P{;xstMnZIa5l+E&~^h2 zFWh^d>1U1@>cYkPv*6VX=9Zm!ysI+?!`zuyUVR{Vnslt=LNP`(;>H^mu@&pROCR0n zO(>Mz^Si(@L?X{B*J>Q^N5r>Fwq9m^GFw0AOfa)sIQ|@y#<0f~aBKD|i9c9TJgyU? zS=i@K_cLSEKdKB4XQLZ$q-hVYQmL(*L3!O3CB`4PR7}PgG<#mdxcN@6245zcx*3wN zO^EA$4{I5kHC1G{)jk5(&Ac3=MX8^$B8B>H1Y43c7tS^{2be`BIBI*Ik4Mk zxQ%zvwwF}zG6o5(q^Dh4*CQ|U{+%IPm-|f8Jxa!g-da;$UYxT(MvL&Z8X5;e|cZTO@8Z}(giT=08j^Ypy zAVqo$;6|eW(%20C6_SxQh}Qe6?eq*nwJZx$LN|T~(#U=-%0tN&yfd;&P_f+h^CcM& z3u`%T`s(<11-CvL|NY_!de6!CW+~tGY>q$7NvJNV)Yk%ZmZ+coEZ8mCXJH)5u?1-R z*)L^_A>q4>jeQUNClnsvyDn?`9Ly%|Z?qEdJnAHbS{x>yWzVw@A3l^w4&TkLUf52x zY4zVe*B}w&^V$N@9YcI;D=RFpzdledR?x-&?m;3uv4cK- zBc%vuRI=VNN?_%d|C+XqL7drnv@Ia+d20d67O^QQk(L?0WPrVFg3y*Bk#Zx-^BLM5 zS-I$my$l*J(X0!xc~yZI7|k>ZhHyZFpC{nIt< zzc`76a$9j*M<&$Xdoq92_r7Y{UMSj~BP7-ToGV%b8)ljuI89Wsz<#{hdhJIqN1$Bp zvtMfpKSbQ{+5!X0vtG!Exhc#X>x*?p^v=|{Gdk`Z(~ywfc*FKZX6jo}?a3sURO2Si zfamPxIB9?(2Lm9Q9uiK7@R3k@?psQKv7V5Jw>J|92d<~5XB~cpwJA;~$A#h9X{XgL zbk=1*wJJ6k3JOZRp?thWAnY%QV=i|w!XdT2Kq-bvDSxb|3^Co`H|3CsQY2p+{}A}Q zT41=R>8pedQ_$A4x5ln3EX*ZmgyG9{qMW>evkN>6s?LO?MLrok-=lmfM=<@Qe!X2c z9ej(T4;)XJeO2}2_sqDc#M8iiVp(BtG^d!x$x58MXh%O6GG&Z$K0a>SA9lws^w=Fe zQlgtyjNWv$!^`cCQW(buigjK)*3@Et)R0qp%qD7Wz_69c@%|=6)KC2qSUb)?rXtE$ww z3$d4V2Ro9qS`(6!heFz&1yw89LMbOF9wcvtGATVjf931zW3(^uQ=d8&wP#Ce>0N#J zoW9KU-ceKA)y8*5QA`nXhw`oYdp&$>$fuJzWz~+s3cvHyM{YN+v394p2@x&#)8tmf_SUXsr<*obt##c zm>>(MhlCw+5bZ`kQK*za8p*Ett_dK)5JS@QO1{n1;_;Ao*Pcig*VmxRo;en2*kud3 zGm15r*B(12oOQC@cJ-(bGbg4uc@;^n?lG(T&-ds;(sTjE$izg&372Oyc=+H!Ga%K= zoPJB|H~P^)#rhSWCH`D*Mht?56}iKhb^Xb~U-OmkRX9+hmNqqIggA-Yd)Kx*U6cxN z!vn4!;2fi_!=pk{DuC;#cKbXLdKNmH85b89`*n5!;JqUy29YrNi%2RK7M3B!f{`C+X_)VB&?R3^=cw&lIv_LZ~VJ_%Z7pQBhYK zMxwZ$@HSIqqrir+tuh6I2}1>JRhiUwI3^kHBG1j6KCXrHx= zk5f2*poVRMqTPAsk^kVlqWewP^VVZd@Ybze(8&*cZ$&L3&~fudqNq$o2>OS%Qr{I_mVK*$0`0Z7FDkA#NKZ~sDi zdisAQG*C!CPiTe~&U*qDZ%l7JhWHuDlyJs@K!xzu{?upz61mLSaT zWkW-TgXJ1ymPIIXT)f(BiP&XjWlQ}MMQoU>qoo=i z84fv@l&$T%=xiuaSzkpK(D@T8Hg3x!cmUA5Z)?jj<-2D9#qIah31PCmualEopfCa} zs`Wh*1PEvJ?~WL=g}ygS&I1s=pRXW?=lnAC;5h4`(Ds~4s6_SFV>ljHq%dUFG(N5a z4r8vH0;ID$NLv1%A{KP|Xra-9>;PlXc|N$92H7t33^ds=XBRRdZOxFsu9g&k{rYuA z$Z(NPN(ep;HZ=1ffn|cJ!*iJueR_UZ9Jp)9vuDO&?1bIds1Z@nWrfNclC}cYRrtvq zu8*(pW*fOtCNyne;Vmuq&KHhxt}6p?po##6acE@+LcV>Tgb$$4&gOvpkJY>1w6(L_ zgmk?LDmq9x6WX*4mjjXW$s&_K&rI`YAIQnIKueW~j11{bwLqA({^^J--T%~cY+_;= z3@!Vg`;4rmzl7pSoqghD5s6BnuNWL@9<*^CZ# zHKdRVqdDqHLFr=th6eau7@(e3S^pxXmhlQpB09Pl7zvswD*Y4)ZA2)TB2RLB>~p?K zP7EXl%@H8|Dgh&b+5zMZcLdaR0y(qNFJ*9)}qmlhT}k*f^o!PylQ6hNsZBqfE) z(0f$?I$4)s&J|2kplZR;<8&Pzom$uBmoFK?6ic=uE1apB_6vfdZ{IYMf_9-kd3H2?20^bxk3ZabxM;Hs7Ekfr=?kCKFH zW}@YUg1o%4swx^%cLc8s3V$Ew>Rbtt?&B%)P++#oAgzEtHPS$XOhQTC($*FikW|zn zE@9IdQc{<|v?30Ach)#gb6@WM|sHu45rTxOy6H9{nwakT&(lMXkXAGki zeVz*V#)6XHyv!jUxU)0+MFu7VRz%Tv`dm4x`o25ahD}6mM>+cQwSSS~-iZ`f(I8&f z_Rc4nKU>5ato9p=>R_k=4*ww(0yae&ze(`{^#=Lz?&VMql08jxAI+3)&DgRjyo7yn8lld-f<+fpq&5%MSQ`hQ26+jMXJ`@A>@i{+7qX_x{xapLywyg^Di=?VZu(c~plMYf!mR2@+(SxSg#(^usAN z?LB(Bu{J{Xl9%|^gnN6y7YCECA5t6N-WU&}czMM|ciYJJ@`bC!otX03eB0HXraUY_N&+~oq&$<&2R(yy)ui65}Ag!mQOmGft`<@y+2$g59Vsm zk{e8UexO_X_6mlClfq@7^&^Z-Pkr0z{C=aE?qZC^-bcd63}oGU{w+hRn=_Mq*9chD zrwC6^Dl0=)jhQQrob_2bGs<`k#%eM{jdh2;xYwAwPq+;7sTC)YgCoU|V2vg@ex%XC z{-ToH!as>2pVo*W=B3(7t?YimpG|UsoJEmAW6{Uuwv@$u!@eeGK9^YePi}oLa0aDU))94?-_z=$B3>qtQd+S!B3A+;b9Jbi{j}0X(T@yb>~MDL$JPhR$)?sJwM;?{rJoiqy9o48u#Ynv7R6;kB~gyq z!GlaSj&n|mRFV+m<2#{J_l7R2vC28tx=&j)J-K(qA2{LLx7mKqdRk40?&0W2db+NB z{c{iT#%BL=t&x5p=7YEIozkfu9lR~GpB2o>=j%d~``*(`;EoepDT;zL;n=yCPG*ga z-n_#_IN$zfOf9yh^>R1HZ$Wd3YnKTl_ROlQ@?S0{*3y=d7iI?4zBCAYag;<<)<>h- zcFpH#bv$(3%E_{UNbj%X$)EMlBoy@V4idZBJJ&9o%N`z6TouzTNzI@2u=-6J=G7zB z8TYkTFF$}$jAp*k8?)8!N}1ibNGeCa&&kNGDVgOrlTG3cH3ux8-~P$0eTRFEU?7gL ziUeIj_5D#wQ0IWp>`0pQvi#sq=3@D%f`;hTju}%G8tRShXi@STUR9rx9q^^OALdXdha@bzD zM+PhIpu6IS<(w$wZ>&}EzVumb$yGPr2>yfDi>(@0%*`Srp9NDg|}-HMuYvu>{JhI(+*eCi#F!MBW{Q0 z9qAaH$U7`*GAOJj!MsU06;(vt8g%j`xcXN| zZ@#AT*HgVxx3@iZOVi~wofj!`THeP_?1&uUE{rDH`kR4mg_B8dLTqa?#{}`$F66fr zca5wdKP@%GFL^Ta%zCWh?AI~zpL!Z9d=>gxf@ zuVU_9cweykNb!ExBmEDCm*1@1xw~K)L4blTf_MNuDv%zPr75eRh2~+KRN{1EO;*Y{ zBNj_w5MfkabZb^EFJS6r*l0N|zfrY_z1bI3!)XTnv4{g+`lfJnTq|a#d$0Y%l>1em zP}k666<;#xpx-TXWd!-I{`nm|>Z)Okxu`h#OYuv)Hcl-2-?rPmA7;JZlI3bwD!U}- z-_aAsE*?|EWA<~J>CLdWSkd68l5pv}(NyJ=l<`xxIChn{U4Qsf4de-uX^6Z~?LuvR zy86@bAT>95l)m<1_0LDX^Tmm7^AC8{h*4+}*pbOUy%Q<%jtr{I8jEH?5yDQmAQn7# zX#zYseo0d=BduGfg(BW9GK5SX)Acu`K`i zVi<~5LxE?d__8YTq8U#l{gs*i+)TMMca$^Cidt)5*R z&$)Zsn`tnej8JllzLD$p;r;IIxZ5iJ8G^?Da}S>$LS&G8bK>6ZRmHU3>5$5Ts0SG0 z+kd9$hiv8xWgo8gm_!Y&SH5xyR9pC4`L$PoS7hB?ar1z;{Z5=oP7nHZ9uAY;_K!)HzSsO#a_ z#^2jx@v*^I?lY(=g_`sa7}QS0#;5Oq!L?U==y+{Lr0eu@6VlV zjuejf?UHUS1?e#d$_if4dvY;>)|;Y7Iwn0HznNuJKz6Uz5$rM9UC7 zyeyIl`0)=!UUpNv+pM*vANRzE(}G|*x|&S&YRdSV>`)N{gygFx=KueUVd-cZ=JrhOe#*6w1N;zqCwvyao9{g3rpI6X(td8%{B>he6lcFX zIk2lp%X{M9a)%Smkb8XrZDK?ib9IokK80IXC>dj z8??Kz55CRZdBo%AL>1EZ0Gz zND02$w_+aowi{w&^W>_Q>xa@YEH%OOmMYa~R^P9A<^y|Mb(!eKEl)W8!MU|!BV1;(bi%M}>qXI9R~`7k%s|sH z$u;lkA(vk9|78NvD|SwIaoCCB{}?KC@BRM_72e`0w!`sNE?v!Q3KruMzrf^@zK7;R zx%?;bsVz=%n{ugORa{MQo3f9~0_MB4-*{TL)jHqlb3dBW8l&UIuU42`dpwr+vW@O@ zOpTOd%3RD8^}RyR{-SkqPLsxmPgdL)`<;-8y=*#{2kgkxlMY~JZ9`cKGRgrmztLp^`^Zq-vT5_HDyqM4jN|za%2W+i-IeF& z_#@a@TNyl&r4v_svpJc3V&-$>=*-(Y1~P*_VFuGJUtXYn{xqE3zlL*fr`7HP3S0lj z2GOoD+FcJz;cizN{6(>U7qyDue$M}HX&+AERM{%15W`WGmF zT@sq4+{R6)?d4Gdm@=uukINtTSSom1DYy=u9V@#jDNYrD=$`KXCOF>gpsRITrGz;S z7+S=G5l5&+YmLr8NjseoE0BI@@7rv6#edbqw_5?vh5#f1?Dq@!#+2C-uio+C1_TDT zc^65g!?fWj1Aw%x6ad!_y=^C8mHph-_6C_|1L8Uue2RgQ*;Xi_A|F#>f+DZDxHyV3 z_WJqkjVqIfmHONw0+*xt<2$hVXNSVlv#Q8VH^In$XMCgTLBjy0z(3}KWd52 zj_1Yt%gkhu8B*vP0YTt=3?Y2sGL(1EXFI8d0{}4h3y9kQ?B5C~A>=l9PPPZ{z~rcu z>p5iGN{TTKL6;(cZY-<ic>*uqG z?3&fFGIXfp7jH8?23lJfFsj3U4IdD9MJJa}6G>!of`|NauAB;8ot^pLpAxsUw6G;^ z$tUw;!Mx1*r?+L>p5pi&Z_1F6k!1l6SiABu5kQPQZ=*qnj^bq`G`gnpJUIDf*B+pfI80WnIYk`1=0a9=e1PW#a&~jD7K^9xVIRS<*xT2z> zL=r#(amZkc`}!0cX7vnmoH+#ATyooItCOA{>cb};o4sk>@|yR?YTjch~HhSK3jZCuMmRDej-VipRODhVSde_Y@V<06SEDMi32{9)4mBfJUN z3+&x6U`~OE6aL`Q-qF@%rwd2@j2yrwue2RkMcBsybtgN620T?U4kW06Ekocbg??I| zNo$A|?EUu!&x#-RHLx?ti-QsFMT9D^b*2xPElZ*0=pTxU+PS9_jB|Rj?ypvhLiJ_F z0{16BE6`EH14l$hkAN6d!t@0^(g%Gm7*J3IqGd`EmwamcQ0W-j;+LFyhqoW0^3`Cb zGBF`R-Fha_>391g0O)qz!{C+17i|!(;Q^o_t%64R2mO*O!Nl-Y;43HsM$?1aQmBo{ z48Ow%AXP6XR!3mE)x1pGWmrS&KeV4P;JI^tp=T{Ho;?%`qwBsfsSOmM62&@f#cQg<_Kfs@805k;*DR6>Q9w6Hj`zsg+;e-sr zBp@T@LVA4^RX9M@!VPpSD7sX(RA^A=z^vL=pjN(qT^gPTNWD{aYwTCo2`^+mgF#xD z`BAj5(E|)UG(um3`!m@7k>z*+mC0bL01V?xvCy5?0bG(gFllR+=oi2U-Ol=SW4`L* zeR6|8a20-K@7;)+OKZ>EPlY9dzK(&*#EMiHj;XIJN+PfSRex7l6-_5D~orSw#*TaFUup!`mo% zNm&V|BH3=;(t-5?>GL&g@`l~-Hk91P0UTtJjjb4{ObKJ;k^Qy-WoQkUBH#)iKw8ZS zv5`?x=4s5RToLC#VSpdgQ?LCA9p{LMh!=y4>OMXV!0CTX!-zud;1AxvfD(>@5oq=s zH-HEFffq1$K2$&c9m&`C9)zfAd1onty(AXCrmvq0Iz10th{_+8MZ`J1>&q8J&q3gG zfY%`5722GO$ZNDq{3uH@uFLrNBe4vUtolH$iFYnA=&d z?08jqZdrg@aq8E`e!@qIb7pM-6Fi={PB1Y@jfSbqSBO~v(lE0gdqG`!v*jXJ(nD~= znQ-Q)9TFy>4$kk0C>^Y{_(th@Cq2tG*}it>5^$0pf=*eo8lHo#fcl4nP5-B(FmS&j zl7xEf_zF$|6|vDqlV1-Eu%MKMs>HuYxPUYM?B74}16?7QEXB7KpGhh`x8@ch=+XlE z&sj&|KaRru9vj9w#m9W8{0H^!>s-l~bQ>I-;UJOaHhG4{0qJw1Zxs^e$62W%mt7H!qYOsUt8luW26Re$Eq765OO`K?~SEZ|$X+UN!m zDlr~x|Lye#*myN$@MpsfG%#7r{}WhCumi=LzdLLIz~S{PoFySSxe)B46&M(jt6%{x zFo9V@F+Tt_3lPsE-$CFHc4O})04OmCMy$A~q)rY797h1G!BsO+4DgaCpL$_PFCinN z6b#GKi-%k$tC~X;(|$+AK>bk!$Nt5%?D3Zu*UYQM1Q)B_Z`11phodJjtTo)rfon!UPDLA0Tv?AkM>et~PBQ z9X6-O2QW3J9d4Jzp~DE=djMK6H#fflW9T(MoFicn{=|hppb`cO@5{*8fb0R?e8T85 zkS4r<$?3@N9v=;%ANT-fZIsp36S!)BqKWPdsVxoUMk3sdU@Sqc6jFs%AOrEcLkY&8 zTm`YS>Rr$+$OMIj5$^}fqoqm1*5`EFW@fIfctZJrRJ1I1H3t&i+=t|xh7H=H-n-js ze;$XyJ;AK=e*{n`FYfPmR=tQ3e*8EeJS!QeZVmvF$Vs^zg{cuO%6yt1y_5x*4wQHz z49>{@GueZ6uFE9As9afDsXFo9RuEWYVFY1mukp;6QzjzhF;Hbt-gbM3U`;@mD-3IZ zOcVzCVf?h@D_6Zdn8p#v8AwSz=3S_ zlZMt4fq`hV^YhEV4KoTu94-(->PC~{eSzsUgjyEh=|w{rDK5X#e*jr75}o>jH%z|k zD}o5LvibS*SHZmJAi&q&L72xo@j9;n0y%^R(+J!g=l#$SM&L-UnRH9B|G1+GGbD*R zIg&46;@uS}T!bZ7%E^y`Q&28AZIi9|H^6>;XcGZ?z{^pixD;}qn zW@d^bM8^o(FTzMHm{QzzbQ5PH<7@(^IQ8_ZZR19l6HTs?ej?x6+7bXd9fY3w?rI~4 zZSStDr#DJ`hXK)?!Zv{QnkQSRTx(au;)7R509Eo0;KX;UHvkDjUalP zuRoa(Gll#F#2{Ym{O2y?fA_(nMs!p_$aUcE2NS*rH$?rsBV^o!@(%?Qi~T?lm2b|K z^mUp6E_KNmh|y2+FQ6z%@*>gx`-@s`E;W6B`)6fBg6Qdm@~a`XDuX-sXFhzV8w`!F zzwv&HMmJt3#Ys~lU5}WF1DpEr#jdA~1-Zpx%AsrQ8)M5XXKM@cPes97%4eiYUuC?z zEUxk@o?=witI_?@a`8UAKbvtaU+izy?B-o;``Mw}nBCU4IH>$yWRYwv{xP&4#3Ur= zPV#@BM(2jS!kikQON+7UCokyg+$(G-l?tAiTYBS(N&4h0h?2~Zogw{Ym9h{96*lcH zBN-ddQDOAaER4q8NRh1a-YY{aZaa1JL{Y=eC$kRAi5hlL8rq?N4MFw9R?Ud|#+vk( z=Di!4vW<}+RBsmYA2iQR#Np+u{JZ{S+kF43LV91pn*aMtBK&9hTAs^m>h&UYws^np zs0V2HrvE5J8%$U{FubX6Rl1+Ijq@PU=Vj&;PqK&$ysYYQ@!Q)SP3v(EFn|q?e445^ zs-TxgHSfbV8guaWE$8cl?+1%ohg1AN{>yLc?B!a*rxhLyjor7pzbaaA_7+Qx*!61{)3ruflC{xzf@=#){%Zp> zSGMxl$Q(BJzU?pGKCyFqA^oFKc2w-(jfG!a0rpK&jffV7nEcRhu_oF>Ydr)B4z$C^ zaT51!R1!Z$ztB3;VqVBzxlTz&mHqQcLyZupPKDoX<;mnwHs-AOlH*C6@8$DR1EZW< zc{YP7gCwJkDY}B#s><5z2Kr2L|hgeHTpvT#;9*mGD&foxkjk> zNQQmsosWAd7Ud**ddvDYN@>Cbf;&Dg@qVW(8HWn;auaa&A;~T z&;0GG{GXnzWLpt>lwYs#%a^Y7_OFrp_mACO9cFIG<)RjT@Z>a!tAckk-KW?0i^D>{ zUTBfSo|`rGBdy03ic5UI+&;dyh*3IO_;caqfH1ZzhSxBT{Kx?NZ7P3I2Dmrk-S8n==buGe^G8slva0e@enJk@|$n<`RlKab~3)n*!X!_(R?5i za6X&k99D>Wx8&2oo_u@FRKV_sgk;88=JtvGao`7<9qy79Ki zQ+W1#AD^{!O6f=A6+5u|b+&{}&iiVHdu=giHhfhMysT+oMI@)*M1^Uhr6L9fk_MY^} zZ)x70Qd{@l(o5++qE!@Tn$nmn^1h>S*X*K^yioQ#-rNEIcjRKWeuj>7nljz)?IEwF ze^$*k;(I;x)*I|T%s6=Ll5a)gFUHig;@B#@C0E~_S&P9}zrat;opko5p29Zuqkrqj zc!TTaQ{vIqt&aySiMvnQ*)G0XT|e#XzKP8^DxW-5DTTYsaK^v)(z#AC6|=~5^K<>T zeT5VyJh`uys@yuB02AM&2Xx#D=}Z@7s4&N3B|SE ziVmOKCB+nkKfb2!I)!*fR~ST73KJxLBul(Sz+z^2m*EFLTIef(-;=$$i-Oso1)nk* z3K4j-drmqVh;a^f(AboV?Q3zuF6X;yz4p{ zdVh0UR_6wN?ecPmnFoE}6>e&-qzrTb`-m-C{`UOJiwGwonxHU^|_zL6& zDTL(w)@vMia^p3xuIo_3Z5h_Z0}tQrAFLk_CRhz9Q;Ub$-5%S#VP=vSQgmYT+!eki zHfj`P`{JXe*hew%AK}yAqDNoUzWnR?`i=81(dMBk&Z9^8l=;EE!aFmAOxu4Fqs(_K ztS6IX)Q6idj4U)->pi_OxXtRiHe61nz~7(7KFHtuFdhv@L z1dqS|TBIb|j3<*nsu!GlQ0~(2`^A&R^!8~<`=tjG(Ps;tU$hk}l)Vk9IrRFyF_%Ui zju}g>y1EVC=usw-**MXz%IRNUPe?pmlTH6SP{Y2tOIuFvlO2rHc;{ADYZ2YFhx(IG zJsKX31u2W<`NN%(BNq%;gc9?+5aF8ZR=!PKY z8^+Pq&z_H;9C(Nx&WYGfdaHj34(|GTC(~{X)5Y_rA?M+q+{v%6goVyW!5T?Z{Dw)| zs#Sj`V;(;Lv+Wa~|BTbHit4t6WUJ5fnjcRziBGbedtY=XDlOhzKjV1#M^?1PEotD% zR>B9C`=tsIs(qVEJ9{!@j~$}8rZu!mg4OxN26hHVn zFy(HqZnM63=%ZFAZ*-WqQkO5IrGry#H8`3pm~V^~{QF zHB-JrI{Af(k<^}KVN2mC!QLt>#o?oVPh<PYGp zjgGPKyZ_BCKdzKscPL7`C4B*%cEDyZw8-X}!S3}*QX#YQOW!|!uivUCPTcHvDK3!JCR$_^BalDnpvu)LDHkfL7pu?=9^2yTd#l6Di=y}&c*3e&X9hCQ@O07>t zRSG}fa~`(zjzZ;zr)6GCvpm?^ zM_N(q(>e-4tJv5M*|gE~+g2|NS|l-D#*#iOdY;@!CirWnRVj?Dw+IP909H7A{&}d90PatF_>MvoUAd zy_$77UgJ=>!n#Zo-t|E??0-gS?bGJi#!sN~r;lODsv>_efGEO>D62Z?Qeo&fc&F09 zpPj`6s!UBg-5X8a!=Hxu6DM8SkeG^2W#TfN{Yz=etCRW?xgY1gjik5IzMf+8`u*X) zm!t>}kMvyl0T1%t_*RNM5}h`OoM8%w|IyibMm3SHaXg?{uyA)5P*Gqhil9ghNN)-f zI-yrtf5hB$0#`6(ZzL^z7a*_wJr^ zpOX(WlR0zd%p`C5{h#;${9>j&rLDZ&`(oE_M<@<9cR-1H{|Q{tiym;I-+yqIu;clf zZL}{7{iA}4njV`dXLYdX`1}=4Lq|>8!}xcWV=ujF>U&6^ZqIZ_n`T)k2+pfSF`#E?p~klKjn_+Pbaa~HWM&dVnm#^e z2*ibbJmR~&XAf3i*2!^w(=_X3F}+R4JY!~YycMA?s@MGJJ}-a9=8a-F%1zrlp4{oA zGb3EwG*~^yHK9!wZlt!22XQ*vET0YMo8$!>BU^pE9@)k0kFLCRouPAcWHcY)@Y`2> zS%Z0*y8Iy-KVk4*FkCcZg=Z%A#{=EWYz0dHJkmb2R355B)1lW_xaK#5u5Y9OASu*P zBVkniyO&DA%7ZEx#0MYszF(6LwmUW4USFr8q|Ddz{fiuW7Q0x7CQ;?2Ea2DYK15!ao z%p-@BZEd4YY~;X-ik($owJkXtG`Fzem1?hRHn>=uu_}@ z(Z;mhPxAPlcs6e)EIV3^t`>WXWIA7~j$Mz>yd&jCij(Nl5M@bS+`6#-RJorK?pQzbQhQjuIBp( zASS4De-wrlxL@lL0s8QZRxtAz=DTl)`4+FJ%y0@ji-mU6_V!ohFRDKoU3IxoEIp`DJLkt{Z` z(3#Z3Gw7Ja%{a`o3p}3 zj#Ktrs79HWUqUa5gw)}fNqI5K*lTIH?BR3D9@{PZjb1ZVkgrSplU6=%l@}ud5hH0$ zgb!Zs9#&Vw7bl&}`{0|*u_1`c33nf#&f|A5HTCGX_N%obYcB!uC1zbfXn)b~fq+A| z5bhwd{X8|GCk3079<)H1U67V$xtPb_X#5garOocJT8&>)f^~&UI1PT9`F!)V(Bb5} zZ?7q1)3@w>>(8e3zO|ZtL3tU{>J-E*Gwc*T8EBLf49x%eki%FR@SuVMlxb5kS-$2n z-YOi`sVd34)LK&1}ih*#N+*gvGLP_m=9bqcbzVA1)1oqz!SNJA5i# z{32+k1vSuD2h%!M-CMMzUApsi_IMJKMS^7{@$jOxo2+%`Qgz1?>O{$q7)-<&#?#|CIMvo4 ze`M`RPgnlR%r6M-mkn5pbJ|mK2zxA9bd(w24%TzIaad~j+X6G3APilF&*_UoS;Xr1 z;CdH7@*GJU*L2yK#odARe{3`Uc8(FhxCvR{_N$xk-oe1$*2Z~u`L%0fG74yW51C(| zoO$H39!ONi`-&o>hLk0yv|OcMJgDkT>CwFS#4TrU8Fo1r-^Wf7-qpGFDmwPs)u^v0 zDRE|}?9Y^X``z1IE2NT?6xP(Id85ioA>vik7}-RGTgaG}axa=%{vNKhLwe&vH+AD9 z#abxraJfwJAPp5T>5>f;=LrHlUx|$>AP^r`g#*e70BHe`rEsuYBJ6`|ySfadhn$Ry z3cY?Yq7|U)^*0iunaIfRkh~Hd=(7gJ3ok&^G*XLfPid&DhW^sWlj1~49ot0Hbhx);Bg97GrdQda6RrzMmTKhE;D!W=NGG19 z3gVgXP9?Ww!9+p3jP?csWrFHRxV$ANVRRf4a(L%6sS#_*7_S<&Y+e}#haTd-g2&OCZ z(Q`8ht=iZ(ZYxJvwDeT=B}xNB;$O~o{6DdMZwIgx8RN2f{#*>WwC0sA!zT0h8R-0Uf zp_Wh^8~EXqS$9^g$2NqNN<&knm=sq#$s-t2N5n$aILCnEhNdGvY#M++<)jy^4#B5~ z%W9YR@LP_h6d<1g;Hs?8vOYOo6iE;WtvSvSv35cGKVN(AX>?zmtYntTmaMB$sT+0W zX5dOyJ$I7i;gfOrg`Gr7rk}yNhIjf-KApzSQ7A7Qk43=o2oFW=5{B?E*2D_9;~9*l z!i54i+mTvzK0iEQcMSxJg)Lqtm8=8i?<-GciL8lye8J#ky9e(vlS(Fd%h5v0;FOr6 z3#bxIZhAkb3$PiU`IY|6v+~>9Y(!uV_qXaBO(^+c=<`9=fodfyDwn9HV)g86q=E_C zRcIrcHA`@k<2+q6QED}g444!IF*FPd1wc#ga^42>dh{X1MdB`0@AX#_5)R; zi{=5CT>~tN=i}`l)qfz?e~-2Y7;He9DZBk=UjVT2)35d&RP6!~wsE2gBe)Ht{^=E1IBVc55}J5D<}L|`h`!I`SIfs z{@j$l_t%O13G0ve?@varname(x)) >, fontname="Courier"]; + iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; + without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; + return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; + + getindex -> iflinked_getindex; + iflinked_getindex -> without_linking_getindex [label=< false>, fontname="Courier"]; + iflinked_getindex -> with_linking_getindex [label=< true>, fontname="Courier"]; + without_linking_getindex -> return_getindex; + with_linking_getindex -> return_getindex; + } +} + diff --git a/docs/src/assets/images/transformations-getindex-without-dist.dot.png b/docs/src/assets/images/transformations-getindex-without-dist.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..e18e9686fe9e653086e21720b0bfcf8408068247 GIT binary patch literal 40798 zcmd431yq&aw>FAMNJvPxfPhGMg9r*pNq3iYcPU6o2}qaHNP~2<>6Gs7?vDMf?eG7; z_l`5ZbI!P9+&j+qI)u%;H!J3P=3H|<&wL5_ASdw@l^7Kc4(_Rxq_`p+9D+I=+@ty@ zNZ`)pE+sSghGZx$Ar1$H{rl0F8v_UT98OAHRM{crmqK#>4;1>%0*J z_$NOZM(`9Kz4d$ksCC|(X-ciEsKB!1WuE$k4&xKam+-x)e#-PCQNKQY`|vm`N;t9h zkmB37*YHxblqiCWLuT!@j+=vgm;H17-Ww^7Pe(ar#s76>jWOcsz+wM$Mg5X{6dL@m zD;c-+-!E&V{%3C&9tgV6)z}y6Hr5sPQvK^vq^xt=Zd;8M(cRgs$14SICUIF#{Hb*` zbLFP~_h$?y3N=Dm(U;WK{TwZ= zk`=VEh4o|Vs|9nAOewR9Q$z^B?6DuP~ZMuZhUs}T5Z$4o29hJ z=elzdzS1*i4*2bh9cA4`J0bZwk?6sqhc0TSy``^4JB^vwoK<|jdaO|bZRfMG!NJ|J z?-$|07>jwCSDHN1>`6wCLrEdu!zCp9-mf0b$ZwZ9^V?e!qpPsLzBH+mS}axm5U+ni+RjN87vI+}Q4h<8*1&!T+c z$~YqCJxACaxla|fYztKxR539H1sT;DVNk^mI(-rp!OJ4r$~0y@8Ig3|)JWLMg>s6( zAr$L`V@F1Lp^tG@^;7IiQA@@BjPoo1IwAxY&j5O;)zGntNLl^Cz1Q(D*+=7Vz~#1f zKY_2f15PVWN%~2=se_|D@o4e8XQk<*q{afwHL>i5`j@}cIncXWSa~@R=xwnHH!&yp zv=-(NvkSE*C>XVCLK$&V%A^yU`2whNhVWph4~98@6hnA6_tTx3=hlx4eEXP>KI{Zm z;u7u~6-kV~TTs74J~>C3OnknYlFSfbYjY~N$;GIS{+OdHjvh2+c?Zp1e zgs`mGxAXlk;)DglHdU{%&$o@q&F_#8!)wwL+L*mxz$}RSFGErXr}5~8Ryz_tw_$?-?<@0_s>*i`Mk;g7t zA#Zhjkc9Iazo(?TX6)`pf{F33d|CarhpTaP2g$s6H#AIbMWo`WQ^hPrRv+k^X%>p= zYJ-zqScX@9H-E9|9UU(Qp5CkxY)1>GdS)(~X-&9igAn+CStKH$j#_Wy(AryamN#lS zDwnRdXnGl{zHsAGT=|&LEO3QTnc@tCBGN);B$|h4c;BM!Zg0x#*XR!uH5<<*Swn8J zE_)jmJS~Bn3)FyBk)w~dtPMiw=#ck~L2=Q^YFWMlgtm|tR&?}|M_#E`(f6l>J8mf& zriX0;p6~gqLj!Iybyt3>roAZf2@+)zJ1W0b%u3 z-!U?6oOx=CF4!nIH0W%YKk#EntzxHOlW?LDarv|El=m+skrjR$-z#rZ7Pmy3P2D$L zPYjo82p5QlV>^E~O)EJe%;(F}crk~`0T{6qqRM>+gQZ{8M4wBF7 zFv3^o&Tm8kO%chLEfK~>RRi8D{{Ho1DlCjj(wg7mA3a?0HHgqI8bbV&9@8BeyOy9b zB*Jzym8^fY_&kHwYY5rMCPxSI!AsEliTPN702CWiPVcIuw}|z(L2)5tW}v|a=0No4Pzm+|cG>(j{oj`>w0~j$f7dxI zJp_%nBRVK~$$k8uQKoi{MVEbj0-rg8(IX%ajgd^6h2x70lbFWT`HYHW1cs`ty@jM6xk~|!XkSCRgK9!2C9aB-7 z_Tk>sKK*IMhJVuTHn=DRO#N{^icoZ309A}cie8{2L0YV z$MYoq3CDI?{JDqA`6$ih4ds`wf4%Qrt}syE9rC9pv0DnAyyib>(Iifi?Ha^ zngf>f3O*Ob<3u6%L0 zDk+N}q$;;^U4J|BQ^Oj*aqmji_O)@dr@N~G)8`^lJwK&gz19e+;EZOW(peSTLg**&QzNcD}z~g zvvt~_c;=kV%nEk{$BRUVdfui(`3E6)B42N3W^X9DSh`u$W}5s2hqamq<>&CwSiPZ@ zn`V~vdSkkvYU|bM>P`F6sY613117dxdD}^%OOoM8S+n*#@o3DuR?F4q+(jIYn(K9% zFs*cl!P}n+Cf$79p+>nlT;41k%~urnX-QqwBII;5*Q~3H$KALaSRBdo zmeog*mp)4qWo&0&4j2&H!u90&ZCs&_!MDOBHl%7rOtv|CVlmciqthNyd_Mnxmkj1V zzg|&)vni2VKesV&2U$3#OA23+eFxk5!mtCc-jitbnD ziKCMzD+4^omr@K~Dnf7dLNOn-)()j>j0wM^>)A zuj2(BrClGmcVEW8;&>P|3yeT(QODzcmCU9lw#_PT9Q1dfjMj@uXTd|7 zpSFBj3E?zO-Sz@K-PQ{N3KzW-zB3|kqaaV6t(pnCORGKbGc*N1Z+o%aB~fF4#L&Os z^es!N>)NOzKFoVtKrW8G_NA)Uckw}T;n1DlrTEOXke_%hlNP z?d(|_Hok|u7%16kre3ujJbFvN;chh7-C}6P)^$50Qlu6{9tFRDj4@j1m!naZNM{c( z{=NHP{i5^Nbe7tquq5Wr^K8v;Sk-|FtBwFmD z=-(7wY}U%4HfGnYPKp)fyfv;OI0U3rG>bU>UqsxuM`imL<2&htV!q=X_otPAWjV$4 zq&FaX#$&ynIY6J%D`U3W_EZo$?qPZ`^qhmie=*M5bmJ$7WpZL+N>c)I&Vk|fgK~7a zHg7pc$ZUPOyzqS@Pqi_=y;n`Q86AZ( zbGXTs|NRP)z+O*is3v^->TQHUZzpw@Nw&S)e(u?naQrx;j~vhc$Hz5YD@?v=j(Y*^ z$o|d9cE<8&F6{RUPxd0nWRk0n`&o{dFXTPaP_#Huw$2FInmb~moWItnPz+mlVHVO` z&;M|;-ka*_EFw~GOBycJG&rvq*(f&nUDeKJxpM;F7?>iEbGmOzP>K9@boOvhxM19# zaLBhT!5TTHyT6B$Vg1SMrjV4xkbnQx;a%HfoK0QY#E`polPh;}HaCh2;qrQW_wExyqvb&#( z#)T_A(&Dq)j^=`g4Lfq*{@tW8^zirvLf~*&+@qQ+$L;r>%#zvGW8q%Vo1zByM&xFz z_wD9MHLM>;nuNo{oZ_s*Sg`EsB09^izDad+ottka){psr^EfBuYCzU&<_xAed}`36 z^*Hg)ci^Lqtl9PqH6naW1&OrDLx#~9B=I>*NC(* zeI>A3&iZ-GTFcGT8TFmIH@W){ip6G*FDAB_YoVT+e>w$H5glF2ib~Tc3GS_yI_<{? z$Fal9_^T=mb~{!!qYGh~MgT0|dNuf;A6QvBE(T#SYhz_COY5lNYGaW7zdT->G3H*G z_mW0<{9evCk8!&SH2vbCRdzG6mL$q@ybQ4}B~|Z^oMn++^!U$?*Y-EBo}0o;6BN>n zW+GzBdd0Ldi5-{UD&(FvdYv46u&B|fepYQiNNE^ZQ~G6`sxY|ujC>%7tsyg(dQsNE z-eLh~T5w;hshk=l4hWjZotxL9rP<7mZl zR{K&L2<0=zR`lk2Y6Uf9EMtzNQqSS5q&UF^rYF*E3KQf^Sq>dnRM8tY(V=f8S2PI?1Uc4DN@^STDTB5EVYwduTbYn1ptPRKN*M9L4^K~of) zaEmTZ<|;?H9W3S4*b^-3vGpxPT?azj34F=!`9(Y|j}mBUX>(9=%>5L?#UxU`L(|#O zoM$UDO`Gn`5hh>sVLPRohK=vNMALiCK2cGACO@!x7`F?o#cyV@Jh; z&?UhfW|@r%(a+!R8I*Dl15JnQ)VeCH*QOrG?@N56O+wzLAZ_rXrk}RWB+Ldmw^j#J zeb;82$q;XqusP zXe=cBWKg|g@=5;lEiD}xvy2*lSyyU8u{dp`kiS&Rw*5O=TK%L?2Agi>D9(fKVz`?X z_BlS;e8nE_p769p+e8GZ&ABgjzXnT=f0%Ca0XvYn@qKGsKz%oRD^dOT>tv<(_W_OC zoBH(3qamZ_e&#Dq{PK(1%-Y4HLLcQEzj;eB7yLsDy=Vcn&}cFpj222SOJ%Y>@Kc;9 zdSSh{?Oz@AIWbYnG>SnZkK<*MZc|8qI(s?-PPwYQ(MCG{%$zg99O_iy7$ksw)WO{S>+tt8Qhr855(#aT6g7+Ed zEEWjb4KKJ0)*Gt2b@b!P(cc}Ib+eYPHT#-U)XLzn+9%N_EIB0R+UA7xy;$^Y3CJuJ zELBer=Xh_gJ<#8cfZpKUz;cuJs;3SBohuKp)siY^v}I@027=XXk$LhWSrf><>y6l# zqXy^3fn&wue06_GF#TC1RcPG6Ck!)B$t)Xn>CeN$6CRw_YvCq(EKPTg+{5Fc?l%Ty z`fTX^q;zsI(ykL$D7ijl3|6!4bz~SwzlCN)X^I$kR7be0CVuKNp%ohi@oo5-zIAG1 zwLpJU~d!Z~Wwo4mJ zQM~pk8hKo=*Upw$s>gWQ+Me(1E5?Rdo_GZ$#e!ItG>=lJT#$1`__et9i1`YO90J=91(N-igY6uCL>*&+d)9_ z8B-7U*X!w*^Y-IC!;mgJV%@P~tyQXK!?=H+yi?UeCUJ+MCIPWBUeG& zQXi zs0P|&8{%4Ell{?1%aKaUbCtg;Mu(tIJ5nJ$IrA1L`hr7=R(>OyPvU3~Usl=w&dBZ# z<>|TKSo-x)7ela6tUt%;i+X}^v#~#yQ!3fMp4FpGwzhAij4!5|j~=4EUziC<3Y;`*32?-a(5)^idwefd?ZXv@jF4p0yh{ewfJC9>Yu7@Hx zM;yzq9U-*B(@ulqKIj5E{9kYq$cE&1F-woCW%j~}|5UWNk&%e{kblD)x@;!hRF4bk zi;~?tSad({TjcvNLlrMx8{7IWsM`*{0+Zu7w1{!a^J}{mft&=lL}Z$NM{iNX(KL4z zF84z|aFr2qwTh`mZ%n?bCsbNGdt2(!&VcUu^#SnmiUI#E!5C|5*Zg0Qq>$;K&ydEE z8zEhp#4*3uWZyZ}C$WkMaHsEZew>b-C3lNSknOTh(c<~vQluL?2DFFmQ>?5FNJq}) zQWrx;d8_uOiH&H7d-P7GU(qIt9`fSd@?zecf0o<7=#w{hR`OjD18I7hXn z%nrA0?jSi~I+ZW`YN;n#WP1nkZ_40WXNzs#j8XaPnL_)?jB-c!L#F4>k4>v4EyYVh zb<6S@zv~gfmzYCvQtu@1jp$h96Xc?&Sp{hJ=T?RP9+_ zT)q0Ihf6o&7rx(z;+dBMB7_43ds^e1wd|HeAqu|PktAeTf^8)1HTnMO_y;tDh=ggs z52h1`9=N~(0mE=(Rj0=HDxA?hpKSjmm$B-Djn!&s;`;O1dVFtc!68eY#e!MZgM$ri zwOJBd%*XQ0aQx?6ZJr91(!4}=+p$_i=(cs3`w+Terma!lPgu5?RxScumVtkYJhgP| zoAIns$sQb+hXMt<##5|MuNTvin2<1kp1)PB{wL!PGDzdO)jfXu_U$7q92_N6#An#G zDOVBPPE@SaG$<0M%Lxy z|5j4nX{RQ8IfA-*+$bpI!_h@uVlua1`Q97FlD_!l%F4K6CnU}Znk9_m!npL{L2&c5 z;>B(p*}V|xu(X_4?iWn&)qc~{?{&sap3Ixh6r^O|-F{g0?^=MhEJvqPr%vGqK3~OT zU2iJf;_r#CZG27)sN}u7koBM%?jCG9NZPjq$D=j^I-N~zSUeCsxX3^68m@ec@+z(I ze6Q`OE2`-n80Yi38F);f)2uifC(Dz!`8+oo?{r_V(*NDbE4MrgVdMDUQ~6Zs^fYGS zZr^wwF7#|XE}8Hb@7Z{+^zQf^?BhD+OJD7Vu+Qh%Hs&;3C^M;+-1r~*YC;Gk6s3BR zT$;NBqW&R|{vltVp;B42U1p7ubP+(c zx(3C}uf}Qv7hHvUq=)9@yi-Nv*nBf=zauQ5L6%jNAsr_yD9nr6C1$QE(_zV{<-F|V4wec z{ZHREF4}?zYjJ=0 zJ1;N6%LPvsa&mHY#$17!q#TqMe@x*gpZeC@ww>5r59UxAa=`+2*Zv*2WVZdNEDtso z*6a80QRQ+gy3gmEC7;uZn?x)Kqr;|J&o0#f5~yQG2J2NC z1{@$nyn+Nir4|sV@9rYQKz5yXo}^)ng8h02y*}(?@3Xb4ZN3Bq;9a|Y{P=QG&-ea6 zyx(D1hDAoEvxW7%etvM?bupFa5zse+B~5^RM5uT1tM5zJ9u2!2jRqmkSbnq*41Z*x*|60x3(y%aeb`B0U*_f=X zEUNhI+}xPhSd}gI_`bIe4h{?473JlU^72FV=`k^oe5G8}1{ z!=o1BkdkUI`MiD&2e9FO>Fwopbap0qSSctZbbNeFU)tZ-hh}Sg_>2h$OtbfI@s=d$ zQBF}&QBq2Z5?KHHcU&;;p6>2$G}LD}IQG1xSvffyh*V%?Yin!YzJC{Yap4BTa~A~@ zVaEva|Ni~EVi0aEvr{>Sex>H||I@78&pg`K}Bsdtyn%hOrH#VczAFh65wHd z2IjZ&*Im@Y*4B)pqoXBdWoek0m{^|IPTl|h&8euQgf_>rgCNwrw6xUloSMh(=<;$% zpy&MjoQ9E+@v!XECpb`G0_+TjfP`dNrjB~%_0np3eWCBCT8S>4a*1xEpS0q|*V)~? zQWGgDDPB?xxm5mG0^(GD=dUR#q!)*azO8nLh7EvmIX@pj({_L)-(B9{-935o;{t;GTp8(Ue+eOufn)Hl9QWy__pvGEuT?D_Lg zEiEk#F+E^#J0LbJJ=8{SUXb%v^L!frB~oUx?trf9aD8>PhG;VWOQqhpBzcyDj;BUS zPL38Vtx~RBR7y$;!K;ntbQ1o~jt;z&^ngD@#*g@w~V6vL*VNGFI(AxTIP;QIw|_ZcVq&z}#P)i$ZF z&3SsYr}C1osPMWV5bBbk_vYEz*?7}9`)fTh?p@y+Juk~$_GcPadukia_FQh=yJF_x zXc-y9W-2X}GT%LIgFsNw&=`)rL_stN_=_~COuSSNC+6xc2V9FoBCLA{#J=wCB@m(Z zW~(E;Z!hexj+Ps4!B|Y85BG@7>$}_AQk53tPcSfO1qD-pL5D#pq{aBJw{C8{;64>E zZvybv_??kjDyr-#fxQx+`69FtaOt3NW@SululKf&s8h|pBD2-DW}Abld(#y`Fl3nxkpuhL@u$LpMfL9tCmF4=H$uW7?FRL#ag&e#b|OJEE{&_h!rba*nvmz!e)VF4kS8~AL1a03T?z!E(ghrKDbD(l%!@PDewTcG*=f&{?wISmcsCjjKlfR|u2%ttiZ zLLl*dRWcXSDB!%~v|t3{JYw6akmp6I_3kG ziltq^!_F6nt+Fy)?r6`&{;Wp53lq#RT@hqfU}v=@^#x805Rm~u8AiQLJG!-o%u z@!I0zy_#T;0_GNf_b%;H-O!Kob)bRu3ZT^9*$Fm-j}wJN0JMf%e||b0%q0TD==wCXZ;)=R`xY*3*6k-e zyf%8n;#R2`6I1Vc@^I;?;rh~_GdF~$kS|?$KFJiHr!R^tGRolbb=7>(-tG1yi~9OT zhEDo8LDC-C(;HUXzOFg-M2WOmEs}cP@+~fMpA8P58IR!aRs`e&=?3Q$lOIcL+=>wy+1*+qXDGmqY^EYSPG1ZK8bV1g$Rb{3F#1q9jeXfh%T_m;*pUsoT zO7uin>}nrwc0{VOA<}hyRx9+GwL6&bTDd^e`#-TZz#L)z2DW_ zo~z{m&hmnlwKOACGe3bI+0Ao=Th%lHPv`rJRB|U7B_zQY5>J{a5Z!0rqB?ki>3*{) zv3)bBbe7-0{GN|J(rdt5frXW)M`ieJ6ua4GQc#-FpLT=0)5>i0yB~B-csg0^#+8~N z2YYHg^Lx4J_TcezK0dw@4G|HatP;u zqHjU?*RSx;fkM)c`hrRkFCnJ~w#po47>*nL1|X`!R=UyC6<9{>aG??GxGk)|4oWiP z-*o1vpkJLlU!Jc!mV>gsv5xif)j4<+n%F<5Fi^AFMUrrQtA^h2=CoyOTP%;uiCazR z?lIJ}5wo)KoB_H{lRPe8l{HKE^h&=ga!&2C*FtZUbGpi!B$R-~bU5QJqgGYe*qE|r zrA5c5M~?}4?Vf|=esYloyLRRK+a`OH`?uKG_M;1pp6GJHR6nrsFO>RDQ^O4{0)-lzJ;WnZOh0~l!P7~R?q@}Z4J9*Dxc73=SN_c#_)vVN42ei` znKZ$~2r_|@-{wlX9@{j)m(_+lfO{*R6^V*x5dtS{m*trkkD-;lIvbTI0`nl@u`vcg zy#e}gd$u!ta_uMD!tg8=QJ z`@G(4Z?R%zfA>nz@OT*^QfFea*oM-Cc1~E{Rolgnr|8@vSEaP+^6tQ3&&;v^Ld3jXxOc_&v7x z^LzFsZJ|_)V8z1E*ci8mgC?hss>!ce1e(T4u%-`n7jy`vU7DCStlT1=B!|Zh*)Fx0 zP_Wfmiv7C2`p_sPPkv`gkeHpX^@tf`=swP5yY;NLl3`&JM3eu!un1avyRiE;Ucx!B z^F>^Z4b8k+hxKaoUd8AxF=1f94Cb9oRbso^uUh&L2Z|$AshJGp!ydypN=t6nYpvRy2l=a`!3|}!LW#q8)o*rg}Avs9nvX#d40<0_!nLcq^W4{{oln0apzC0ow@`18N*2|3c zhp}Zgpe`006j??jxcB$Z2Sl6?B{3DMR$O-@4MZvDV-WwJ>8wh+NuJKEdD zV?JYsaCGL5$T5}LbzP)i-WG$4DPT_z(p+&pQun*f#g6C~skK7JrLleAH@-g74Kp_H z0>orpvx^dgP*%^&CpgG34g zoO?R`t{2~LJ~?VQ@{(?(@HzIA3wb1bng%~I?8poD)N=mEAS+?BC>C3Fhm z6z~}=V&Yzq!D%$Ou~jd4?5?|xr?VR@&(zzPbzBK(TpygD36GhLo?L*9^_Obnqy3c>KqX%^p z(S62FmPN7H*GVC=93`x_9CNEIEk>@F7+{{1kgr6vhNyer{9>3>o`u6P`2GiSRhurP zHeSk>F{7t~W7_FuKwch3ozB^4Od6@B^Nm&Q2z%=M1of7rV;2DJmxc-+wxE2F{p8*&!> z)fIbJSooc}_IAA=Eb1=GZN%VdX{AonR+@%fjW{U=uL1-@hw}R^6FZbWvCI$5+Qf zSh^D(2e3cC8x~S0wd^mGEy^j{58a5{dqd75v!m?zjUY!2$}-T7)_k>14yQg)K%r%m zgI}B0Co?TP@5NRZKmE(5bla!`cPr7r5}H)IDif4H+K7lgS}+-8YSna86X9jCuwhWru7J&#nOrpw9B!>g(LmSHA4^S_rvT##N^R**aBA|8L{qef{AvS;1 zRjk6SrXX`Zye`!~%3IZ;W6rE2Zm_i`rlxRzYlJNUUjE!-sM31hi}ez#Od=IaIihj? zm8=%7Ei|qW9m(gRiSX0%YEX&6(&3@&MiYy0U{6#Sql_78{biQXhSf%P)j|V;eo{h0 z!p3&Ny1w%2A8b0i2Q^(5Tltn3)Iynen*+5n=!C4Fz)52SP{Kx za#e%Y98{QrptZz%w`mg86@+!TH`1 zUgv(Zv~QqqY8tI!JkB&mge%LkD83<|DDQ`&QIN}aDvxG;P#0=>GH4*JRK2oSG4k?2 znBftJD+6AB`R}*(7q&fp`s#y>73!sMfgRo6aF6ZkwDtMz`DIyUQ{*N1Py_u9UVHq__PNp}1NxP* zfqxY%Xu02ne=N2p{aUhrmcG76)~D{?e=jBYUYW;|Y-hZDiQgpwt?{>l5!^5jVsyFP ztJ8?wGONEu4>gvc=;3_QF4u-&#w^wk(|JdwE4m_Pe0QkU@WTd@s%2um}!_oSXl}O5)s?pYb|J?*3 zBfKsh<`KowAqws#h175QkGpOXWflnKa|DQjm*qd^5oA5|`wHud?&XuFB+^YceiY0M zZ0cfm^-*GxowphYvDy#cosx6G<6kwmsV%T~bn<;O;$zyo*5k#o`6O*VBgEpHZKC&ll{TdwOoZJ;KoyWLac4u`J}IkY=dUvg5w26DBV-aeEP?a~3GzL9CgK z8!APwtQBlbo~(5mRAWV72pI(lLe}t_and)Q(@}qZJnL&P*d@<#HbJ1S@DVeP(ZzOA zEP~0eTQvZY)q#?ThzNGp2MLy{lL9zepA-_GQgi6E%{?^>>4r?#MEvRcnF;^ z=}=t975nP7`k~w1HGQ6?5USWaek;QiO36bIgixLV)W< z%67ehsl&$so(9feJWQ6zE9R9rAF8Qcp{~f&!uvIrcJ0>{+*32d)oL$Ep`RsYgO-6) z5W+I$?^N$7wLekt%yFa?gi-fSntKoTGGDH+<@d=t3z^**A@lZ&c* zG(s;`F6?25o84>nw4H(arPW$^pEG|HM`6{Q3*T*JV=Y`g_t@4SH$Pmj+?0)?;~IK` zZPai!%(^y|QjwVI9cCZfKC9T~jcCIViYU4(=SGaAZQL4vtrsNSw(h`Y+E1v>OyD|a zFY%T@LOe@xSP!t1UVus)h}ZM!0sI{i=SM8S#^UL`plA8)dej=3ern}JK}8a@Y*xKY zG7}t8E!1fyy>aTa`-awr^ZzNAbMpkrBg`FPh{%yiLoM5mD>RNVtVI{QnhcgD?osT` zMzixt@Y|5*q+HWq;QKa5qrGEXnB)-rCA^W3%kFoBJukA>ApB%qwYs-u=8LCUq5h4d zYa021Vf8YZc6#?iNAvY2{{(Bo#%Z@jn) znmi&5=C!cMw*uyK`Om3~S1*DV%`W`nw@pQ$=6tgor_CZyHPOl6Sn8booB{-W?fu1U zBD)Eyp7%K(;K;zlffUohI)4?y4K^rVA^NP~?sa33M8FVOqs^@AeXep)yGhQ7Z!+Rg zD{Xp^XFA8$%rgp2=->a0r<2wvIcTaIyw}1Y0Ukx1AUyb`F|L&Ji!oV6{&&_b-FafE z(WA@B(Xx9(KB1fPwipr~2iJy%VpC5INBVMa2+nvGEYs2_q5@4>3*P5lw2EIF3Ao$(7hoQx6+9 zJqOHm7Z1IJZES3UBO>0ZsKicXV>?rl^t)ZZbz8hbqOhs`6yaJS9nM)RCE3s3krlg( z)o=G!xouo%>NgF+V5l1P_vK!leC1bwuxuS2#TgR1Bz^Pdjon5+Q6Pi3c+pZWDz*ZH zm9;fC9$o;Dc?55^GTqi=)t1)QzL=%CBjxAk?_~yfyf7Nd4)*YPuDIxld}|x5BRfRt zto2!}QbkgOEo1aX@8?BcSC6dis%3xs%(#+1y$hd*Bo+cti=Hzwq62EQy<$i>6i7oW z{fQi4B7xygS*)q4ES~yOiHle2*5~}Y7GUh1TCsNPX1Z_0kO-6%&{%DMeu^DmtwtJc z^d}yjp87{AOk(!E7Z)F!9<$(zj)^G)6r|Vv`DFLNKt}?b5jZ@2rAzmEc7Jn@Q!h@H z!8}QHjU=CegeLpofivTro zd2Q{5OD!O1i(b8a`C3u&84z4bZ5M^V$Hp4KM6{IQG@vwHma%5g)@a z>Wb)?Dl>xFvBGS~5HvkiZA+2neW8)3{0bzz@R&kpSO@io!ZInm*pML1An`UIg5co1 zTNJ-fi$1qcB_c>$0XJ=Tv?OY5Vj@Z#(>gSSS)f|f2KHiBKu4EJ(@?R0L{|?E=I@@L zCu3>8W7qY$HUVigj0Oc^LQ+ny(=6Q^DfT_0mX;Q*7r=P8e?b8ik1OI+v$LT({Mgv% zuj5(tIzc~2eJsr%#bUpHMeR0bck}c#y4afrqEQzJI|ep3%&_?jAw9;Y%8)a*v55c@ z4FUoJ9Nf#F(E;}?F&i*tAXD| zQ+YZ2K2Zm?M8wkl<)Va?6w0S$Y65q6cK~7~k~%=ZekDVHhJEe;!wUe`x5Pw{nV=gR z8$ajZz*%2khl8utjsQz!wA>ajQ$TY)Eh;|NNLY%6?OXvUbpznQ04{+SP2dqm3zETb zXk=t$SEc1799%?1#L>x#Ur-P-*!7Uq)YNKfYYn=;Q3{esiI0hmO-`=11)y~g4t8Fj zZrUHr@udiR6N7E?B@+`O+}Zvtfv&DDp^yhBKBGFEyu7@dyZcYu#U?5>d2w;r+MT@y zJQJHqFRn}?d-n7_sLmO96AxyoKU-y8Z$SezGiRo!~699Ne zjeQ?*x;bPi_B{tW)Ys>)S%hTgxW>=J!vjQJeJ~x^pyT5S)*NcP55e=^nh;=9$w+!! z)R(xZGRfRU)AztCuAGA$92{kymo~Tr1drifJVOB*+S1Mrs+pM?uYiEjpC6)j4i0TV zj9~lt=Q$wmB*LoSijNuTH2aVtA|iJ8^@)j!K0-x9i%L(AY-kWPHZ#l2%X{MY9InPa z%oe?a!ei^FDP?VVczE~Wm*8M0*ur+hP0!4GkWz54!#2o0qx@} zz=^W5^2glDvNCi08eTrWdF;-F#6;&!Eo!O}t;)*Efj2RCcV4*o_}pm9#d^)fsS6Vm zs-9uzM@KD2kwO!U;>%Bj0a(#sU?uNtYlF{x7v2UmCQUExY{j27HH3hAeaX!H4qynV z)!$N6!-0mnwz-*EUalz6C`0S7&D_=U`Eez*x}u_fRainoLNa^gz#~~)JT)gL2Z@G; zX2!_S-kuGx?k-*4KwbOukrNJ1NlEE!cM{W~mXw*9*=?_^>vxwp0RcfxU0oX(-kR$> zaq;<(M`+jzEYvYQODikz1O)M_xwUS`RH&$^AC#59WMoij*Eykp=4-%BpNDHRuq3PC z0ouHRf>GhFu`yf>42*Am>x8WOpSYd27n?-@D-!f@e`^V~wzVC4FbK};1J3`Wun-;Q z1M7Y9_UpZP`$fmS!V=^eRC*X!ouR$g|w_uS9mxh6O)0h zZ6p{5WMP5);^JcFC>!u?c6N3jUDAt+n8b)Obn}4K0M=$&vdlk@xanzV5HagDh4sa= zWP-7(sHhBV7#JI82@Ue{^BXR;d?w_!O5heZNB@G1P6+4e=^3>^kBv>lkHiR2ZxgPk z4&?qrK5JRor(gjl_R4QNn@MaF6BB1{SwO3Sfq?=Wz{2`wX1S+DU?h9;fs!v|FMV}v zU&Zc3Tyv2l6Z>WW>B_gJzTWckGDpPhcSQvR@Ynjr#xL3g=%_LlzJO)Ic1Ck_bWBK2 z7GsXDrvIi;Fg`FS7at#A?ZLpv2y~smi$kE9RPxG8OG{4yIQRCVs;a7%R#f^B#I*Or!OmX?nVOnzjpky8g@yh0 z#o_kKPiCk>o3X1;ztf z8P>$a1bD*yb6FW%VNuZ|IM|Qe5BHgiot-(;v2b9$z`Mf_);2Z^g^MIOI5_GW8kOp5 z(!~B5RdwDE)(RRLAvZTS-13WrGQV>>sQhmytd#$EYWAtLg%~^8w#7a97X7{qPGVbU0 zPEJ-A>P8=?a^)$k^$o49pOTP}>@U_8Q&9aIhr>Ef>+!vvEVtomG@L$(ws-9FSFVGw|h$1PCl%TYfbPEUwA|cWp zf~0hVfTW~K2$F)7v~&v6DAElg0@B@e=HhvO-}t_N&N%0carPL`GZZ&#uXW$^o^f5* zoEt8UDk&{3wisjvw})Kr$&)9tj~`39y9g0k}snq;~lDWtT42LnO1;9MnRufB!ZBl$ru);cwuml{GYi4-Q<<-e+KJ zZEikz{1_|r&6~|3ZQGe3YVTJNi$TPcs>+f=?Dm>kmy}KJ>Dm&kyBM-kJ4)P?RuP{D54g^H>8>D^9!-Z|&rN3WX&<2U| z!Lo`JBJPAxEY;cJ&E zHw;)4kWsXzNe2Dza!4X8btk>LT97eD4)VQ52-;$&I|NhS$JT^IM7Qbbufk0m#mTxi zLDmOjr5Kr+CBQmP^_Mqh(fL5N!j?YHmTxx)1U zDmVV4x5aW)D?RNhl|C_R==>A zfp8EM;Zab8fk1)|ZpY_rJH>9M3H{EUJB`!R)Hg$VPnHNJUrAWvmmNJ zoWBKOnU;nIb^ZEvyekh@ZUSZMj*SEuDcu!_s=?vm0kBq9p?~}-xQRT_+(@upoH?={=tWOrCIv?g-{C+NV)AqAkNzLMEqHBwXB*Na(WOW z!pFa4YikSgm&@RZuVG^cz%!5BRlzkjF?rqCD8AbB<9TKRiUTrYAZ)2eO5#0p}seal!J0&0v9__8N*ar+~ zYieuLLBR%fmjvkHKv+^_(s2{X3!&tOB>~HXZKd+PxI>JLtSmaXb4XT^8j92B%oDOf zI!;b}UtiyqudzFbkm1klY;;^)5WxC|kn+E@vwL7*Km+B}2ooDTBnF28_?$jaTS~wU zLk^3kudk2f(}}#+5>SM|To4YEcD!QwJJl>zIt8nQNY$Z~qdJyS_E1I!$=@!v5(@`8 zr0Kk%e4qe*5qf}xZ&48ka!{dqm6ey5)X*U9hWd~#T9yGaIye*(pwZ~-??>|R+M}gB zI5^jEab+ONkKBdrB*VHe4VSe44n1raaxJ;cn=I-DQig`N;i2r{iJ|yY&f3`9lSU4H zXy^vicC-QlWRQLO!IQt2yGu?^esQp53ptVJwD^`OA6Mjk>JG;Er&oqLKrXZCXOfWJ z-ijt97)?`Cdhj&WufxN!!9_(yNBe?03G%kruV0%BwcKN7zKN_pZQVu8&kotyaeN&V zWDJGLL7Ui_FK7*3G?-sSNB`E|E{j|zK$STF%30)I;R8T*W&~!9Y{1Kx9IyfZyD?+~ z()9|<%ZcIf-Z3Z!78bJi_4R=o^y>c(Jk%zH@WIMem6Zrd5D*Z6Am4}`;^CpFsi{Fi zlAfC@cd-#tZHPZ|&xdpJ1*B8pUo>9g6#S~e`aII>I`FVq;wZS76+gFjYQw` z5@a7yQF!0m+M2t%^3WWJluo;BGCV!;0u~C>3-n$&wm%lGzejrNw1lP}&ca#vC(nmO z!eN;zq)v?Lc%F&zv7i!F^fhv!mR+d5K^$G=w*4FwxI(VrXE+_^A0wKJ<^%$K<;YeR zBxh>N$;-}twxG2-5_C;dh!J4=FOWx=-`G(^@SADrxyIVI>6z7dVpWVz1;0au&aYu!kage zu&0RN!)m-7yP%)|39cXtx#Fwt4^hZ)HiwIrmP?k;W}*>7>L$c!BuK(aDQV zFYa35|8e)D=kvQbN#+Nq^?!Z-h!LE-E#oqM@zeWuLPVFi@@BVPcbnp@W*QyW`&6Aj zY1F^@{D?aA204ubHCiDFvBCSe!Ti}?mnh*w;`u<_!_z}289Rd`41Tdck5~_WQOAbk zZfJ;04cv1TzDrB%|28HDPV+TzL16O+kZK}04mbzE*v9V8g#KKLnEBPGk+Cik(B(F) zq@E?+yvtH)e}KYa*MCEet7kjy8NmGabXqLxAcd|nUf=iV=;}b6?)SL?sf=2}ppTvO z{T9qoHQx7}3y*OG$TA^q2&$eJWRAEy+rH&=uxNnpT89b&?e;Z@Cu1p_3MrzL5JUVW z(!@cM|0G$M$vK4fW~yK<`gFlq_zg{UWo<7?qPBON+w+duj+1t7tTePsytRJt3_HnM z`9sNzQy^f}(be4?vn+I4{$R5hY|t%4h$SvQ7k%~>XR>D*sID&TgBLP6-O zMzCNDh-;Am-f+5V3pg4Eugyd-r*SKiezDgd7{%#+7|P7RA*DPflSbncs!H0!-(nE; z|K)OO;9R(n%YJ)#yQ|2J`EozizQnXBxoXlOdX{(bYnvc!svD4%5FpX9Pn(2Y?>hpl zMv9%0f)__*SVe#X3Ga;;bT-i?8 zK7Nl&NtQT6=b;E%p19Ah9JdG+CB!z!?E^uXZ#7YgFCWYL0P>Gy&#gzo=O^2w+~!du zrp_X}JhCThv%OhCmbaFNjH#MVjcv}f`6op9&;HxLxyW=A>&ttuKl(O}qn`%yHw}4R z`Php6`a^j`0M~nxfy+z68TWL;xKWk6p?)uy#cr~w>h_{eR35r_0vHP#cLYa*gje+J zz_|8!ttMX7OAul*htrbkDUOy;V4)`2N4ltOeGf z_(eZGU_Eq#f@20vM5$CQLJg9SVg#;sdk~kpc!f1&Oi^?mei#_su>aRiB5|E{EhVze zr6IDB=qaap5t9Yi(~WI?x?<}N9KzV?Q0pt$SFlkJUZwUccD&x8ztT4*lijqN&U}rS zfWv0-@AFe~bGDPfw|SQh#`_2q3{0@Dl`(TqEN>Ct7aCEl#oZ&g^jk6B_21K#gJqvh z`6fvvR{PiA+o{T*&_@x5w=_?)`qwDnigYQ~h34ci`T6ZI7PelaTtWR`?V@rbW)=oF`8`f(Ei49p4XFK*{;BzQZ7k$_oO9Z1v-}Qm zHxCb_qG@SqIcPXP7Q4g102##%usOIzbMs%N;o27HA9zv)wr~72)eWj%P0Ic%%Vj`w z6pxlm-O#-G$<$nFSTe@zO3<`e@GXgcW2{JY1cFot$XmIBiJiR-Is8Cp(};;tM+$FU zgOWZ(z+t|zJv-~sni@U`9SPT7a^y{Z}0XO0f)SRUUOhdvNoH^KeU=I_(p za%M~LlIGq_g!;RY<7R*3(zm{)#l;UGBR|`UYXI~Q4T`UapsWUw(5tKzZ*t&$gpc;u z*_}VU+El_KpT;n&7mFBRNRebliDTxas&n|X$WHdSb_#P?h2AS^5DC{$32!V!(Qx|x z=*}T^yp4Z2cKyQjKe=%I)jCMY-`spVu4{8X(bj0}>kH@x5{LXBtjk@g(*E7z>%)WD z>NXD7c*@6N;RaS&jj2%!8zY@|E|dc8uH8DqN7Z74Pt-WxmUE;Hx*j|{N1y7iGgvo4 zBOst79mjsv{C+WhYqN`VVaP;4Zk1zakSG3hp0(1A#o&eM<(zKJA+|MAL0f#PNq!PE zY^wq7^dZ?y6;SZfiHcHz3SZ-^IX<8i!E26J8QBE3{X(}n4RqIA7>d6mDZX_^GP)^X&qO~H`i=#8}TTod^rYRY;gc7;_ z_QiO)G{A&VnT(9rT3abHB7%3+jk78NOMvRRX`;%Z7-~9znPB^7A&>(+A|-V-7%5@O z9_i{*LNNk3&~IoM121~Lu;EN_YkM0Kb~JW&cI0#WMxolm1J)2o4kRRkO;myod1iU} z4K#xQ!heB8kzGSp&QTrc0*wZ55hRWIPN4XI%giB)0*i}O{MpTrq9ZXDXgL~ywvfik zYDXqeGWkJI!EJVSMWk4h?LmstbmB~SFn)f7aykIqsL0^kWu3~Gm*C+5tP7H&MJk_k zwwc*k|H?{!AWdd7M02lyhLHU0HD$c732%r)zdQpZT%bWWw@At||804X0 zxA<_NS42ijiyU$Y8sNp?S!KClkkJ1M`X?!(;SgPoVGb#Ljx=<@{Ffwx8Dk*C->wRI z_*Hl?1UZB*pkm8mHl#`g3Jz+AW?%y^F1U_RwUAZ-wAJLXUKa#{_b{^iSL^HRkqX1h zs}3nEqh&8H$OU8&vKxUol%asDp`o1Dr^12!rG-)w?2M>Nf|mAl7_eUGB)tvAMn^{n zRN&vh-jQd5%}7g46-PEBI}zE8rMrMh-~;>!r4dqV!<7({F&bD>z#yEpJf(K&*$NnN-M`6}lFo5wD=`s|d7?LN6Io zZKYEI&0!2eOGZ}K@6dW@w>rjy;C%qS!J`91ft38TOA-jI<(>+(BLV=yk-v81hJQ_s z5MXd$lZLba{)EG&9BB&l&?+Gxm>RC=0gYOr9yTB_8yhcy{)KuUsFh2I?TWmX{`3Iu z@-iCQ%-UK6G^zdxAkh>54^QEA+cH6{X!DNh>i^LKEID~X9Zf*s|L2bp$g+q5K|?ss z|0lL0cpY&9XzF@;5&!;~BQ+Cvcz}pTsvH;qJ4A5D3+fy0v(OBf5HKX`X?0-jZr{F* z45UTUBVeWwuY>P|cX6%3kr(+AX^lv(0Tzxh%2515>m&zYw#c$nhyP#D3UmnuJbXiJ z*#B7pNnOjI>OhJ47+EzeS!?TiaP*s{f+k&KzfEEc03tzayvG&11qW>F= znj{nipLlR|L<=7^I5>FWN(2oMdBKmd`#NUI%J^`t8NE;pk0aCq=xFH1ve?E#H za}$zd4nS9+KmvV3Aao@H6lyS9VhRWs0F;dZH%J_l8G-`oX_vf_yR24*+8RNr)$-N_ z@@cNg%E?IpK7_amIKv3q4~PiZTTnxT51;B|$R~1&pH56n=+tg?vW(Pu@WG!E94J#| z^m7kxxU@eo@w6-~dAFG9Y15@IKA{7I3YP)rS($F2p7uHS%IinCWPl+4ht(k9u=P0a z-?_QlVDbPNO=|Gq+cmX0=NBnMEcMvq0 zknjp{LHNwjx6R4p9GVyvb)m((s%IG)S2Q&-Xp z4vFx^V~)Tn!Y3sS2JD=hn_E0z%^2Dh`91bnfp`H906Kh8(A9jh-*DctT-bo1{m_qO z09|uNai9!vZvv1D3<3D{W+?v=Bplq`lW`^+pgj>H!{^+4bwL44ippb!(-R;88Vx7rSm6{hgne$Wi?R;0ke$7MulDT(J`lI0nd3gnvE(B@evY z)(-lhhLPiMPJBRsue^Y``+*KvDnwTU!vNNk`>U#}CDN1}pPEAM@f1`Hm6es^nJQ(1 z;WXC{{=QS>05q!R3FH>52yB#1@JvnZPo&QYUH!>cmq+sL0dsC7oK+*3&(fkOV1ph{ zKB?txLZ?3BAhNTw{{`lPZ^K3GB`E{QBWNKK^FCT+$R1qsc*1_|G7?Q7_}XMcE4+IN zl&c>FopKdGwq|bL2svsPRDZY-IwqAU@s-ybFYe(ZR7zkcfFn!iHF(9zWg=uGWJTTG z-Cvv8d`>poCN-H62~LXV)^*gxrT|?``NTwFVWAB02J{LEuVKGj%yPh0#%|UzEK=MI z$*9qTK_BdsqHmQxLHnB_h(u^rrS2pHPzcNH+)?ZA=eIEzrw@M$+0nJU?IVBXfjHct~qfE#?l00Ha9O%SyvYu&NlEwp}OE$ydd=c_xjJj!jA#0$GNEC;WuD=A0Q{_zn9=e z-gNPHWemi+|9dUbAp7*+i}E7RxOjWEKN9Tzf4exDY-9R^M}2}%u~m(a`}2` zoRZQk_!w`TGO%hY+ZwB4(h%>`8>JHTo-1`B?ERQZ%2~ek>_+fz$7ezt3=Y@3&05oR zR7i$;Ov z_iky`r7zEWah zmpK-WAR)o&Jx8-7S#7&oF(u3Jw%%8RT=0rp$|;&Qru$yzmH+NW(6Htc_Q={lM*nfR z8I-}R<#0!%>{Xn)Y3fUthPyhIFj&G81<^4byx2#7K{~`2Sb=-}uAJxApecX1UhoTE zBXv6BW^LOHmZxF*cz)5j)cD&ekF#6Yvbspe%B{?IZ!X?yzPL1&U0rkmR_SC~g|?C8 zQQii3v^Z_F#OB1Mf$MX@iA?=x<*L)(`dJ=zWh15SS5-cUooVX$e-yDbh}fqY*|mFh z?5}7z+oSH&(4Bw8TJL@TiuW3t!M*$bnRPc>#p*AoP_vwSC5#nvPLGjFrkpbH{eah&ML(PESEJUett~Z^>hB*SaZ-}MWFLv{&N1u(# zbtiFS;S6!Lf4ORK@II?Zv^e$9QIK_-ci6)+`61K!w`E1q&`mnRqUaAzCDVAWinwha zVt+q%`7BK&n-0ydepy{ts4%^nDtMztF9oAvCgPv*^Q+6gYozBRDpq4DK?y?hdCj5& zv8NJobHCQSWc~k$2yH$sqdcMB_CKU#Px6b?2{&9hV9V=O`cjkbn#8HV?dQ@y z516YQl5Z{_zbGg1*h3%8Oh{qyIo_I#f&7Ni5Rq`q5+X|I%f5boIoRW&dag~#;?A6FE z7}vZtY^T1`rdvNHnCSiG4V%E)zaz5LZO$^~vLX*<{Rc*Zk6iaXyPkdxU|S7ZOAz~S zgL=1W$7;JBHE@vK>iuF=E@Wd=^9h~O)@5a)>M3fB*uzo!Sjx80fOGFqh-(@BnjblO zB2{3{cbv5A%-8%<;~EMrUFw#sP8gW4H><5T6{jy}KZ}`_*vfKs_M^-|6|Cz9@tt{~ zt7LsRXvo;E%*}1tJ!wAJVZv=^j{KdZUXiYph=rL=i5I%g_U!ig8@HFLW`DA}z8%b{ z5Mfq`T#`}X^;WR856u11z0r35{sG-JJXMdPc4_;_#X)}Rb8K#W54$z3dlrSs4?HUw z9xp3Gt@(+MKrZKoo?K2{l_ztBaRgQrx_5+SMQj`mH_dr-kb(WOR}6#HpAU+qG-Oq% zUa^ngGMC&3)MI}i6b!~2B?jfQOn*z$nvAneT&^plxccPUx8;;o8EHcSRc!3+Iz7Mc z*KwTRzToMfd9Zgn>Q`E4^>g(*3H&>n8msJHiZpPM;Cy}NBR=rWO6S?&rz3mbh_n5a z;0abO>toiUI-c8se3a^ux%gv~&-<7bvagTo{(#*v*O1Lk4^2?9rv2D-rV zArBV2*To#3hv%<5iL-Ku_QbOAGE{^N*Y;R`5OgI_2axE7T?v`9nt5aC$NO-l!nII| z<(l8TQ5c>9JrPCnl*{&ddSpP(bNZ&rhNPS1f?Ip49*j;KcKX{59kU5< zNpCkkrft7G``WTXEXTfvL+0uO;)BDa$m@PI*N7xMi^6F*$rx&#O4iET$X|ywxZR*{ zO|H0mVy#(UJ(P!EVAguWl30{BjNF7lBVIN!bBn&-_C}Vier1vyeNK)CTdB0QHcoXd zUs9~*X-|`!i}5#;3G$imt9b>__Q!%8N05^R2P-#5q9{{}gX|o4reesMZP#t4uH*TS z9u@N*i^}Uh3f9$j|EiP9oGAIc^tw~yllXv2=*F?JrJ?EAqNr<( zIXTvCAMWTr;%sljh)^hqd^;jw^ooo|RXtq`WwlOZ#Fxn?K6Get4ZE!vzOG>C@+%tg zS11?-^Z<3oOIbDj>dSF@ui?2o?Yi9$)HEBuahv^RYUyi%?*CR5(bkJ89DQu9b=q2I zi%FHMo%TgsbVsrLR9+>PIsr!^qd8@w34;?moFtIFA1+_EY$Sr#z}fiX=@J`6Gq&hYDufLS~=g) ze0mMOoE0~Yc>S>B)Az!s+O`PN& z-fXcd6-w~{qd}V?FDWgQDDuY2ODF#0U2#^#T~&2FkaTvPDKnpk<_FDv^w$Y;s!q>u zZR>TdIJ;=$ta{s8GjPTkPkeQAJ<6=un29j?sKORF*Ng4CI=wK)`Bw{L@6FFKsst`) zzin|Q?#!0T@$|R+-QSzUMt&dRjz_<&!W6SU#zb!=;5-kR`K2}%4d$|*|8%iuv)f?k zq55hfn2kjEo0d!^c@rfpdjq;AJmeJ0cof0^+bG!^on^Gix~n1ecuFb}o;1JzBzn!c zKXpaSM@l}g;`l?;es9jdNXrBFxtmt}@&2LJ?y5wmem$|Gkv_V=8vmS$8OTu=FR##i z@-C`)mnv#aQchKOWsvf?0Q+;m3I5)*X(Q7OmXQQc;<((UUL^-2w+}fZryb0~nj`;`D9&|lCct%j8?BwSK0?q*#D?%Jw6H<=br(5yE_2?vT~12!DhWN^ zjK~;8Lf`#^Rhv3&xg(?2Ch-w<%Et11?ah|8oADx*cn>5K^zudnmL(il^=$`*7MgTu zeZLEn>N~f-@Rrmh(N&X_iO4By?-Y)_N}%W(*%zrO=hf;Qlx$QZ>0;&bug4qK^H%X> znnx0Zkw877BGPox`+CjVkzMh-X~PfesQ6otXrf7*4?B zeC#Vkq26bgEmIOO*#oe98axVR&ZF}G7l+!R8t-o8(hUU zR*+}$uaUFt?2byLPEc==%tkvN?`?NlFaG+H$xD!HJ+X4(PI-%j3CAX988ePBW{>-` zhS$!y$Tkaq*0_C$&AHyadbL^TC7I}{9re^lgWJvllnli!U-n~M&7!F%jS zFuMz^;U39k&1JWq0BGS!Z~FvYbFaNQdXDMq^zKCU&` zuVQkpvmVzNHLc7Ai&T>2NmuY!&ekmsdfyK4oELsv%-D8j`}F1T=ZHfz4LokrrtszL z;-rWG-#wR)H~y74rQwd1O)c5|879aM!q-VYx^W`3{5QZCf@@w{QrkdaSlz*y-xVqc zf6j!-zx$8fojg}*!3x2>iHxz2teM}3m1=dCX?yf~+9S62|FB?F#{qY4Las^xXyU4u z04=S^GhY$K@WzzACl61m@?9sSSKQsB0%u+OJ%opJKL*@mh>2e}|CcW!sTL&3Qp+)y z;$OVEBPzT+apu5X{gryh?o3MR59GjIMU!bVf?1Qm8ApOJH|<`rjM-OeS9ECi2pTnL z?P6~_Zsj|8*@f=d7Ee|eK+ktQMDug}S9?mo(RTcpmrl66)*S)ce+I9uidTZi)^tVWd;`}UU`rB8fb>h3VYA;m5i+6kB`7pk0uCMZlx;o40K3%*cY3}tnWZ}EH) z>GRmd%QUp<5LQ!@yI+n|VrjVFDy<4GTu3htd1n^OxBp|t~eTP0EjgFGNqkBY-;7AHi+Knaf4{PpB zpsBoj?J^9>3jnSMbQ#8tuP((4yK&JgCSw8AUfe~u#Or?*d#S{&VW&?X$fl)|LxROb z5LF=BaULEbfX@J5M}FmOCrgae{0{As^dxZ+@PRJF^7w$OwXz(-NWU~y=3>(>V$a<{ z^^J`UabRj;644cCFpJK+!=(e$h?)RO`_AT{El)&DYy<=KfcS?wj{(3%zyl#l2+&yH z*SJl4^U^UOV>d&xvKSEh0-oYF=uSc3jR2#+=jX2i+%5(B*sotzSGWM|Rn)0-CjvGI zh_6?rrT1Z|$~{0+4S*O1@!dnv4+BX-T7ZWif;7TJw`Zs@!4F`%X*duscj)PXXhK71 zOF~R^g`Rg$FA6Jv2*zoC1X? zm=P6iU?A=7+n2Dhv1!$zeUuUr^uK-U2ZR7{#D>uOh%gN>WlKp##Se71@OEhNky}xL z8z-ubje;gN==z}rxpP`t8n8HlDmkxJa=wFN{}*WUP8Jn?n$jP>87YS05WXGV-QQt` z8luyF`}QU>a`kW_rR|-fXvhVEN(C123R!61=nGN?z9#m{*49>$aT|{3-l!Q6;sLNp z&wG-?kw#YFVT3l0oCegMlt%&r#(Gwt5!!d1%%kZxL&#;dqg zVIU`jkvmtRFHr;L=+vL?X#){tF2f|~b5=WI3VoCA(9Jid?2U|4UCc@`f>B~f`zY{3 zz(ltIo!{Pg!7bd@BHueqIX__lMENu2-r=8?H%4*L2WiPospbas{g1$BKfAj6ud=}o z+(*R3ONI$J$2zL216CtuaCmTzA4?4cj0t`z0WxPK6VPJ@V+8B>FXLV*ee>o4zk%&l+S^9g?XhQ z7WfSYgb*`IIyzy{MDY#01{^4$9ih%g?7Z$V$aTPJgD!ic5e*mVYzEzzG+Ym1>Y%9t z`U8u*?f`+^sIdYg#Iwy_Q0xgn%L4Rz92_kdmL*B!K$9bA9QnaHj40p1clrQ(0zS$U zX43KdoB{PJjCA+_ukP>{x~`VE5Vw@w1x*g{ZR(eSOBiDphS|@E3KyQo7dnJkVT1~( zPYA#zalM3NbFh?Gh#YRvr8RYQTt{Z|rOE^A7$}Ah#D-08Ur`SyC+ABSmmtJJgTMs5 z0CLZ0Dr#zhK&v8ie&OgK-P3UV5OyDwA2@lB;S!;UYQ(9g)iC7-wAg<9h$SW_hVaJF zlZ)`)AlpNFJdulus6w9{Y%Ma*7gSUl?FSt;{gx928oqKIdof z8`*Kt5jkcU11*}rmWn!KV`DF_eg$!4_={W^zen1#F5YWSp*9K>(8cv0%rL_B zGKRsRm3xNAiaIGw;V5BU(Dq!U_ppyoT1fN}>q8sh<`99Yw`lM^xLmG~UTo-mYDS@` zW*o3pQS)|mT|$T!<}Ug5H^ zsZ6*?-PR!T;RD6%WBThfezeLDG~OFvVY=R6^L-T^ymB;-e&vC#REv`nx-Frha--8j zbYBX;S0$b9uP)ItUt_(-NPFYaE2*juo)0Q`p$B4D7Du_)E4yL6P z6m+&5XquuL(L=7>K>hanyS}a;uk=V>o~2~P0Wt!OJ+f>AGi|lQs)b}@c%OK1#4g>F@8)6MUSiLR9g^ z9ul9Jj~{g^zi@RBp-`An2A5F3x#1|f&jgS_Q2z{4iruwIZCD)@=w_TL>49G0xijy1 zci%!chX_g-_Sat03A1OPLK6{Y+_Pl(6p(&ts*#~y^#%gOrsU9{oZ)90)8E&JNx;UMhlOe!jfT5QC48d0 zCqPFB1FL#s9p_7WU^>CVNiy)Xi!hT?MOF1Z^v$hyUZdn`AGdADx0$Sdfy^QR@svPU za`hK`cmia4>FP+aBy`>kfy_Yzxk)ZsSa5MNa5PNx^q*gjF2ENk{{x+5HFSjxLc#&d zB|_Ch<^-b)8G;a9V`;Z_0YS5V?kPD>>{Fo1&D zd2|63ivK`l!SuzrVtf#Y8LxwxAPdmYJqw7iV7-6;DM%3H9zBX4Uu8S-f?xRx<8~i0 zz~YN|l!Klw4aA3x^z?zIF_%#M1+-hY?r5M;8@_O7mT-*(KHFsEO(5ILgXx`M7V~7L zlsG|4gLzpXu7UhK9W(u*QyJX;E(|1SI6KPIfw>+jxVLy8v#5Rf1Ul2mTU;)izaFe|-`!fT^k#;RQCMyrk*}njx251}m z5^d`>(Rbxv`rC#o3F2Y>#*W7G*vw0~ulbwS?p0_OErgSH_C=6C-sOYQHPTUb~g zRM$sl;UgQ5qNYPO{;o!`Q2-YgmmI5En6js5Epl-fJn&4yB>M@-!-&YpazNx3m+Sfb z`Dd7a;za?s5+z%NLMbu9fMb@RX64)x2!>-Nrf&qDU{qaK+%YeQ>Me+(YM`_$1F0f> zZd`6e^UN=hE_zYG2S?rUcm_5A$xazGQpJ>dLo%5LEv)fJ1BKEy+vjX%X7M2|3=4!SO=}32*Ou z=-%OmLAXW>=Q(W0ie5{{>7poA?whPA)B#L$KK_|N^GH!~m~YzCVW=9M*jSZA7U&C7 zLBe#zEcLMs`7_MjA@=yBk+7WwZTMpqwkhC{Jjv)$C?2XVZZs6UUXh2JCR|cw0rQrV zmCat_K%vC3p=mb*R2;cDXyVuse(2(vnsx4F(AB9rXo&m<%-EHNl_`KX56~b+iM(p1 zz{eu5M*07bf25iK{cSDZzpFZtqEM+<|G{8XxWQaOEEG?0T`n3*$pLCkMzG{u2x%xD z7%HRkEC`hvXo%Plyk5m2TZp_|FCqaq)#{;Dx>0qxs)&!+D64z~&g`!4*=Q$qZ``Nx zj`MVUE$2$ES=OM5yW}c8^yhlEj6Y@*72;3V|LvjVeL1gp?){*zRlnYrcGNXo&A=Yl zTw+3bzImfcU;l4Rx#(&7%J5B9YB#O({hydtZwegATc$k$8$^9?x%7`HWfH?ZmNAgLEC(a{l@1LdF{oKxh_d^*Echaw+F?3sTfS} zI~Kj=3-A9${805p1HaP~;!fEFB`lge$3yg`>h*7zGu=THKBkuZqVaFMD zY>8KlNtlb}lrEr7?q3g0B%Y+`>x=trp_}AQPC0yZa{lShw9Go?4&(mzF_HZg#C_%|e}JRCkF}it^21(T1FlkkV2P312US1w?b~TL%m4Ovj^1#b)Q??mmbA&V9n08cXYai? zD>|V2bePf3Y^oz|iC%Gh$m*9qgU=VC*1JoAGQ>>IOXPJ8_v8cmuf%^!5WA}!ZTyGm zP&QPpGNV?((y=T4z;8PIk&>f|w4vMn+Bx%D+Lg)D9r50Csrf9+CgaWOncdY+RxY)A z$IaxUx}dZDcgcxc(KeKc8RPVxIqD5UUsM#amBUWT6d5NBJ4&oDtIluCeeicLv=yJ3 zAZY1H{H=WS$2Gq_w{Xfuig|-|dbL9%r^6>qrAE4pP_?kEpl*n6l+H1-QUiX7`2I~cS z@+2zDg2yWVX9Z3_OP90TU}VJl@2ca6YjPr<#kza)RLY%Xx3q{s@C@jNU_|I= zb&}gIMtmqddibI6uBMp9%+un~+Ke0vjDtM2PrOTGwqK@B&r;X*nFJNbniv*r*CZx& z#|z#_cmMkOATH;@WbD%Js|)5)MtIQ ziCf`h2fsWWR{y;d8}1n~#BmSfOtyL_?31%7@cG}ua73~#Ewz>reL)Yejd9o}1?RS& zJpZTejuh8^cDoci%4zqrg7D`X{+sk`v#0du14TM7=0C_RBra#2ns5o5M(%&#$4Ram z5lhxGW6h_fZgW}- zn#M6Fk4LiE74!`81m-`bVTXSIy>eLn=-&KGA_|O}Dyw!z2YEW}Ueb^1@eST_9wi&m zCa_tl_xJAY{4^&}p7Ofi8b6`%d<%OwT>mtAzU)U^#ln>76!ovB19WHZ*?K{fpY~tn ze}nl9l{0qP@l$Q5R|>Xnt-nY<9ZuDMw+L(v|Q)8_t|0d}4FnoY{E$-*)>1Ggu=ky{n<3pIja`HM=We`Vcf?@e-U zKWkv@FZ;gx?t`1Qu`$baf-KO@UGIt~*{8^2#2f1H1HSK*ZtaO8=4 z!xbFf3`-+I@eD&$wqacRKL|>?(>=&ZyT}abt=SiCl{7omupYs8g4C;&qo*F_1)p4{ z(?ID|3d@x3lzL&-ZEF2+|J+dTc~>jMxH?x~K8W>^44?I{E~X$;9_vib{ps9DdhCJF zxb*=cpOw@nzM4mln`+jbaVF@?1)Q8(0;W-F^%QATI2~?>JVz#-seaQ6WZCFw3GK@- z(B*ZL2SV?X^@|?UBv!9k)*Ht>U!o}5^KiO{+)t_5< zTi)IzOyIcodci%d3Qq0v&#Qy=zR$WV+`k^zG3+i?v!c6D-fMFuhthZgxr^$;l<~#z}GVkC8}>_Bes)d$VqpQ%4&0 zs^bbqLygLh1=QR=XVtTc_99~f%a0T$<39-w3uk;kf6gehAM9M{t*b2N`K7R{#`f)B ztl@v7_qjO^SVTk*nDPZb=23s1%8R~H>U6s8z%x@V$ov8~bO9{ub2+G-7;kp=B_peZj<jmz&Q@<}qtt&li7RST)9k2za@t1S!-`)z+7m3pvC6(H4?qh1uZ8SC2Z*~{MP zd{3a(G5u_>qp&1kZ?G%Hdid<44&_u#dPShKQe!WV`(lWI-0q?y<$}DnY`uU_xP2l{ zZyQb= z0irgWPBQCgQ(RpQ!N!hS5AJ0?sRE3CC!RoVE=cT`9@1yPhhN)|K zGK(qH?75m)j_D}8Ikq&yS?bEndp|dJCc{c)J~Q$ue;v=H4Y=kF8a8N?x<(t z;n?!a*fCD`g2e#yj^Dc&p!8oojzqJ!Y2UCGxtPDnmuD@K^>KIH zva$aY(cBKB)3_o!i$VOU*4ZNY+=0_CG~3$jOL2T^ukLH4O$NSbWpT*~i93FNBcyjn z05^Aa=u!3YPf>DyQB9{c_owS3zaE%^24}x5?QyO7>=_0{v~ z*8Bfv9WM@0Nay_7ZIC3$+J84~*2B#EhnDA>BXjB#hk$)IP0I+fx%Kf+k8N$1I# zQoydB&TVgfV9`q_H-i(&wQt%!JGeWnsF27PlhUC)wuY5dv;01j6{V`!J&W_Z$uuuN zn=PDBX~Ha-b+yy6%N56bsOwR}T_UW&SSC^e>pPlHOsTk3%mvn^nYpGUViR8TFm9#T zvoMNdue|N(QjduCV`0*nd(Jd&ZGwKKyx;3f@wA@_apV9q<^z&aH3u$@% zAz7OoD;JTc_o+*+Vpi&reai_2ku+DUtQuP8s$JKQ~9|ao)pB zQDUYRJI1c6d4*f!|2)}5>W{#ElN1oTj`mufz z8Li^d)AK>Qe0rD6)XX%F-}Z@S=VfuH@7IV?By23=3QHeXCFv?S-$n~lJ>4)46P|aq zfB5!qbe?eJ2x~`Q&a{KkJF2-9%f;6YtsZZiZG={dqGxZtXWke4a`pgor_{vW{{8G~ zV7Ru8%eQYXHIwLKRFT5Te#4lUmG523KM=FLznj&t9cFB$DR`D3Ia7FNxm!ri!M!=o zW!l~czBETW#`4a7!JJn0*~`1rQSZ0--^cPM?CG(!uaaArn1VXV$(L_Yl_8_i-q3xTA@Q zg~?icuS*5|6rFSts`8~~tX0bdO8%`?*+5Bjid6&eAc5k;rTIS5`7+kgJ7kg3*~+ly^T@ZJEO9X ztd*yDWAAj)i)o{C7YL-L_)M`s6_Spu!^>LB!XoK9HVjZ(+m;Ov&HGn~$}%a5T+(>- zX!z|>i=EoL`p}xpf>>`ZMR~5{+`|d2p7)ZvLKRArwT^Y0Kc&5nw(%+3x7$}c;rm}2nBC^2MRw=ut@*vahy2K? zeh@oRqf*AoBjeU0Tvj$%n2(p$n-;FkHWXtB^uePe=??iG11t@ z!*D2LO5wmH=FxCg;TA)z;PcpJa`XLLMIXPVVWsdTZF{eM-}x_HN>5Dd25bk1clq=` zq4seTcg{TLrAJ6x$9X1p4&B^F;Z&50o_Zx_aExdS0A@{4O za`@|_XVR^b@8yhUx76?Cm6P+~L`8U~`6wt370?~pFPwa~PEqLnRYPyNJj&%*E)7ts z?-$7wi}}Q9%k!$6nRj>JG-;6kE3Xu_a9p|ZBk8`(I{O&ic{YE`SJ7R{X!};}d;GgK zi5@w{IOa2z$2((jLXM0*nZ_@s|0OBY?e^#kV=j5WaVNZ=%g*Bc>he8#wcp}*pGo;- zvsurSlGZLJR*jXr)ofADEO`d_Tpwpb>fdC57qYnl>_omXb%_IWaCPL7apCo$-js6l zrh&|*ZP|49Fx~7+Jw63(1Qq|f1}5%LXI1e0;#ob({IrHvn)sdB zKZ}aU_mK1#B6(~-{$)93cQEF3V3O(4(5CifdF)RNFOGhd3m(k9sb*MG;lXz9#2-+~ zXOA3MU-wZCnKtWmIy%k$^u}gwRHO4m1NW>T=kh75W~}Z#Y7M>0CFz+T@unS^|MEtI z!nW#3bCeop8yuv~`}Q2_YMxOfYGHdxTS z^yOADoS}z9RQKHYnI;_V~h|$qmmekW&6FTn@xi6KZ(uCH%RRToCRvMJw#! zvD^P^=~Hx9#?J}*bgWwBu9ADajjf-23ft=Pm@Qpx0rx6^=Xif!DDvvgxA4DzpSgW{ z^DBV=`<{ref97)W?RBgFaa}F-`medi{PwsX-CGG9gG+g@vwQKOwxsaX=qdBoCw*9y zzsoG^>a4?`vabi*Mo-`H7ucOt0G=in4jd$3n!^n|>Q#q93wVTk3ve)Ql~vE=H4a~Q ztlpk>q%`3Dw$dva7yS8_2Lm^R28Hym{&J^7`^w8N7hg_up7?jyYhWKyL@H|cMnYlpsBn-XAd zyZ6ies?>}ndU*mX8g{J}F)Z07W+T1SA@URd2m7CAu8J*su{~(Zuf6Ibw-+}@UE!Tz zykcQgYy9fE#qpO9`KxQZ`oc0b)W3Y+tQFJWb%xwNw)M%1GvP{2>n|72uURAB?y@TP zumkIr%#|GrFK?|l^>$T#mBi9J=EV<+ZKZyPuPwQ=#&ffL@bg1Yz5iVKb+7)_TKkra z*HbKbXYc?0C%SR5Np9dgvvmojd2^QdKkE&i-={0`W`p*ILiW3VF8q%ARwZw0b^N8U zgry^QdhZq9T-X8Saj{nu#r}JN}T;1HY^2w#PeDf2k_FE-?)xU}jWL8||8T~`2 zt?1j?Kfc~y-^c5eN*Dx$Pn+;o%ll^6B;e>JaANq>O$i2%O=l!Gf1G-+Q~e@v)2FDI z0K)<+%OyvhZ?FD;DbDLhX5X)?y0L5bgw_YP%LPBJ`I4PmvyEH0=^<0_zPiSs6Y=Yp zTa{luvbQ|z+cB5Bch~GnpSfzY&04wYomFQtmaep~StY1@{_CA>I;o5cvO-+ynpaCR z9N;c3y{)|X>dzDHzmC6&?~)8H^pLmtQEmI>SWvEqy;gYLv~TPDFYIW)e@UM!@cg}p zD&Yl;S}Wrh?6I-7zg)k~Qq(YS5!KjInMg@*3k6j=W^Ri$_sX^ zckwKmY+~@?M~tNEzaQFJr~L9hY2V!WoAuP`Yh`P~p37Z6wehXzxiHP?W`5Ov+*iZT z_q<g#vzU(P)Kd!@_7UN7SLD{+U?=K;s}uMPfoUuzezF`N$?6t)w+v1_`U@9U+3p|U4h zkJL8^ZK~aMhYvV@d#WS79XycB5C@#d0go2Wiw6x+H*kjq1x*6p)VdlpD$Kzt#xOMv zcmd0ya&-KtJ-BMDV>+ z>YI*3(3Ak=6?&|g)!)1G>1_O7f z0}nS^5Ag=WiY!po*U_O-Ub`OVqXeLbz+gSm0Si`6F!+g3byHO}tQ0sk1p-%Z{%4Ng VTKPmKAlwzi_jL7hS?83{1OOC&nhF2_ literal 0 HcmV?d00001 diff --git a/docs/src/assets/images/transformations.dot b/docs/src/assets/images/transformations.dot new file mode 100644 index 000000000..2ca40ddd0 --- /dev/null +++ b/docs/src/assets/images/transformations.dot @@ -0,0 +1,28 @@ +digraph { + # Nodes. + tilde_node [shape=box, label="x ~ Normal()", fontname="Courier"]; + base_node [shape=box, label=< vn = @varname(x)
dist = Normal()
x, vi = ... >, fontname="Courier"]; + assume [shape=box, label="assume(vn, dist, vi)", fontname="Courier"]; + + iflinked [label=< if istrans(vi, vn) >, fontname="Courier"]; + + without_linking [shape=box, label="f = from_internal_transform(vi, vn, dist)", styled=dashed, fontname="Courier"]; + with_linking [shape=box, label="f = from_linked_internal_transform(vi, vn, dist)", styled=dashed, fontname="Courier"]; + + with_logabsdetjac [shape=box, label="x, logjac = with_logabsdet_jacobian(f, getindex_internal(vi, vn, dist))", styled=dashed, fontname="Courier"]; + return [shape=box, label=< return x, logpdf(dist, x) - logjac, vi >, styled=dashed, fontname="Courier"]; + + # Edges. + tilde_node -> base_node [style=dashed, label=< @model>, fontname="Courier"] + base_node -> assume [style=dashed, label=" tilde-pipeline", fontname="Courier"]; + + assume -> iflinked; + + iflinked -> without_linking [label=< false>, fontname="Courier"]; + iflinked -> with_linking [label=< true>, fontname="Courier"]; + + without_linking -> with_logabsdetjac; + with_linking -> with_logabsdetjac; + + with_logabsdetjac -> return; +} diff --git a/docs/src/assets/images/transformations.dot.png b/docs/src/assets/images/transformations.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..1343a81e701af229c436f68d0704646923107450 GIT binary patch literal 56255 zcmcG$2UJsCv@RO^FGVaAK>?K>no^}BNbe*e(xpR?-lU@_C|!E*H31?e^r9lYmw2h0)a>sunKvx++ zpv&EVT?am)=tlv6gRUDZ$-M-f6aJ*v=S6`)k3b49UqIg`uTHvok-MC=ZtcHm&kgLm zlA*Y9{YBqjPgpbkSfniPG&{D9s`~mpd}#dLQ9$))&rKrDyX4noUU*PsD{v3oe1#=b~^Rbno|01&8-VOVpB;^NXT^5fIX z7guj7jILf>Y=U()s6e1^F`qtvX2ac8fB#ZOhQj402&4xmx&HMkNH3VAiIwoVef3q4 z`YB?am;}#6$n@T==w=qFEP>i! zn^Y1aF zAP^BbVTzhx4HqlWR6fed?_?Dytq8&P^@f8T`-pM3lSdwkBaU|MT?Y3(w(v5Fct>v# z==Z>FU>4N%wBM88wO5|@5{Ix9Ck37e$kf$na}7o>PZhb@;j4FLx*5BDtV4#cO`|O` zQH!2;OrqB_^NwI6XE*IE|GZFp2PGIld0{|IQGo{@kv){wYBhGr7TL{Ea&$2MHs)Nn ze4VJeFIgPXUs;K>;26l8tz6Ps?Ya#5h7SX#+y0a{s@&B6-r*=Rf4}_sd_$4oy#-vm z$E{J6jP=pXAJ-;3$f-~GvH<5`6*spdB&7nvE4!y#l8_}TWtGFnL zv_~xXKQ29R?PCcX#8@l@<0v&`GQQ8l^17R7cC6nkFOuMImKL;pQ{$_!@Q%4vfZRiK zz<$hVA6cv@5|I$W=h8=oafj9^4xKI5Kc#80nFeMb4~|&Vm0e_H3(+Zhqko zztx+Z+iaDA>l4!Sp-LDZUZQhBS|Oa?gb0}H<6AC)Brh=lTTAaeF)R8WL=Zq7b?_!hG}&9=tUOq>>FT;#=%SsLQTxaE<(Wf@T1oAodmWOGVfsq%}K)) z#ap${xTOscY??iKmt_TiV6Qx7@zdk%HJHdcL9 z27%HK^j$82)bD%I;1WKkv4ZVCpJCmyV_8HnE?v?dj^h}M7L1r3U7nHUmGKq=LLK@566p}XK%3Mnh)VqJl|Nedcs}c%~v3N>x-g> z^ci!bS-XFFSxj$kAS)M7Gd_^yMUoW`RRun16EusNq7hdJ+^(T_+Za9&TrAl8YF94K zrQH((8LM?pzFtAnT`l=^zOYYCf=&(l4(*A|^bDqKm9tvkg3=~VC{c*j;>X86Bu9MD z_!?W!Z~Vi{M%-DET3g$ZnZ7?(+aEjNAV1g`pK{M$@9~I0(Re(U(nk*@{7BOWB90vY z8vCZ0Q>XS>Bl!&wh@9fWKlxR@0`K>=M)?dbgl**adRtSJBMRPos`SIv1pCZLvfj!L zFZAM1zH*o2du>4l;~3KpSI&g(*i>=exQqd^X&+e8<;!DYxi_rCM*?RXu>HmLy$Oj0 z{7wsO2024(Mcli-Yn`TnV3^Kwf4Iaz3)33@`*vD6D zCbq2+8Wug5{J33R9u*eEvBK?nEHczahnluq&Tex&E3LH23fQwF_g$?L1#_da%4EHh zGxn$6s?^IBj;v=XD7v}#tX7`>y3_2LT{_Gz`j@tYu>DU4CI6?es%$wnE!bbEN2ux- z4mZL(vk;>1E`ieT&(x8CSYMEQET_(C=bp+wXl}HWi+Pv0Idi4v&lL6E)5*PEmLscE zgA`?z{?B5KRtv37yT55kq0jo%z8kr?=yF)Ded#cF%qNiIx& zFvFi^zJykr0!K`KZTfeAm@>lG@X}xaR%_lt;tWQ}b0~nhm)Gq~O zq^EyNPP;N^7w?!WVy5V5LtVu6rG9rH``VcEQw^)?_kSs9#wx-2o&9-LmTe&cX!OIl zUmu!BdLOq#J#>y>u6KZ_a4i(@QR=17-K^Fp)*KTfkH0L*<5OMx#$JE{x5@8io$jJO3Y_U`iMo}T3@R|BzE7yYhNGi|uzI z2}4)GdrI*?nz+2!UcYk8)#WIa=iz8cyJsrz=qVb>Q*PNMqLWBosMV|5MOXP`kaLjF z%$%z)YUY}K^~`2m8a5)~OGb#&$w#wn3wfTo#SOQ^Rv?~}gMoM@*>03NkZDxU?`4(l z;U%olJ3N-tPj(CXMsJJ$ci&$txvlHO*s#yD<sUv+A#1!zr>HPZ3s5Mac5qQS~r$95?o=1o$N}i&OG_+-iDuT6E&|o!{*(*Xt(iS zx^~HP3|8TXrk3d7-(!0T!A>$9s^`izE0dwQd5zGzK+F?0OSkGTiQosvWh!3W=^XU-L z8FQxl+MJKxZyZ({tiRGqy)wOgC}OM578riH$xiG1+(ygen+gpd?ghn5wa+Kla6NKI z$N-0zWvO;qi0`hWCmqNsegE>(=6gZNjTs|5qJ#kTEfA(GGDfX0eZu6?dbKNDKFiU_?x`^u7AW~YZA(*#ZNu~ z!0lroFCTEH`7avr96@@r!ELwcfs;Y^?wNs>60~(d4KBvW#s$L-y9V6qZx9WDC7Qe0 zt|qSSLd=cJuKeB(DFcSP<|@GFj{Q9U3rzk06fXkS{vUy7DFqOML3#x->~Kn#n^iqq zg4aQkGf$NV`GRWz+{zVabYDjQvf^#7I1q?cy`-FgN}{TS>bx{GG$OO(E1cH4sSgQ%mRrDgqP-MKU(c$*chv08sZdUe5g=bxJ_h>QZmU$`z6XY`{1k zpfl6>Tq{Z!0Dq@%%E-tlL;?#RRCO1)?ML&#|7->Rp15x!G=jEzLrmmsyf)~7kXJ>R zP}_dBQ&gLSyOQYijWKR(Jgv07><%!k@eXA9E&s@ zFG4f?-6se-V%%KB3Gf^i4qsq9^$_QvR^aq0+v+$ zwd%&N^vBOwppEbwp=Y`({OI58zO}H&^Xy2w#$lYfNP0MBYBVaJ7}<L9v8MdtG&p zf2~%mqpAdxYQ5llc8@G=0N2MMKUnz?`@PHA8^1v@<_K}ZD~=a9jF&?5^x5mj#&Wg{ z3;PQhG}m->mN3UZphvw1vflyT z0!CrlMpeD874DKK(6{5ZI4JhBYbXpm&ganl*yed;E_6_i0tJOGhJ|9wiY2I&fS7f; z2Mt&$S+%#0RcpPBEwE#{_8?L@Pk>7YT@p-Y4G)6mYQ~SnRJJKQfZMwam>o`Kz2j30 zlp#8TZbh=I9YaUn%iHo}6 z@a>vszW*6^C@wb}u=Xk8gsX#;*T&uTZa$_M;^~T*>z|V8woUL!MG zA!POeIvnMlRN$oIs|~q}dA)}SHSpFnJg-vQ7j^LOc#s=bIF`K}C`Od81?S!^M(cL! zy5N5Mc#B%4EVwgz79$}7=f#z9=FWU|m0nltbP?<8Y;DenjS77a3G<|du8CqU(xif$ zZ~(uPv%dEs{18!LXHgiKJ`+r-P&^x?AD>?O<&H<4QTpXo@tTb0P@1<{cE#)D0hdv5 zXcP$5fn`G{YDs_R|1reahb+Z(6a>J8V^*Wt!Yk}Vav|FqNyj=!c#7n;QI2hHm~4vY zM0RlwsoAPf<}@#3LV`}?X*N&jYq8tdmuo*BZkLQf8;(e6PAZW^Vkq-CAKeFeSC#UQ z#D_G+aMw}MlM&cfI3`hGV8=$WJ<=YE3QHYN20X4Frz2px;wP0P0fk7y5#D4M)E7=! z_=24-21#!-rVh}vX<>^Dz$488`gl}eTg?u-`NYeHhml%b;E*QrTw5YZczL@mT6Zgt zXTIbE(g;)58W14{4gJBH*dV0e8mZF$CBr07xm`mM&z^eP)r?l8#(rHL%#~=|%wCqP z)6Hekn4L~7-ijwUIin%;FR?e-U9V+F-^}Mv)+c0v1sHQ7M7sfRtN2yjSKyS~6 z4Cyu0@MqUZB6DX%rFGnR5IK_iOD#uc;mTOiQu3#W6N1F6L(FYKY{S;{!tiIS@w2r{ftJs(d* zmKQ3o>DL;h74}yB6u4)Bt~EF%0_P}mCRFP@tJRGDv&YK7_M$8gjV-(i38OJWEU>qK zU1>Qon=@fvN&2Ot+jW zQ#f$iph~uJr!20)xP*CQyx7R@O>aWG`k`zLc609WJWZ+8W0fSUzV6Q+W>|sx8mDph zam39$`#t(yom&q`Wsy1j34{8mc6L6e$|r%N9T#%*aOCb0Z5 z7fitx)e(J5E2ReJthRzf#L5Yh$aS}P)Q8J$oR%9VshW2*6AJ>Uh$FyiLB#kTa1{nH zYzp9?mpGWJXWguqjvtWII|NVFN+{>J-vnaIb3+ajAX58XHsN&hH#qV%`DkO1A=g?w z7{d8kICg=u2@HHWu>@cM$(#LE4kqX~Ea?c^`ZK9`t>Rxf^gf5Mdtup2(A>Gd{(`Rb zR9cy$BxU=qgVH}_B>{O1&^DQCtS&A<8~^u2UG!}LJPhQjBn8!w0G&|hS|!Tp15k(c zKOrCc3rs{9=Kn|nUgY*5qA4mvuZ%cDA8p&p<1V*?f`Z8G&CN~TY3(Ad(1${r;iB80 zMy!ERn5gUq0puoJ?tAWo&DW5o66t;rkWufskJP~+43L0DewDy)v)4yz6(#DFTRskE zI)~I7-A9a;GQ9wv-xMaqm}1E35kAW3{-J8+D-PTUI*p1Q*gkEtf&k}M4B6V+A|fUh zv7Y!eQK*HwMK4~sJ>ThB4ZtP!AN?vFJLY z20q_oIq;@%0`M*XgaJSw^bGK+*M2+=$>h3JWoz$#9aPnG0V3631G}Z=-lXvxAWh^U zk>=E`;J|4{gvZC<$FGi-<#z%T*Y`K_yDbGQ;Gg%vVfMTGz8J_Q(yM;k0J8$Lzu95Q zo6b7KX$72eq3;rtyf7;rLjiFNbMTBnW_qLF3IHy@HVHh%$~+KGtGW6^OsAaTMPQ^p z&*^0t`UR<&mzq95w+vPA`)G_=VStYh^vgzF`mOKqS^;`~Z;z@P?;&08h6ee4?>^N_ z*VKWZGs3Gwz|3=^!n@;W-1(Gn5uS7;N^PQOV`?>bxpyL8%gT4@<1-Xy4w*JJOTw#A zHcgY|@_m0LVF`E5@tNs=p8*DzZ2n5A)DJTK+);Fj(jS3VYC zdQ%AJnB1<%%;W5O(i@lS)@BRU_r%6{g*6+7d2kd-tPsuNu^EJAC93}uM!4ii8XsO* zFmQGThSa8{9(mOA1ihx>YD>wtnqJdr2^GXuo1ZPF#d|BFq{|-BPP;>I@Tx8O!;Oc` zhc?E%rY4IEnt~TllM)t|a{m4L{RJ3H*ti{Jnwo>;ik&GpJ}E(%Jt9hIx^;g^`nb;-yao z7$?T5T0QcN!Qd^a z%Za4}?t6Uus?EyUobc-6pmlQOL>Bn%E#*G{eetZDo%`TtV){3IIR@myUoK-Xi%c|B zF|_vgc|wT2IlzPGazfk%9Jc1=p^VAn-Tvg~Op7|-lz*Y%;`-=OPfeGxd9Cfwvx(2# zKQ?qts?6|M%j-32(A&xnFNg3ICDm#%H0Wg6?d7mHB^0X!3oa#9C0BQoa0Vx!eU(Rx zWsWtSJC?t&^Ge@v+Zbd?ryZskyce)*lEyPni&;?i)E|NNAYA@SZI4Dh_fj|uzSNq~Q4 zP*;WH27WD?NwJr|xt?EkP~a`JTDN@c#)3ghk%VzW)Z*yqSj+53;8%4tK_S571TQRme&xI9HdZ?2 zxHj#XABOBo{j2V+xbPj9!T^i!V){#0%ZevqTdPcTd*WF+107*%_l!J5$JQfPjjN6rp4`A}X) zYQn+i%*%00hTd`r_fm*%t7tZhMfS>`AoB`#{C&)RB3AULoES zW!I@eb24e#?YpSzaw|HU!zNx>B}t5b_sy=&B7bG&6c2PZzy(SX%!q~T$EEf!-b?Kp zpskHYYJpv-$1C}LRdn^x6hrHCfBG|K3p5WSR)zIjDQcg-Fk={B*$Q}9s&J7RRn2}n ztLOE+sq_(9gBqjsF5mJSo`P#sK4)wnDt5V1-b$h#p473dOlgG5Zq1rVJn9sv-!;kn zqgp%XoMfT%=iYk#yrpEDfG-w>SuF zC$IU{wz0OtyjUk|Iy_A)po4YO5fQ;iO;l*0p>nyfl#rGU`yXY=#wT9YF+;&r`3nP` z!J}*tt*{DQ3Oza8A6p?Tr}u=OX9 zQgp>vD9=Qf>X|J`hX?#|SkPqxkTye z%_cJ0c1uVZ5;>D~zz7i*46)B;o3!cJ#K+9O_q`GzH5M{3M-^8vziH@PwKte}JaeT3 zkJR9i^28@zdOFoWiP+zI;}WN8+xicU zm9B)zB7Cr$rAdtn>3GFJE5li?`-Y6w=&=}PwcZuyn!(qy2|N593^%US?h$G6^)WIq zA`LJ)IV96|JWqBC5zS?52V4jfWczK`YPeO033y~~|p3RP;@ zKUx6e!0~D&r$yVi!RUz_lvaBIbHQf;JOjLp)Z07zoq6m)5iG(I;1R22@6XqQkJfK@ zpScH_H@m@p&Dpv1xt66a!{Ec2T(f*+1iZ7Wf#h32(c`4 z*i6SUR=z*e;rLoAJUpFlSh+T`9jyp1+lvU(Rz!Ke>@!rO?ow_}vG3R|%3R{Y4*fbC zG1r$1x1P0v5VfJXq$#sNwmInB0OFFX`3P+k2MgHppiSPTF4^VR4;X*flirdh;e{`_ z`TbUte_tfgGNKpFB@{`5W1`4mLX8$ewzOhi%C=4L7*zD)XEG3o>YMvrkiE@Q#g)`0 zE^tzfWE6|az{oWHmn@ZX4=DEuM0+p(j=3r-=tP_@<)QC};Vi%2-C!h)C^9T{AeBR5 zZyVQNT>dAnXz;uu7tF~H@f;~0VOi;QU->wbHOvWSW8A6o(JnJ|5;SXD48S4Ww9;|gp~#K!TdK6_OZ#rfmZ*EXTeIHy zmh8A8VK&(-ndhkPLjhk0L~E=>w3%K0r<|_>e9@-VH8TB7&ebj`6}TKY(ppN0p97 zii#^9AM>x|O`G+WFXvLufZeoAt|#e~BXhr!`dS`_T+7tQstQAck{=}Su`oT10_uH}} zVyo1=fl#f<&h_WpI*mc#f7#>dC^K?4CVbAS7>Yx68o=7aV{S{gxIdTz2~Fp7$YoG5 ztxsivT{wbAjfK4)$|B>&=Gnd9S*e9ASa;DoGF!K$-IeEgh!`l25%;W4qHGvfnHHVa zxpF*}^AXJl-mw^&g<9JSg43%We)|uQ#F3Q-jQ}_DG>9KM4wQNfL z-H(Xu1hSFAD`n&C<|B-g^qRLB-(OzSExliIFdnd1P@W#D;$~Fgq zd*+DMpWetZ-}obaquapK3Q`O}auxK>59zXdL~Nu}Z5-!wFb`LG9+?SAh-7%AUL-P| z`=#jevaYt@8`89aq;TJi+G%g49r~kJZrAm_5>Bei9adKoGm2>h3)d*>A6*5~&HzHD zX8gnc>T;~>othkd4Y=mrs|Azs`i^02pIk<%ZlN$s@bUHgAQbd?-9Dx9?n!yb4Nx zdXbn{Yv$oDNl5$XHx3kJ7SG=`T}#Y%Al3}32{C4{?tDdhdXoR-Mm>-b0wt@gTs1Pd zpf{Sz&@dayJ#qt>K|ufyYWn!q1ravL(jr6W|obf?YbsmtgzmlAA#LGjBnjubt`hS!pC`<}{`RtYWM z?q8B{|`)ZxYmLdZ`w8hBlUIUu3sFNK|2Yv4 zy@B`QlCc21y|M zxZ?FJK9keODZ;d9eWk{s2zbMw0{r7T-!oR#yGsowSBh5P5D@G=5V_UoYjFyn-1Zum`d)I8`1oF@IrJ747CJK!DmE6mv_?1R zkGZ!yS!L(4Tle5dxhR2GZ3~>!SdagBhCU@FayeklS7aAE+LH6A3y>-3^c{$-a||hb zt}WtQu*^M)HjjBfH~W_czD0PU8ip(=G?63sRula?ZciW6ArJIcZWS_a1#c>_%AD2DyyuELHdS zA`deJT|BeKv9lZgc-&GiMVWozU38uX@!B2Sd7Fx3&x5@7+TO_$K3d_onsmBXFt*gl zvQpaVB>QS9W#zPQ+FYifKG^7hw9ZWa=y9_L^gN`-*t1E=>S3bx^LngNud?s7^|@3FYi z&!e+irZr`{0j7UTjoSbGJE;-I{69#I=7RrD`oIM`RVs?w=;?}7G0B&>G1!9neJ%dE zYVe>V)jE0;X=d6!mK$T;26 zp(l>-Djuq8<&NhJ2Ko$!EVGkuY_ak)t(`0!4)cH?Alfhs4^(vA^r+#_;{7UtrJ&5L zrQaw*2Ig6B|Cw0h@N$NU+Oj zQ|gPP3PyItVQ$e;e|(1|?2wNDtlRXp05=%yJQXY!L_Mubhm{WHiB4V>XE(jsusH2e zZKCad%PVhjqv}P@LvbRKM(xw{O6}!Kf>wR`H(WS(T z4K6<>Tk;3g%&(kp$P6LVCFyG~cTA?o`g*FSaV8gKsBFL0&eKrbucty)(uF{a950np zy%fA=SHeFe9WK+cF%fBl;eUOk*-WyLFe&q+m8-E25wx_e7OKl9$Dn&esgy8HBNaEHO~+s85uNb1=%Xsu4peAVxbADN2!p2(tew(fJI z1TIQBQ=8PHv^^R`pO~Ubv3Ud(sOZo&)g66l1k!M$`-uWJiJ6(1K{e#RFGuq2%y@Xb zcJ*vtP0T))-cI6%uwUob99KhHbqx#Ayuym~+Wjr1>b-nFo&tc%oUs?Wk*hcR#r)GW zelBn9e*anDVOK@QYn0ofI+2Gw4J=5Ddk@s86z}#F1ND!WCH!8BcH(ro^BWC0hj}}R zO*x?^h9e{{SoWqd@VD`606lcM?+PHE!VLO$v8wM2n?NWs5 zEzthfJCVEfXyf1=Pvnln$>XLU0o(#;D?)i~yx8bx8vE zKxh{t{{O5hr*r+T9FJmI5i&(n7A*%7J4jw%63|{qRzcrKR)HizP4+`Lks*C~AfM4YI|Bm@2=DgZw z4k;6_$NqFwpH|h*+|`3j=$i1^_{+uA%AmY|r%ekxZyZEz$LDs1BzQc{wkPzy$qY=M zL~$QT~3Ag@Z-%*t z^+~Xq<##3NT@|l%;g@cZd8eK>3#r>p5i{ekdr`_}TyHgJhLbnEbXHO%Cayt&60e^d zp#oIwY5vEnzVCQi$9yB-jrkVrX98U76%5D(_=|nbY3x)GiEtG6{xc1t+lHn9>PoBzB!hhQg0@tk-!*3cjzP%zZEDlk_KA|lWmjBf;yy~8 z;xmbQ)j#sgf2**KA-Ocr<+Tu=!MSAL+Fqh@Ts2L ze>ly+C!l}#CJ?aTg!W%4BmXJ%{~iMv4xrec1Jvg;xvo{2(+WGwH=Z4#{)MWrkADnS(Fxwra-5nHHT_K6+vS4?^wghDIOHgjM7qj&fpyqFYy zTRQL-wl70E_(Jb68dL*x`I{CP40u@zxMciSr7HD5;}MkNg!W%e|bD_sYs`Uowo6O3U`N>KxMRM7X;rJ-x@g zWMC-mBW4_JpzcCdPD~g0!;zx3QXY2=q{mH2VI;pRrXX*ekQ}<4D&_^%04$6;2f+-# z%=rsHKbL$?1{2@t_?i0&F4$?7$TL}sUoR4u(a3Pq+k8$Qi+bK5qsa|O2fh~o{tjq= zFHe<6s+b4&z!$M~$=7z?NsrCCJmfPhTAo}$lR4osKZwL0H-l1f2@w%u_0oUIP@ixdeRzzUs;TI>ao8dF^*ye_`_=j3{Oe*J?r`h&GNkQ}#?FX<mR zz=7!?hgW2B0rsbV%jOu|3`^M^-mvNjqu;Y`K42M|pYujK+S1JkArr2Q7mO^9$M~3! zIjrez=bX80+_g!nkur2fz5S?Wt1RT}37a%X$>{D$R`dzRvQG{#^$DVO-Qyg2h@9#_ z*4V|hI=y~RSGUh_>h*-B+ z;1IVEQlm6K>4~4;EIATe={@Ayf3DZzLF_cm)j0LPtEa{d{>k|2O9euYCND1zqR%&2U_q8OAOk)ML3m; zpTs=(EPGSMWb|ccwjTDslh)RhY-bunfbz6k)zgRn9P%$6AUhkNd_C_G- zjTWj?w1s^@;;IP*)9JK#u%XluvZ4U0F#ZvA|f18}v>0KEd(n zNsuVC9O~at-bud1pjahGBMV(b0u0@*baA2 zCpIpldU(0rfK0fZQS~2>OP~=R!X&GIEi{7)3`JOX3nrGYSMdAgFMM)o zJ{1Nx7jHd&{QL)7m`kmz+Oez%nLzgP*>CdEO0eI**{@%}0UI5n=VQbf zCYC$rh2_;m$E2p8>c*}dhX*3zi(hm+DAZ;Lsp<+_rxNqT6cRd=$FkhdsnbW|%-eKo z{e_3>2A=-W9QOQp>{>FbYe{tO2Ec>>Wx&8EyG#8|tNmGWGj?rJmOY<}U+XYJEX1|S zIv~W|UCG*-dE%Srzze!>p=M`iTb|5OQ}XgG=ng>+mkNM zX*C@+r2Vk>ztq*Q<|ux_qCY;#1YQhy>KCmpefRDou>{jdp+<>E3>gYiJW<;|`XkS0 zx)ONPoK#!doqBgfTQ|Ww2iFAmi(8a#1aXf!pXD~*@19gyEV34OLVwi3%Ax;w@xg4n z>r}w$038(5303nq(9Wec@yyrGM^ALeIE?m`)p?#svrl_M zD+P#S7jr6B+=9c6|cm`>m`RapSE5?b;2S7~PE69r9KS=og7)YR1Ntt|%~ z8{mF|1w&Fhi@hTZ;i8W1t?&K)J5P>71*iI&$Rt)nN`YG#6_}fzlEA5e0l*^#kUa7` zcQPU)BiV}=7Z!4dhM*rme0W~K1Kh}I(gXZ3L`Fo!ZT!lJnysyEoPKL}w+bMSWcUPp zuS!7HlAQeJUHA11u4EhCrx9^2yhjjAJ`Xkmba_LX?(XhHH*bEyBI@U60V{=FaM33x zC&_x$AW;77eZpMZ%&)AV-d(xTF*$jf*~&o~e4Ui$B~Xz9>0Rjjblw1G5zNKaVCk-) z@L4vTZtZ;#F!IHtMOsj3HelQQ++4Kw`o;!&ch}k7!=ol2uw6aynKGaV8tZCml5Pqh zW#@%9O-|}J1(LB44Cj)*FkBEZ#0qFATN`Zzl4L_^=6?WPOX{~i1$3SWq1wsfPoMtE z$;(rUl#-H)PD}&~3JOxeuY*81)&MJDgzTWy0zt_DW1L)W<{OKPi$`Wt^O{#x-GySzI%6-^znBmx zA1UZGcvaW9t|6C}Ea7qQE5Q1$|7Zcg1AwA8L75pI_YNE#9i0t`C94QK$>$z`F2Bue zwbGAl&NS)kUq`l0)q6520N*=1Fy(`l;<&J%?l9=-1Ew^7CPZW5_qTy{f>J5J2EaAY z{@p(N7ockYuR!&`+t!5VRsVUH|G(`BihO7j?^rL2_Izb%URx83G)i_i^xfoX_xzTF3l}B5Bdp*vHF87gJ#<&~@eWU~kqu=#s5tMQo?Cw3s+own3Q~_7 znwJ0$SUyy?gQvdwiN2A4>r#az9PXdkBrO<9R}edl#_xI7!v5eKfi=GjggeP=6u6(2 z8_Rb4xB65#X;}L@r5o@``LkKXB?}H4wZU*w-%WUUu51a>6qWC-~ z`-2y0t2$hQXVWyS-P|yTlZ|TbiJN`o( zJ#^a*OZMI<2|-MoW0N7~Fw@^;Bm?f*)>+iBt{wY0tE-`TWrz%IEfs$M+VMgD8NL!R z5ZKt}F}kB!t*Gjg1wR{768vz~k!NtvrEx=lt*d@$;`+xUXDROl73PGV$>QhmAJ}A* zX)M|@fiOr^?a%xiug8Msjs z^Yo6}3JqgyJjX1}O70h>g$lkgLmdCS|G<(*^r+63G3I`Nk-kyLF7Jbe?=J#Is@D=H zbRT$o%2C_qj8~NNq<)sDjEN*S!T+QT*k?jKtv0lnICcoqaO}U-{#2X>XAv)I$2+Bp zG2G2J`W0fMR>{L*s}j;~sM{$Hts6P$Rap-Z@~SIg&RcE3JQQp{@cvZ&hLp}G)S{BY z;ItZp< znld^=Mnm!#Uiy(Bnd#`|8I4klg4orssPBExa4T+SPKwU<%ivKHiNSXvQWUA9s2sA2 z*g%_bO71BGjM!ztA~4J-AM>>`tz02rU3-l7lgH27an2B1Uidez$d4~WM8CGx(EJTYPm#)SV&Bu@i>FAbO19NPyWlr&*Q@Z zYmFT}#zBz%z^xJ`mmi+#frnbV)eflTwiiG2#`hy~YFzq;nm4Qz&nb!U6>WbWWb8>B z_N}tfnu;Igh);XJFg*HmPdL^%PDoU9ChpML#ac)+tM*lNDUPA2(?wK6gpyn1{i!f1 zy;G-5lgNb6e)N>LhhHLgCqho0cv8$J$iw4TO!^`DG12HfZ^8q8rOui)SG7mm6{k@r%{{Ti8QlCNI4Nmj zqKhY~w@1y0U7`PTG;t7H7yQNN$5C-{qTD0uf`)lUS)Og|-l3G<2`;YM{bV}j&S^t` z=t6OS9A@qfeTA^dB=1|<5Cy4M<(bYcg3mP}f>27&pfDy1r3PAO$V#a<*@^IE;c%?= zFV~oqS+M5%j@C1SsnkdkiaDx$uo@=-EiVBP!Wk)-`8R8!M&&%CnJRzigDH%C$s*TdZ3 z%M5mcQ6wnXaAyT$cMzShthX2M?x!!}TZ6)IvLvk05G$rJ|D9cAU|ygy6$fvE zvu*s3d4s&jc*X=SRjm+fpPB6wap;YCF*^zq&aa9*?c=SwBLiE2A|TO^u;+bb1Mc4I zZQE>wsHxCrP4g4SppQqxqcg=G;Mdxf+;{}t<(6B`Gr?S)HR3bL^51eVmKbOnlQ=YbEh9v#Z9aTLDxA0ChwHL!-h2FuIaIK}PdGpYkyy^+CmIL;wT!BNm zOvkB$#5!A2-CVOB_G1?rsc1kS^(lL#u#vi*2YBdLF;5Dzx@z{AViS-fn8eZO7$S)_dK%@w0H z=!D$bugunFKGEWIDWfQ}`;w4Yqg4@BsHpH({#8$qTG@$9`p8;qTl;yjx-)mboynj^ zyDM(I+F_{H(NXEP-!(f1o#=!wZpuCZe(oh?s1eb#=TuJTymSXtfN^n3%b-euvP+}TqFuD(V}G1K;bKAHWx13 z#x;4(P1$lNxgJ^4LKC)c$y5{RbVK_2GT-Dor-;?=Rk6hW*0D@Q!?uCXc{V$n+EMgt zY8T$a4ToFj6a?Q3GacdYuU|9zarQe6C&HdgLUGF}45!#)|2a`biBi+ggqo^n7#Qqi z|3v$w?_4I;F8-m2$=RrUz{OGhEj&5@k%X@KSc;y_%OtLqx0T`tx0AJQPbiy_UTOuX z$J1NWcB7JyVS6b}&Y>hSoqFYHG&|&DYTDDgJEM64vkU9#UG}NFMzL~7RTd)q?P*6s z+WoOJu@7w;{Gr~btp1dDc3oaudU&U6QcSDwM`DkD1*&+pAmOy4wQVn~`k3n6uV8Y* zBiCD0wb*?)?1*bj=JfYGmRh zM`l-L!P2QBT8~w=&v1JnsxtfK9J!CSX1f#PfE8DwSyc5%3{%O5i4zV_YgXlKqHr!*snkFBfAfpg^; zKcTx>E02(9krqGk$au88!dogjD?s=}bP}h7CxLGZd>cbux z5|;PDxy0m8ZLmbl{o0Ipf%R}{bH6(tHAk8&B&+TvixY_+*(6Q7vV}+E+c<`4<}(w0 zbXFd+UB!eN2{sz7|3iwe%#aisB$UO#WAWOm-r%UF;X!e^OX;wplJf-&LH2j7`EJAv zkQ}~YjV{t_xrs~3KZ;K8d!u@!=X&jQtFT(l`AxF-rDun8{tk%XrB$*1h?~T{QPNhh_-e$l5sL5n8d1J06~YC$o55&+B$9>@N{Zm;4}GwW%&cv+z7GqFj=N zcTAEfVHpf0X~vO<=tT_K)Pt3N)hh~IZD_lscb3Y>3qNxvxs`z$^|a~!Pr`!hzlN#( zf0a@}{`J3t99Zvo>y*Mg3eOct8KF~9vAerFpBYOKiHdt*6+Z_C&R1Q>KoKk=+%NeP*8@xOo}P3`O@x#|%|t#i z_zxQ8|L3hC`5*H0|2*x45C5;u=>Ojb{R06oA~pbv=^HJx(JvVxZ`)fNp%aIO$`ak^ zA=7@h{jp3y9(gK&q?7fDzTWUy1L>bdY4A9#EBN4%=Npfa^Vp-1K2HsVaBN`x2S!{g zwwTarkL5gX-;H565k&x$IAqG3r<(EV;CE{Whnm%y_4W0b$2uSWJAekB_`2|2yKQ>V z(ZwrxDAmGuuW~^cJpaKrYCyh~Z~eZj&l=6_{6~)!u%cI&;H9;+2)u=Ag!v0^6cR-QK6(6a#j##qUReg6 z3Hr`rrDbJV|JiCY!$-0eSCbb zH^|D#g+wx`&pSS%r(fhOA-Zux_T5b>3kwTCyTn6X-Q4D;rl`cX_xGb3>k~)SzP#FlT*t zW?YgVg@qAeVX3RDrece-M;m`#Xfzg2yni&lec1bLZ)+-nCZ7y!1l zHZ_U$q~BeBSE%~_{rgZF2+%z|JjAU{|GJXwDU^oykV?3n9WJm&OOZZ)6xbRB;h!0u3`r7EG}5fQW%W6Vnf33u!(G*wkqL(@)9T#1N@8F+ZqIbx;Z zH-V(gM3WkWjI%Gs$H!qSmZK6vez1ze<738q_XOefICywUmX_b3Wm|i`y2t^FSN>Tg z!FHY7-aS1%;gOLU(EA+E>+~ArC(v`MFH4@p3opvpFpA^V#fukNj7$4-lsmsy+Nv6T z{Aug7#JR&RZS+R=mdma1y+*&yd4qM{6JYzjI$Ic2+MvJ^dj&yCU?n zOGrxEJd%`@ln0P=#tD#8Xfd~8dm!}U1+%e=*78Ek;O(YjMu0(44u8A^qQ)f}@&N3F z<9YT|=er6aymfQa5_#*RM^Xd?1b)80vMMTHfj}oDB1#8nvwH;uunifGsK6B4o&kTG zo|Xo5oRGw;SJjIP_r(Ia%J(Drw464KI&OJV__O0d4ZZ#qKk#(girGx-Gu+MkRjJ-=99m-?H5T-0{@ z7dAMmrN!=42?+_Lp}h4(gH+|K)B_P=;l=WmeEHrCX(05UvYQR}SzW`$wcDbAxrqM^ zi1Gs5t;EZhT?n##3IFrw&w(wte7wA^Y`I8v1iVy4Oe{yI!5dEMo}pm;yONQJn3(LD zrXbLHarn_6m-NA>w3oV03flc34klz}<=31!7aR48Qc+PckHesbd~768&i(S``r3Gv zIxzlLqm)qW02VdW);_@-aBTVq4@AL)8l$r#)Jn|rprf7LeDr{`U=vIN9HK2;fd9D1 z6+AM_OCHakKM&j=KisH;Pmw6&W7H}_GplELHYf)?TF%wA5~cwZyX>3O($Flz=0e@g zR>Q|7?(*T1vRPWq369xK|N6}OTxZD*j0Md@8X94<;Uc+}{ycP+Je}S8sA`^CF*VI4 z3}Cp!J;Nj;Z(K7DvkpV>m53W?_<$GWb%@VR``p?Fo|oJhL5kAAe(G<8YNr3A|{&bZxj;609$nEaj#^ax@GV z+g%@1A1k*DD=8_-k-Uu2M~*D}Zn!tTw#R8P@@z`?gXi!k#&9#vyUXdZ7S*cpJPt}D zKh3~8tuI?+mVy+AxSt&wn2r3DRxdF}gS)a@?4klYi2^*le79d6oukZRx6&^S3Y^bO zf+6by*u9{%9mr~)M%inV{@iY`^Lm9twx_!T$S{OubWW7>{(9a&U>-XuhR6M+^nNpV zzEnI4UV40dd?d&MKiU^eCDd1BB(EHWje*hxkCv1H5GI_&K(=B|kujP9E|DIt$YIR}ofkb#>_Ozs2ERQw8OTSUpY038^Si0R>d}*RAK4u@-`K4VcB5%8U>Ng* zZlTt1-n_Xp?y_NYxIOpGe)TIUyYZ|c;SS79c-IlC9hd9DTOq!3*S@DmEaqXKZF&T5 z{CJ7-)AJ}4r3@i`!LMNM%Jgyf?07Yewi6dG6EZ~kzY78;0f*H8UAL(Z zSc)--WRP$JBp;$mObqtkA8K1x_!z~~-~;B{V>6jG%Tcg7wKD5;@b)?IpNOja`TO_g zm5i%B{pf4W`XgvLeg9_BluEU=_cbZ$d(p)TA=6`pyL~)a%Uiz` zc5}YmOX>d~@qYQ|#Qq{~M?2k6_hUQTOn``j?(Z+%engCY8Pa!=>)NO}uLf(YsmXwm zEGe za<;6R)pG^Bzw|+qIOANHRrbQNGec}hy3FsAE=qxog1KXXt|UQa5`{4$qrciCs0DTD zo#BMeUD<}=zvRJV<^$fici9})HQY|Nn!t_q&$fiIJ8kQMZ>uQh#Sq*T14tKz2T{Vi zj<_u7AX|=5*6K!LMKIllQ}temG_RQ6pb~d-5@CoG8JM`CRQ1vKS?x>J3{K@VM$^sP zW1=^#h?Izszm%xD>4b+^aH#fgjj#6q_JvvK`D5tgm>>O7y zc69VST|31?t7R*ZeSWKO<+_&rYmSDD&I#3dN38;njUJXgzdl)%3|(K;kY4E0XnWzt z-Dg&1W^?}=Iz*0$%YG$TYiY{K-o^cXaXo|Tz-Z3BCKDATJO806+eD{(QGahavn2GI z?PAyhTam617S^XF?}Qn4leTD^200wN>5=`N5|=BB8N|4Ri*r;HN78*;dslWRSc0+^ zE18I!1XBCg*Ny*{IL04M*-0-Fq`sk~MAw`tNgr*Js51Vr&>xIAX}M|k2mf^qk4y7$ ziqLW1*=Qei8?Fa>KE*%LWf4uW4M>Yiq(0YPp2kATkBGu?Ysv|mM`r6?YfrpyMlzxE z)rR?l{QP=6&i+z@KQsp05%bJ0+r0Yl9oWD&bUG$R_7l)<%Vb)~NEBEtu*dx}0;g#( z8wP@hX0-nE&@b2XqWfg-pA{=itO;brrBKghP;~v2(H@xP$y}Vtm2e`6Kjc*^89SB9 zJ9#J`&As&On^ARwt4_qLyP-G(i(6+NJpuv;uSfQ187sOnPM(VG_Ac1$3<&IWoig?? z#|SmJuet@CHV{^wkLl%oZ3%?|fT@AM&?eT!oshj~V zQ$7J1S#Fe;vp6<3Hbir>QOue}Z|W{6M`2;0GQeckij2D-n<3+U!X8K?6NUOJootk& z6dL~=jiIQ92{~U(ob40imI}N43|DXXsdlh@)$J~)2ID$jSVfCaO(K`n#P5Kv607?~ zTT6ag;tld1jvGa*-c$`opMPK0N>();$wr%m>VG=Z>gA+a{({ap*%3B7KVfRzt9Zqq zF2mlm*f~Mm`uO^tGJm!{$!$_D?K|F9_R(^=x&^zD$LU(BG;$l#+BqwUXDR~m%}UAa z@;;nZIX)Ga;*UM|6!fA(jz*4G!h^#fnpl0uDU>gAY1c+KJ{50UaI@zqIk?_)f>%L$ojkSLorsG>F{N@>LfOr=ZBzDdf4koIHKmjM@+U325F-_v zk3Ex)VH4jUcPSWm+t*Z%x`cu9MvfSZ!(u$QC5&#xi$tsI%blls9Di$OjGM^Pj_Tf( zu`gziR5Z~y_fl^J-yY+deBQ3{R9=X2M&f28-K_nBD^nteFnP+!+|(V^1HSd%oL+DG zL7JAouTKBAQ(3DYvjnW#$A75enr-D^H)*!ZS(9LZD*U^+sFS6J>g`YEB4>f6O9d1wsZOW z3_%-Mm6d$7cNm&kP~%YILJb^|3VH;SfyKKPLu1MS9+|`Z2GN)dD=Bv5OC%* zU%`+K08-vF42mt6Hf=3hYRH<5-Rub z&gX_XC5uQ!V)O-~a~Z3O95?!7)pvEXwfRc{6nVHmowHSmN;(OM7)+@L91U#zj z=?rpEzcS*|D2uHR_az|5h>G>FigGDA!og;3`MIXcG@9^WrFa(W|B8snpX1r{Ta9&* zv@=~`I&P^tn%f}?C6@oTe*38`mD9>@cfHqCO(0#MY}*(n2&jji27(1}lg6vh34${r-ndH3gQ za&M+gMLAjK_p_b&qhHK&L4&l*TT#+SDS|)Nf(C+{I+%|Csxf7MNyb;6G;EIf-@xgY z{%-5yDnrL*IW zs91Ho)w>rlEF;SIv#w&GYi&CD=o}mza4C3IATk7bjkNDML=`^9VE}5uIzsIgjbD&EnXnsm3rv&UkaOZhL1(?*e=yhlrH7RoN2M z)+-u8b?fp(O%2SED|*|AGiKRqm_Y9;D08g2PQ9~p;}hLjSIO>nBcCxmzJ=3w7?(A?SC2_al9Ex=la z<;W_aFLTPk6024dQIR(aNCx<^&9s#ZK>1-!cAPX`- zm=p^odg?z-y8$6C2JU2N%|3uqKr4uE?<+iTez^xox{H84C@5#jB=x`reU2rO{4 zQZVpesUpk;)FH!&`)yug0!Tk}WJD8E00EW-iH_Hdy=(EYFU zxO9LR1CoNEBSTgb|I+nqwJngzBWdYFe*ST*#IUfLtCUE_59n!@VvZ8PdH;Zd3PcFZ z6iHt}HpF=ZWp_lBlm$>bj$oMxl7pauhCOMY0lJVChrDX_D!If*pFe-3R38x+gFXOK z(J#$8mX^Thk_`&;3C7$Q06OPc{_9<#(fQe_Ur>-TKt9#S>pTc9127nb$X&Yw+678^ zYFPkT1Rk&CR|I4jcAt?*K!v-JTT}e=R5|810$j^mpF4KQ}st&4^k?Mh0-GK6g0H zXRZg|Zr-}J+s0C3xG`RZ;A((aL;!*<4)_`UE$W{Di$SALa6v(Vf6f(*$xsBukFuZt za|gsS{P4FOEHE&*-joA@+yJpzqO|~AntA+U(tI<(ssPDFkJNp<67lsb0Q58<@8Yi= z9ZO*ivWL%}#RTI+ zO(#;k$shy3Q7VrU;Fd^A3@buJ5O67ZW8>`mxt0zN%6fWf2;6e@QA%nm5tPqD0_^4O z&A`g~8enMVamLAQ1mX`3O@UY;JtKpGoqac@mWrEO6>L;!#~wWb(1PTX*Zc*1Fz{Ql zc-RA|A+_An0swOkIlKTl^m|LA&TPxVa3XBa9BT#s(3BtI&y&Vc33KpjifI1-|4S4(ufjQx^v5Wwi z0}KRDpPQf0PD&C~%29+k0HE*I*4Edc?a`i709>)gwzqb4Fcq*NR{$C>Pi)*~Et^j<+_j>*(mL+}ua;&+HHni4G*^2?z2;-MlPaN{9<^ zRCo=;!-t3M15n-#)Qds%=8I4OQX$&BFOHBuQBhGeEx`FaF%UE=rHWtMvYPsoxKQB2 z4XaXi<-LD@#xIBjI1~Yanm?=%xx%}|4Sd*VXJ=nN<1u&%ToX+T%%_Dtg@z|47Sz|j z5NEi>tYJJ7H4UZU2)A-@!1aL|4hq0z@w2nD<*ww92vz|l;`CsIcbCyNm&DTdp*lRO zrsmwD8oI|UaUdPRkXW0j(f&K%-aBGZ13XtYNYc`0x&+w@`!(h=ccawGqN(_>u={Yd zU=+m~cv)Cv0L%^42VD$p<>-=hx%`shmNg8G%ihsFdYxQnfrY)`b1~|#`g`vdz1LOF zyUc1@q}Jhk^dauJxHJ(+}Xz)HQM;Ro1Rce zuc=9q{{rNdm_;kMw3KUYs89^%;3}J%=6YYhQ-1^MyR$R(H73b1Wa%LOf~xgb5F(&G zF)!*Bt09j3bi0iO&>n{8!nikY+(-gOgMo_bxwhMG1TZ0N*0V2x9#DZe0a2?Q$xAT$ zED7+)q*p`LZWRa^!lY3u3wY-Ks7(-s%j%P2)1e-8dh$)A#nNW}Pj|&ebq*N!F}ndx z=hg2`I~5bB9SER=f$*)%AD~|V+?X2j$>CxXbc)C|zMU?i^7FG}D8XM@-j?-3+&e6@ zYP*$jxXdOKamlbogg^#iv$t~s8VmV0HqT*3V6BiA1HhUQDvM{mROtKn@5>`41yT>f za@szIH8wRNXXsDDq9iLD{6Ok!5|V=lkg{9b!CLy*|2@S=lrH~MFtE*0x)7rZuDvN% z+pW7oiGps3WuO)IR)Fz|%8y1IxnW2rs>LRSlk?qJ$)ySlT}V|I&i-m25R5!fv$hw0Y@$7$?h zlYxKtDl3olILPvWzeI9rD=Vv`on9F<1TteD=cmq-@2?P&k*&hTUO~dq?i3Lu9JSvV z&s!g>VC3LX67s$t()iZu-@VdYhw;w}EN}{de9KEoeYt%7_C3&BP$m{KN5b`iNBsQA z>8VvZeuuP(O{e}6-^peJV6)$IB#~zzE|078FrWxRk$;gm%o|7C|8kF~L=@15hkqAL zKw5!I?I(8_O-7y!kotqL7W5o6J4+xf7AQ6F8kDY^?~xR!Bc8_u(y$mlm!E-T+$jL< z-V#pghnS!Xwt&b@qd!R-*kzCNGo%b>dU`rMDrx~aTgyAnV5YK#Av76xS|G1FUax3v zZ%1o7jHNeGdT0P)W@BgM*>(bQ;JlJoQe-*x0wT^a$Y2qR(Dy@MQaK7-(bIo|;{?vI zzd*YHq}j01j|g1rz(suCBoIKL4}Zbh(V{?jk9(ZEwa0PGBk@lnzZ;v?v@kCDGlhR~ zJU)hJ_HWRN+Tym3j{Z1q8=wut2u;$XHZvc_XGetR*evzL@d$Y>RPkMgUxh zaU3vghU=qcF;5Kwad2?hOa~wHZT-4sJdhv%@4kNg+Z{jD5g;}k8l}@AV9tWlmVapv ztp8uy!<+QGt)&ID!hdJjq~IYKVsOJf;Sc2zLtIu-f$%OE3`m|extx}szErh8-U}pS z7I4Hlh;?Aq_gk<0+aXaZm7IJ+skAb}$w9v~d537|dUWh)Iu#|ZvSFEbXVhZfNT*?A zCCyML(-W_q-uRr~IhbZ#eS#D2$9c%oAV)$<1t1vg1m-FB3O=OKjhx$B z7<3gWGeUyosRkbtU>Ne5JIio5R{Yg}FUHeVD<`T0%a#frbDc|i`_y< zdJ-gXhKhb_u>9-uErbjMr=pxyU@-Dsh|anlhvJE%ZjEY^&rjyQU$k>;L{#GDI8K)3 z56m@DJ{&bE6U8M8#_^sxZ17DFh(Ckim6%xrg&ZBoG-BL!1EGA(_$IvAQwWq(%XLen z@z*cI+2#=Ow{#>q1WFs+c-{wPc7^bW)SFHWvl+zOySbkdhy|JEmX@+0sxSf( zLRf6X4Iriy{y9UBoh{_&%2BW_MIAUuPh^rl2}vgC6(bjy3an!wHHbeO5(lLrlVK0l zY<-U-;XAZZ?v<-?1RQf~qq%%J94mwCH2Ud!ZRAr`Lswvj5rZI!9b4m{Zyb zt4?GS+e$pm-ATOz9c^+}dk^D>E45-NJ)Ub@H0KZ62k?@+EeRYgSLI3YMbgIRsr>BC zk~m*ybUR+ax24~&v|Ceswmp4K?X*+Ij(#c)2gj7*VY5=dOl;BamWT0-_oI;Y2BP|g z26k}oUHqTzJwh`Z_0T{#Us_%{F>jSxE>&!_aH6Po zyeFrh*r9B-RWKnG)2WwTD|^^*wXmS_!XH*APWe=Oo$aER+gIgZ4)EhFIiL9{(Yk%t zJe>bszNCE%{nKdaPt5ylkCm}yE61qFn&ZMia%MT%0)}&{txijW}-z{vu-4na(wtT7t8hdPz40ZJH=6&(>U49M(F|OlwwevHXII&b|PK zx$i9`NQl=)qBi7PK3w+K5w`=;IJJNJyz8Ywe7F4b*n^Nz8|Geu^0c(KnrUNi`_{KG zC7g5u`0tb{<39V1au2(Tu1NMuvG@|6shfLXq))*TE)i%v@)djdf%t+}_v4P!rrFF& zdHc`5uV#lp=GgfAH{y=eYTT>9HM70_brnq4;`=KUU8nmK1xLFp73F`jr-<`c%uVdq zJXW{Qu6Q!Y*__>ki5tbBWlKmiE>ir-5qmD+vQJKojyS_SamjovbW3hkMR!2B@#{U`#2( zeAlso{|V#uJf8g4u@lAF4Hk!S?)}RBIH!ei8-hD`J|7%7+O7?wyOULuFpR85LK4;j zMRn<@A5Y62un69QzXawOwWJrEwx%_Fi$6|&(hBco;c#Z zuEj-sW=AXd^_~>QC&9TDev*2HAyOA%ybM|T8)Rf(AVz{f;4-6f{x5(gA4B;1$;W2~ z9F|Cl`4|~_hTX&E`rZP+xo#Tlo9iTm61PNeOS0s)ozPMLz$U)$>uZc1*76CAzivw? z?O=%oB^Y=Ak_q>_Pw|sjmziU`4(c~r#y4GVV*H+LNh2COSbpVNvuniWZ+=~T$*26x_@#1%Zfz7xa#fJO zK(Z%=F?h#@lf%DT^RX6hfzmlxObvoDH4acva| z@WRfBUR+i|(Z97VxC}nc88nK{d0a#my{hRuR|GOy=#@MAAFldXzA#5p*qPoAiyg&_U?mMalSzKV*_e$ zZ)>QO;dSntDQj@fuc5X>!OT>CZX%AoVc;`O(0~`0x48jj9U7DerpGu@XQ#wUr*|=E z9KR%dL~(2SXK$ro3<*$*wtjDT$$Y-B9+WgDcz&$tf<2tslsfUvf3U=iagTEBs5VF3 zy>9W3!EuN`1Qjo2Wp6+c=r`9xYuD`-I+apODqsgbLrpU^7uR4n)rIVS!|@zE_rv8& zcrvb-KA0Fdj$Y@Kziax%&p1toI}8VEm^;jPK2v6N*5hN`dpPyua?aT zmp$YV`rAzM*wsFWAu%a=nb_N498dJA2MawBk?A7(+H?PNPtRn{sfCS;#cJ3Q7Wdg! z1qe?C+{pQcBh|jVn)2_r=x9JWh^CkcK6P}?O73#5tB`gnqPQDi5cw(j^DV8RS2SV} zf;|B}Ov}i43qt-3HmemixehD8O9_m1( z=eX5K(hW5p$j9)bfe6t7cSy2Ph|Sikup6trRn=UT?-vq1&bXl#8hDWR%34lX0P{H58VU;&0VYBi z$*k!IVf%|Lg||??^OfC%_?g4{>SC9#Z*$oe2QD?W7|;ZRH68+yEZTtp6C_*B{61)) zJo5zVs}>ld?kNI-XxgVwZvj&98`fa8H7#XDIC6A*CU(Yo_oSWhgS#Yl;s(D%!iUyy-y!}m%dhp(sD zSdu^_aFdOi`T~5F9z(g^%KX3GZT(wN-kktZfP^keMn0;mc02BZlus&}6$^w+2=Y&I zateYFX?=Zt9heeva4-{uSeIm2%~x)l2LL1|E9D>`A>h93ParJ()B5zt4&oae$kZMJ z-FA4Jm)dzzai??|Ijf0@iSA?qf)Kaww_&knkW{KvyK)1zlH6^W)Vs6^a9S#mGJ$vj zzReGja{*g(0I6?Cbo6J)8Q(7VWnIO?16w03tVj6>)Km3i-J4n{>HG$W4Gk?V)xCQd z$g&j_2zhyV$&i)HPnU)HE+#jEjLJ!Gj_^@auS_cH3k%~DA-?HR-Ejh_`V8P_Dpk(x z10z(_*<(96Vq${AvkZXvBgTbyZ%}t}q=cm-Q6LDUWE#F5?5ltW|25dHQWr|@J{NrWz+hkB+<%rtaR$IOmG5t{_|H#l^b8E{BbQOR zjf0>h$YoIAWWI{8hen+5)^if{gg8*R{T76Lu_xU>RR})Zwh5A$;s@2zfO}Gc@%asX zCf((BCQ|Xwh&(_XWdWNP^AQu?k!}6+M_5CH3^Fud5W!?fZV{OVawYk77IG*A(Lso+ zMMT1}0zH@t#k03r^{xV9euIR>2kLQ&08`YV0)zkF)z$Segz#x;X>S1FfU4<-aOK^} zLRTKCe8b?h7`L_cg}Zow2;U{G3sSmxd7?Pjg4 zRAawJO8iL0vp-K=w+|(U;Rz)gli={aJ~hBY?t>X%h--+}L9Xiy)MyGQ>sMBN9Ec&` zLwWN>D9rz{*OY00gJ6ib;LAPgDB4f8fYk*836rZ{l0Uo_78dp$t{m)-U=I34X73bK z9Es*A<(X1GA|M#jbas9Q!sZR-dxQY2A$OheD&B8$a?<$RzWx(75QNxghn*fTLTPw} zCE+5b;1p(Kz*lVo>#Pf=S~g1othGq0XaL1v$D7A*fXPMt7Jx<{0GyYK<-mnPeVDXo zDL4zqYSE}$2XIDkc=#s@KIcgpk0beX-8w)ECpR{N0Na=P`WO=yH=RczEZpBX!sgq& z3rxU08k$RYIZQDyUWR^x+l?A}0L}`M>W?)w0su=Re+$IsJ#Owha8mkk%ZpSg-G(

ClhR94d&9H~&2CoCLZM&o|B&wKp9> zAUyjTir*jGF3Qr-(WSyDo7sJLclVc)k_`?MZa#qL-Fxzc0Dkk8%ZeJ31Wf2Bxgshp zEZi{q+8^WI>2XErAT&*_faNsFwOaMV^p}fS~LH92m`m2h(9P{B@iY zt{;&T1t^4GKtw4A4PYwP-o``_L`q!rYSbxe9~-YyTjSEnB>YNx{?QVu?yukwU?eFp zAfj(N_ze@`?k`e<+fv@22@;@YWxWa1y?3{v>*P7wm>sm{o5O~d?Mi<;Cc#YtG^xhZ z5592|$Z-JsJ_)9sh?p9q){t!)@-=<)v3u0ip+Z;RhtbI>srX{}5zAEDEa1Vn{rN-k zG{pUkI%U`kS&uQ)!V5tPZ3UdI@ikoIVp7oDVtGAbVTBRboo+ESKRX=%4alKxb6MU4 zCU}+b4uRZS19DtO!13dFoXk*Cee!J4V_mUeBgF+~Z)#=5WMJNaVSBQ+@I)U70mYS0vv)%FO4+(L=TD=Ta zc%4SCiF$ndI$~?Q@hQzaEkEly0dixy_bCc8mDtrN)KA#J$VD&T?B8TmmpArybZI5#y$W=;5_)Nu~CtWUi~ADIL8nRxbTp4 zhEl4TI>!6igI|4{=dWMB2qc_riMggbDn+e@;&eE&5s%tA_{If?_0jh}K3CXC15z8P zsI5sAHCJVgo`7K^fbG*+3xnv=h)=5l9m%?7L@N9rKYGfDYx(yRqKSB_|M;;-e8_+N z3=tu|?%&V<+aHd=^&@zYwsCs5-lbSjGvFqU_=obp8e~jGXN;;oIgN}Tbr@~!h`O#H z5h_au;{8@7njRd{Opc^^x$hC)u2p;HD%Lb7q4C6LeX914eOFOJ;Fv>Rk>tF3fbU4o z6Dh_zKKNE#BD2Y^z4oTh4i_iK>B&}ly{fcFJ2aRQHJQe<``+;Wc;|lGqwYAU;`!1( zul=CTqg;;txk;zN>v8*iaf}HLTpLy#ar=bdou9l&Ynqms+L-iDRSu4}_)sV_qQRu) z(>Ct=Wc}u-*ssaJR9!2g;>vbd4~{(l@4JcT-bn-V_euj(Q>lEO@NkZGt--`(HjWvV z4UYiRW8sAx`H`LZ#4Ke=YW&_K!b}!Vm3cQT$AvKBtjBeC-wN;5tqmq-)utBOQ?WN) zrMiUksye7ivEQ9`#j@B@zx&g@`9N%MHcJcAKs0vaACyu*4>s$BX61=IdzV| z{>l*i{CJ0JBkN%l53FS-h9cA{C}JX}c1*n_7@uY5b{}Q?Xnd7Hxmu-g$M0S1muV~C zZF_Oq$cVhC_yYsIHGJ+mOZy6ERu+6WRmAxch}ZPs-%)FUOUri{pQ`kEq?_mzq{xbT zd?;-zCmBqG6;t(%@^BjOt*D{ox`@rsW7KSuNo}MdC4Eud!6PP;g_0Aj)Qd3-`H#_62fsJgDy7HNDZ3vs+Sl z#vh$2C{PpT81Hgp9cei?M-=srC#9Gg_FA-drGs-ebS z+q>N&peU{V&$_{U4X2aY<$DWkH^(XblkfuM#8TfogzYV|yI$rWcgXs4YWl;bIj%Ce z7ms)$7~4pFdQ&=(<%7?ay3_Yj{GXI5)a@6J-VIQNzY9@KOZaI z{6*Q`;~T+D@%i*b=f2qS<%CVt2##AuZteLGhVm}OcgHck9!#z6)YzH!!{y_jUiM%=I)DU)AR_B1Xg2UCn@&W@>{?E0)8SB%h5R}t11 zCKPmyPbj^1Ao!SI&Jf<{k-Ge7Tq<(Na9?+fBP$_TqUz^Jt&DK?_aFAMpI?>VnN}4L z8FCz>AhIB5Cp0tUr8`46<@wDd5w;tU-8gLI;mGiBm*>IR_ZxmbrY0rzl0?7ZNr78? zpz>9!?Hy_jb*swI>@we1r4OX$?#`YaE3m~g@7-xV&y=zrkrL%;cCz>}{Kr<^GlZ!`{L0^X&2rD6GAiQO+HzdJNV3C4k6f1u zE*2;9FNOL{WoGu#i%P}(FS5HQJg;#seR@3I+H`U9!QqP%cKjDtGb@8yryTs{fuykme*o=zjOx~n?@3GG3KjL(Gw954o;{!Wyjujr) z&3U!7NQIx5t_Ro-y7S$Q8VC^4)y1hUBX~bHP#99~PDUErG37J+dZPcU^hcc>^E#SK z!sEok$1l;Q7QcoLSaf2FFB-63e>{4Iwbff*n>R_rVa}_hKr3HV$LMoiWG$>1Gqu>0 zLl2jAy(H>}SLsQ2uPi{e4nI5U5|!e-`;0CSZ=7Kb*x<+pM+Dx&w&|`}r_GH&#SFc^ zJuI5?_j=$m`Cl4*sU*Bdp%Nunqy*$hsq-DV9G)kx%3H!JV#TX5@;NhUjt3h=)mNtfi0-vsp63%jercMDf2g zZZ%mNkT%SHt8>&&cs#w_AAOZ#4?hn3;I94p5rz;cNl>%(Zfr`TCWTlG2jfvKzWzg=qHVn-})ZhMF*d5zruRZS9u?NzV* z$*ffM=TY2o@z){IhcumXZ#BK?50%o-X36XCeR_QQQ|fv@y>NrTI>XUETBh+{fZC4Z z2A!NrSV>fg2y&~gLie!OYJ(c*UY}b?Xg|1YLX(X9nc&XcIlalp+j(AH-nmYj=bvu8 z(DOUEN=cRF;_Kk?NL!bYP@sn7USBrz97D-`QNgIA^t8D|EAMWk(c|e)gB; zrU??dHqLzZ8Y=p{VGhQ3TK}QExs&QoDyk$cogTpN>NkN zGE=&W3jGwrG}QD+_ww*S=$VPV`5CjWv~O3v^p3i#oQm^T%!P8+X$R2Lch{qMC{%4K zS;|=7R;m=e{a~0WceTbK(l?RvrPQ4nH)(5aJmfmSZ~mp|aA8*q*afSTf=tN=gP! zJzVOAZa$tRH9g70!JzM|G>~CqM&F{D_-&YmDg27qdiL_0$`8E;HurF3CJXpRA%#wQ z%>a;2k}TpNb?F~x%osakwIwMGRc}u-y;SuLPO|!1VlUuc9{N7+gLRM=!z0gIV;3}) z*w^lzS4BI0QLNhBu2)@*enu(d;6^$nuifNwO+f7AeQ)zOM>J`flbl5HY261h*4cXl z^!JR4yA+g6m5M|UKHW34Fm3c6ix#PAJEyoiEpw5)N$2F&tDoDYtB!ebf33gvSaBEM z*jx?N(roRYta9f6%*=kcH}>Fk|GAn2F2?)OI1Qe-#FlL1x!cS>m7^qgtQ_rPD55^q z#>)##+P*6*u%7#}|MQRA5X5)S;`T>_;D8Ik1g1s)af$nRd+Yec6U9?{S&wb|!L_%l zBDW4Wvr<~tDXDB&R6+^&_tRwsF$G`0Y&^%r_UzURLcgIYH5zSgeDplj^1O#Ux&ml{ zfD1P+h^BsDY|{JCqG0<_Hsr#rvaPg&-I|GgrPG0Pix1aV)hks7jnkXaE6uGe&O4#a z6G0@W@jpjMvVM?1aS~IJhrwPpRwiID5Jz%;G3VS%zU}_b)V~SsHNp44n0u>`DOfr z5){736bqMi(Wpe3Qwx_hFQui4_k2Kv{nb8iNTX=5Q@XC|inwW@FRW2o`P946ytRmt zSWjpCt6T1Gm&%5JW)`u=__4VbzTbhW(q|^%P|h*i?4o0~f4TEni-qN)n*J?e+#++& z=Jppq*%yM*#i~w<#UC3t+Fe6khF*pC<%K_6G>6cLA_887tjAdgDalsBp3!l-FI~^{wGl*8-wTX7*-pGpV?MQdmrq9Tx zyk(w8>W%Yi18>s1De=yy%gut*s}EIdCwW=_UJG{Hb;(WGqUNu8^Px$;GsS~ktVva< z^=R{NgsdeSrb&>d*!V>{4$AAF$u8HdntdhT@=$TvRMzNZ7Svfc(~#i)(k?ksTM9`` z5|grjNM(1@d*bHdpa@C)>Fuf)xBr???Tj~mWizLGrKmI@e~@f8;vCHCpfj{RQ)-si zB3Eqo;A8qmJDEdG?X`<3n;kKyFU2eTU3D?n)6YDAv4mPS=)JI43dD*s;8xW_@lZ`Z zarl~l`axbXZQd#eMRR%o$aAyCF>?ZAG)-z?Oa?+>+q>HA7zt?hrf4TK{%s0h4_aw^7n&M z?@vxy>b0HD9n9Y!Y?|Vl)4bmQ_4MLh49=f3=}mX*G*!Iuh{X}_=%};#_g0shL{q==QL;WcyTZErBG)WC+0rhZ zFk8;*J;N?#PRzA%&Tno?7|S`dFGFuiXKv&T{VmHed+Jij`P7$JT@SwpRLV`QN-GaL`Ec0y=Eu{Y_N(I|S)^H9J_L2=v2@9ponPZ+ z3g`99{g;-lHiY*Ax!Vkg;s*Ake7k9!1kX7e(Fob`*ODej+*#*ZLX?#98O)TWS0K@6 z%c|*XKU@_Juj$in(_$ zsz!YLll|>d3JXZWEd)T}-=5#3{(WeMfJvFfF`GrL{

>UETfORu|M?F5$kQ3mPal z_$fgS8AJiITTYSWILc@gCfznoRv=%tU)-bd za`uTy)Z`p06)w)Fv!=SZ5IK6idfY-Aw7LKOfYfF@7&!YB^D73f&gjmuf${zi59@fTNt2xqkAkSk=@q#<}Tw`7ULq{2h)@_1RoT z@#tgks962p4CI_p8OGz4QwlBEh{wMc^kWuXTJR<6 z%=q9q1V{crby;aDwq2OuI*1HEA*SMq`!Y{XvcccE%X|Dz#*7^f=EvdHds*; z{;=eeckSZ(T(X*@ZyRjHG4XjjJ$7BPZV46H4R;N(-tDXWQmfm@5f%P(vOe>{grR(O zl~Km}lM9314+^j?1q8FUpra{kprEZvf$!`bUxwiA^t%=0F>SgSJw@j?bUy^=0-=97 zkaov(dvvZ#Fko-Pbe`-eytvy!+6Irnr5-$~*0INM$#qVqDU*t<^Af=j?0MZRH;SXc z-r8k9SPJP$z-xZzx!=1gyfr5cA9{{Vf%bz6#Bc980!r!{=e zPJvfpvk3`PYp@=P1^Y2G)W5<>TEay_e(7zTmoDQSie!J8jR?0~=YxH~fLq^&+0nG$ z-8w(2H37!3a38^w*ktcdfmtiBC)34hS_p;r^$3X&!KNFCOH%G#&Vx_PZ3&;-(9(J# zyvdKl;nC55x7r8`R41G)mcJ(BT?BjH|1(=-|DQMA3VQCzLG@3n4G@fW!LBmEb275p zU)$n~hY{M?KNDeO07l?|>OTZfv^|Im#^M5Q$?VPmP+BrgfeU)0q8Mntp8#4IhVmV3 zQRXTAKojEP0rH|W@L_}LP8=NA1e{hPYHHY^Fd<`f>n*6o0=firnDiBTWE3s2@Tn_5 zATz^(V+i@%w`faCOSpobe4rw0aT6^tz>JO`>?f+2udc0q0xAno*E_)$amHW#;zf#_ zKajrK`uqF;ei(*?q~v=G3wlsu69I%;pfQmHr3-L@B4CpmCd3W5e+7HR1u1~1{0`)i zccqgng{944smlpm&wNW-d@Es=Df&FVSAV)BRE0;-Rdk*AO zdwXY~lMw)&Ebm+zs1`y(kHKw#3Yr&S2S0?EAD<@$1tnEj$R|8xroLyb@I|9(3p`t=TRZoVJYP}~fP+(9&JDnZ^E>}!O<^?c~2>>KU7SJLC zfT>+3Q1uvdD%igR>#SQpW@=e1Qc-U<^l#rMilT4U0s1CP*hC;aGcc+w7V17t*ayeS`By&Jpwwq_GJhV z*Ff1$`s7YIsOC=wt4(b8^)Y^~(Q4|dq z_Zf&vpwKzHx$(kUE?0eAlYfx!|Z%rg9NrgKA|vWux8Ig1h9{zcKV+&d&puymZ=m}`et1vzs*ZxN%bQX%Gc)^*!3}V67+p-x#)b(S44^Uy*76l~%T%9WG(np$GNpVQ zU>5)r1GE6pC(w_uuhb2qS_OB;<#o?TwF-uy>iN@5K%8a(K%yFyfdMK6`~#yu1Jn}YvKqetfiRpaPcokgsEBM>(5*^{WuXH%sCTen zwVd_=3e*c?K3v%72R{z#eu#i{ub3S)**y$yw9bwSyu=LXGjOAtudDw22Mb^TY^6Y~ z!M->k83Wd^1DU5QdtCWcWG1S&r?0_?^YM9#8`|}Q%~~&B<*2wQqM~iNRDs86bfSmfTU>$vV=-70E5Wiyg>l6%_U&*hwAK1Kp@E^ zbD@JGK1Wy{S*`y71Xo*1P*esRU!Zc~vh{0#1xx@72oHw>B;gAnHSD*?o&$*`{_Us7 z$mw8${`>89nafnRu&V@=MECFC?}7RSQY;!C-g8jL1RLUvtN$Df|DK*eJ3m80vW0!S z$_;D;u5Jh6LESg_`C z$vM#>k2NV&75h{$;t3^}k=b)Rwo-TyD)33@>YAl)z_ zekyR-v;!0lqhVlt2i(l+`nvwFua7`76EiiXw%!3Vk2ow!H$O%E`0*HYGbltz7W5(p zG`EbT{VZ)k&0?4z+j z)(Z}M04zd6LV{X@q8ijxz)}-VASLMg=depR^iVI+ZUt^VWybyZgo2)gz-mGqMCTj% z)DRFDu;6Q;(N6l$7wUO~hd0tNpU{7{FB_ zAP2#+!(VdHS%d>reg*?VBzCsz=YO{9LFY#Ldo3iL^3R5L&S1O^Obf06t!`+Tyvd5b4fAh!{BvA=g(B-az*!LI5nIR|tM@{R$v3 z;nGQRFeQUL4R8_IpaM0ji%Uz2djepl^;gUi`vg)P06l&ZW#;&Ig4Gs`y9L=6;A>^m zd(t@5|6GKe4A+2QQXTE3yTCw4*YFn0f;9(hkro^42Pljua4--PG$VjYK7o}QsD3T7 zCjpBfQAbA(5Clj24F}&K4h1x%PuB<2!T}IQ3V1RQs-?C!pscqqi4zrE2n>iT2Vj9r zng9`u2Lryc%)rzOn+z;UX@MLLfVIy6i+Eo;seTjf4L~|T;*s+562PS^P`@?oMuUYv z%-a$vP5T2_tAct-ACPtcQUv!kE)K_PzVUgem~zD&xZcS4_yB;5t^h&;#zp|j)fZF_ zlD&TYOK}O@U>*K^4$#F1wgB|>9-v0BwJo6e(g9HdF6ABIuR!sGH+agTE(|apKyw&? zH3qObjCw*&rEa%B%MH$NA_TXFlAEKs zs3cloBvJfbQmoGUpmEu*K7)NBfC^?CT`)kkS{n#TlG;$=Y^B!z{+FQDHh5kc#)6d; zEG>X~0c!9#@uH3So0_tv<3Q#L92!yph!sFZ&@iIA5$mqWg zM3MgXW>GLKN^NOCwqZ8s1pqZzXs0_J%!7qWV`vtcx(#T;)9rEOB;X2A@k~+X2St%i z!vsnsm^}HlVN@S_T?iny(*ej?LA;W$K)txO28sfnK1D>FdCOz#*tsJNP$3o9Bze(j zDM|o>41+=?0x>{a0ct4`y#%&I2Wub)dt4CK;6*f0N(z#$lRd7A+ooz**w@v-;9xuO zx047;6rjjJs8v=o(9`?0eso^E0CmM*?CdHXO;E;)6CzO|7U-=B14GKt$}Xr$J4F*@QNwqKWPWu@iL%Vm4vMaA--ZX!@Cxx zU2SulHXZMjrXJe;o>Dkx`QkOX-U{5oOJ67{j=@f347_E`H311)F3I(x7;554s{t6)U$kqex&d7 z;fU`IYor7cg(XR^=8T@|nW&B0bkEL4dqa!G3i@UJtE~m(8VTSo#Fjp!F&5P_MBc&H zq!W=%=mxWtwc!D+V5d~5y47HJgReJtp$CMnmheyHqFP6i0me&RebW z9{Kwbf||h2i=$dN*=PFu@gM(OJWk|Z!)wR8!TYTqY&VK$D`BV)MU?(hBu8y51}U&T z92BS}x?#m~wx3}9xsFTf^RD&x#ZhgK)i4jD376Ht*S(K^_#wh9w#wMVoHMpxZlunH z-U?45#i>0!DOWFOW)mK$2J5d;kf;z65rj945xfc>iydpK zShFis5#?{5PAgs5O2y2_S^fGiSqL#vFH#n{k8@fWhxY1Tm#(#0a-(6-UU>}P%2^3^W?Z% z&d2)qLjKV5pk7w`x58@jUmt5vA0zlV zn!z!J*P9O8vCYrgpSL=8bBFrWK*{oi4aSB|7w3NRyS_~G4!_WGUDQ?kKTJgn^rodq zim~dyfI)f=pnb~*7ARD|W%2GZ0=S|5eS|-*<~{YCDDgP^2S_nSsymjF2=$(#qJDFO z*TURW|3*PGwCj7D7Ptb6l&nqSPb~q7BO-l-OkHzR&(j z%+`VD-a-T+gCn*!p1pc+dD6Mvci2R3Jfq51XJ*iBmz!GTuQnXG3x3PClQtd)AQhaK zu5>?PGH$!_eLqCIV~{Mca$hGk-~vT3>@uIa>bve6lug((ANZs5omp-CjQfQp`C+91 zv7nlrXKW=yq|S5O=YaGOI$vVBrN>4&#FAR@)>jP5*mUaZ)z|r;oMC_P#`u9#IlhR* ztTSYzqvg8xx#;MYep<#4OQeASqd|5wr00*RZ^&IPu)Xd`Z!Rn=Td=GzPBE3X_9(JV znT?A!SN?-{Nb1-dZ}-hPtws_&x>d(|`Xd)-X&Om1gvHWWMixz(r#3Eh zh==)u-Ry}I= z*5yjJC^Ry8izJoOz8hPD%RmYC*qh25DCGD(iIBBcbI>>lVIg4~?wjmb zOLDTF$0IcI0@F+HO8h9G-XhG*uLP4*h9x|y??i6vLSv0fC9{5(9!-415u4VC?r5X5 zH}v{naTNY5Xg|klr}+uDH}~plPVzas({9NyZW1|%_$0;;t#qJ$I3Q{aRAV6{BVz!o zaO^t4#G!BhhY+zij7n!GRkiL(k zw$@S67j+ICRuq-v$nry+qx%)JjQ#VH%kslkj==NCg0vw+OSCKq+H?eaW<{d-@J*DD z<_?ozV~uk1@3+s|_w+<=s~CmIX+&ox!#-SQoD2LTTsDP^DJYB#m@k`BmSx0J-_CMV zoez-ECQ_3|@9ARs2d{9tQ|4UK_+&0s>fd|YEj#LZch}j-5zbsC#Iw7{N_5k^oGiEf z$n|abi6$W5Ob-|koH{?Z<(|mFj!Q>;(wM+W3YO-vz&A*e&i1LV0pB93JxGHFud%f( z(x!c6|HM~8;JAY8LvvJuO$ftA{486GU^k!DntmyZ|LQ4`v@NxuWaESyO*DgXkadCV z+pOB-BeaR_i3lGwV71P?ti8FwHnIvUJ$doxV>&w&_d> z#Q8i$cKLF=0iZzQ+FZ3lji&=|Y+DB)YLuFM{2%OX)B~|GPL~Ua5Z{pc;C-)rzw4Hfpg`>| z`o}NHsO59_oo1g!7UqwQyiDtJ*_w_-G*r@jo8)tQHcIEI0)T zj@6^y{*>W;wxU{Y5#S%8GS|`gfMtDK96zT^ImW*p_@K&8$2x~r=vRjo7|0 z=qw!c)E^2L*Zy3p@12df^KeP&U|rjPi>A9*Uglyo?VMbr#v82${_~S)ABQ~s6_M_J zX?MZuUU#s1mrHJZvc3;@-Df_!`*CsQeWZY8I<)B^b-JRjNd8#ls~sS1ymv5MGv*$L zl-3zIeZ~$c$1@l5!M4f-dQN72JH5$F5ZI*RKpSKyH{*XUaCh%_=H)`j**x`qv_IG2 zjLKkWY%9$DU*HxY+B>~z=#dI4=j#Vr#s}l4GN8Oy>e1p9hIe?ZPWd$5;&c1_)}RA; z8f5fWY$zua`%rmxmI!5q{c=ZNct1cLmCtS(w2GzNA;V_fAz8QW;biHsRto91?e%$YNF0!KJp>S*&B8LBN)sR$N zma_vqsRmRPGZmW`H|I#@3fbR(oLvCvHC~0+JMYoyI>A>4N=i!zg_K`(6 zM+d>(s%Kg6?tagFil{VjKs`l16`LO>=lUc8O--S3z8&XeUOl(k<@@}r&bkHaVB~bV zw`U2tP~v`WgNWaS)^Fzvk?!V<^l%ww}k#4f67xLPkWhuf5 zfvqm=yLg$m2_pHH$Pw041<@QI+h?oHydvnih&~+88?HxmJ=vFfPR4}lBIj}(_dEZ( zGy&Q+QMYwyGvBjumPlyumj~KUHkWIh5GYmuRWz&e%gY`U!PWR9^qQ!a0j@nVRH8Qh z-Nn($aQ_x@9UZ4G;^6bXe%UF*kU92Ab;JSzPRLYL2Z`W4_I1x^T{xc?hoXItNHF6CxFA_P(}8Q zIubnQkEe>JKEt2-m(o4oR32b5FCU`{zHaJ!bvYU7eo7Y7V{E!rN?!?CoyX+&3+W`$ zbHO38$%L0>N|GGwK(E4FdD$uM zldnI&Dt+<|*Tmj1c34rMF$M=i|5t=hgiBXBa$KOgoRwzYba{gepY5Qc*ln!iOEJk{ z<9-6QlMFdn(6={#C`|Zbw8AFBaF122ryZK@mHgGKD8>I*pR-J`rcWB(l^H)3nFmSUNI6$ausO!R3a8O~e)6cl+2-?~b_8 zq%mNSDSq*2ta4Ki@v92XlT_foX3MlG%;C+0VyS^>&yJKYWl2D66ZgQ!yp6y(a>Lo7q9zLpsMexn6Ci@F4nbv~zcUigLhKOnY+%1VwK=A!Gra68s zMo`(f_C9t$5*Bbz+YT;azw#5+An|N|g%tMY<6ikrw_OL{AH_QvtA#qIgQS)v)Q3Nz zb_rM1wQi3P1(N-oF4l#nT;C$pv#PzWe_=X9r0|UNA$WsP zQE3|_AR^uDe4Rj*!JgUyWf~CG?_t12vi)+_uoi#sN>~#28~OhQ*7Whpn&^Lr3ym=m zr&bfE^uLfu*lDlnj3Gxt6E1V(*QE@0qZKyuO9lMc*u)0r6>OY^ME+9DoSfON&WyBA zJ*RHUJ@+(=W|C24Tc~|%DhAl*w@(Dr1J%{6dJOmf=p^piMgJcRNsW^(8vJcUG-XJG zRKX*SUDkKEASfzG;(NG#mny-i;gPGL{efMHP037jQXM5AXu`3fMnJ{%5vfZkPN150 z3;o7oVWx0`xFwB9l;GZ#M{CD7y-pKhI+u=Ab*7^Mr)-isXOTuCg?scbw8K0sQm3Qu zOxu@5t$Y<#nk01B_i;oBRHvZc@09}8hATzw>9Q4^m~N_`8<>4_o(~E?ugY)a*Vk0` zji7*R-Wpd>qY2@bx|1Q)O9M~7jxtqFDQo~|afz80x@P*<*IaHS*cMtQu!!iZkZ0sA zN2+v&A8jA24evY9PRzhu|oZ3Hx%;9s{ z7xgim;Uqe(iGvHvHTsXA$B%>tT;{~?)CX*$9Ox~glVZb8#)*-%Am2j_NFQ)<1gGsH zKDa`TIT@&Q>%ii2JWQCAZj+34|R+pc|mTj-x#a%d6sL@WVDWJ*B&Pe1KX;_{Qp7c$ zJMIm_-zt13AKxDMKV-m3RVADvKGiCB-X=aJex=?N z@W^uKz49<+bb(|1dFT6%7VhdzTM16~Sr09;*Q=C8m##STE^R9pmi(>;^I@fjrIPQR zf2Q(Q2{52@eWu~KYa*E?eV;Y{;nYJvrIkc~!*A-@)&!=m8P+U;eHSZJY5Tss(4u-P z3Rb_EL1$!pKmsqy9S^&AxZ0D_uDzqxP0lh}V{BrF95-(E@GLKD`FF#wg#G!Eu;4^T zO_=%Kz09l6v`)9~Z~h+hZ&;OOXs;nKmG*p7a{%}C&7!P(`90PPD?QZxX`VyWXPpn7 zd(fVMH!XVA>#PVQd)nT0imlH~XP)U=nK^SfKa~%0#p^}g{Y4NSqiNq;$TYuDM}%zu z-c)OWXqtxs4cu5YJu$+%26UDG7|h+>vKP(qX!Ea@=XF*mh=TIa1nK53tg~Fw3OneiIM*Ur9dV*u!mc0A z_xTtLwjWk5Z&04R2za)Rzps%nZTvYwE2l1|E;G|`rqnt(M*4HMUS(&jwHjkWwa4g~ z%CD*lwtfakpOSKmW7OA3-52M-GDc}_2dM#?GtA97%oOC)FHNx#51R}HYI0Mpa=QD z1gGh-yA@4{Y!YRnQTQagCxn1= zmIdKEI?Gr$+qFFNxABDEH0c)X%8~6UIP=+)N?P3GIc@_DP*58#i^1{YOf|KoVSz(7^}+clnsjPKT-Ln6 zI|F!)kk_RNRr?4|H>0pgdZ3OfxbEkoKZQ%ESc*~&%y6+4A-!~P2h4idd4YY+j zMb2S+OcIb@9O~fhh)`hC{Z$EeLK*N)zqsVAnJEV7R1zIHM3Czr@4=jRfmnIwJR=9l zx;Yytk!~JvQGHGD_k&ehJny1`Q}PP8o}Qkszdtw|@t3qJ7#QG)4LAie*g2uVAeuY1 zx4T;b=)+Aa;AY-&{Ppam0lxuin}Lh{!I&MmAeiAIP9gYWm;q+GN-!|32F|oUcfVxn zwRqa{r0AA_uhriD+Yp_=G$8|UHPSuJB3ZyfLbn8V7U0kPiTpwyOUZ4s)t&d3^ zw9RW~d5m`%&rwk?zyRR`L{A(AJR&0Oh&A03gs)Vsy+uH<1K#?j))xY&kOy4r`x4aCalwIt#ukq|qm4A? z86Uw}K>NTIF+PTMUOE@d78LfV{*Uut|7V_HmdjY;PAN3UQtouLQ|&4;NMAVWT$zN) zj1w_RKUx}=I{MRAd?zhN*J$(7_wuIulg4qep!N>S$@DTagV(a{)ug9LFa~nIU)9w% z$#R5OlH~MGbD3o<}Uz&2YY@(0V%tQPFm8r;u9hy6RVUw69KODb$R~c z>elM$4{s$a$H{*w{n8n=ADe+B$EB5cxck)a*zsTVAW^zBfCH}=(LuPS2XK#XW;67> z6?kdX0X>C$!K#{7m zVE26r`(B>bXK51EKcd%k)-eY~(Cf*O{rplhR|^|{hz=ptR~Uj&qM|wR>8?iDCJ!o& zD-8D$xmh3hz+;L9dOZ}|!*5-46k}q4*-!gWDJT}TL9CWt5^1JBkn@TO-_vZCNk~1r zWXMgqnznal4aB21XpZQ&R>i3T>I|vc8H2AXs;qmlUy`uu5&3!K2w+Um8mnJ#qivJzZzn{<`9-zs8Az95Hk|w ze5*OPLC7;MKzHsHL`YcNfYY(T_v`m;Zd_`h(vH*p&=R5D2~*KLYfVy_TZ1m%LpF~9 zyLgJOW=YKiWx+3X>!hc}gs!2FlgF{pA?hZnKq$aAf5-22H4!qF?9sWqYjb>%$Vn^I zUz%4`!RQr}uOJ@Uz@HuNC6Kr?f_RZJj_>9rt)rU}@>tj`f9eT|*-9!+-GplQ2W|yO zvS8%3VLFRcxYP$*^^yy&okgi82QNAu#iLV4PNG4NCDHY3bhAbC z-xY*NZuSWy{fe>;y$)tXaD(PzwUC2gYL6tsf@&uY^>@V9?Y(n@^Z!wd-$Ac`)|{&(!Q5cK6^#O|1%|c^-&;Q(a%? zn(;LY;kh)5WLzTNO-_|X@k}sJzrrZ)cPW`tY{P7=jR;v5+^3(!^MLl)^#`!+>F|q^ zAABzDI%`uISj%OJ8&k>(L*44{bmQ}sPq+-dUR}7eH*YsXdig@(F#Bw*Y6i{Yuze}> zW#EXKdAE@qcPVZGdmWP9&R}`}+HswThyJpBqtg|kFx*$KzR?avIBN7)UUTWK!(L_w zf&9}eypEqoGNsHeN~dXaRf@xOOBBst#WhAXJX1dlegG01<6go#|6izC+-(+`I$`N^Os@1bqMq!lXd1;X} zafkl1ORyzxC&)~G?7{eK$1Z#Revq(VlT}kz>niOY{7D;>dX~jPd#-E<9a27f47aP= z=9TbByJ{8+`S|<2YC}^(PU9cp-i6(gc<<2hV!u*)NobA5NIZLB1?}Y6FLB=OUQXAo z8D<@}v-|6M#@8Cb&6l!TER-7Ww^`q67eX&4wwYzed(@B3ouitG7kF>#j+)qNOtQUF zXH#?MEyQdHE_$lKz$*1MzbYg;Q>wJJt;c}a3pXOU+fJ5xr^erxxWX5YRo;nhHFU7u zu`Ch4jE~FI(o4yY(|n3km~zwspLI4n>GfN2k1Bj+9g=@$1ov2ihm5t5^PsnCTI z(X5$#r3pC?KK)D%BV%i)$>_+$GH%sY&X;O^4=6O0d-3`$Ug8X1S5kt>5S@*eOy=f( zk<1!FDStFQ)$*20EGI!c|-&~x)0&V#KZY}KI4ka*?Ev(O>-un+wy5Ef{PNrh*i+L%~PTfKNe|8EK| zL4#*-3e>RKKxW^N8-Mm^9Ix8w+FJ6l!*zjK3Xf~10Xw`F-u34@T_PKwf(3qAOi!aS zA$NwW%Q}^dX+QiC7LhP=#H#bJp*45BCQZ3X+-8%bta}A{nqP&Ry<6)c?Q8V|P(WfE zNcFqzY^*&)o?ONIt^z!4$7-Za&hVS1T0Q0`tDrk+E5F(aMj%$huZYwF>YMK)Q!VTg zgzoNCs;~Q9KOR%?f0)Vmd4O$Zlu2Qy(u?DsYnbMWf4Hv|RNEZ%`0lYP zx(z-0(7KDq<>_8MeTUvXsl)jG)CGH{ue3%B!vrU9C54>`9d@y5fxHL5z?R_lR9U9B z=lcT36Jghjet`lNTqt#d8^okVE38LVb!wE!MsZ^C**N7I(ed*J4WguR#k>~fM_AjN z*XO98XK!t`bzdB8E0*2wWc8@h^+JD|S?`j@>`tAF3NkVVJfz_B*Cn~fHWmfsKvV^O z$(Jk^m<_pvOuWG}BAzX6r&`$Tux2yA|D+LjKItV$lGTtjK1|7|cLlGicvt-}q7_A~ z+D?|5wqvL2bIopJ_F%#aIUQs8!2+>yjIORVFH|=S=3|O{42Q`N)~%FLSw))j3hx=^ zz3iyW2bbvaoEmdN60%~yXN}t`RB56FDFwfu+F>zF#3Yg{V-Dd`Z_52ppwiShz^7;h zU#~JxlW;1NeV=8^5M$1*$(q&x%zf{~W(ou|^AAoHZ0dQ)wJMhNA7W=T^kkEg**&Hm zvaIUn<)s!1`4KB9E_C;T`XhQ~({t1}_$XeQjfuSEBi_8sQ(GahLx2mDc327hWLvvZ z#XG=IcVRBCoujhO!)_IE(&#nxl5?$m&Rt4cO)55W&Gz@gN=E=$QR~!)?@*Mnmo}wa z6Mi+cw|c^7BY#jao7|I@gZHGu;dfd};JhToXnxadH1%&^5sgc7c@w?#~~bD^+@4wF&8hu{Bw&khrOJwfe<#0 zbTFx1(tg5j4saB^EkQH2Q;vLHk~2%C#>@|JwSO=h<`m#+U&l(v9P@oQk2VT1q^OxO z6by-+kysx~t<=JU=TI*Rk+?jRQ@~zaXzw^ zVpW<*#nLQeE=NNd3i=h8X(8y&ZFAL{k5gl-{?XRs5Pp>;4~|6o4LkcmM1Hr~mprLl z&eOYnBF*b_ax--%1^)@T(XxS;_8OzPkkwNCDLSMvFJ)yVN)4oEFR0bx$gl!xx;km{ zHJO)Q7E3*w=zo&C)3E1n&&Pjr4te8|fS}TA5H})BQXMWlUN62~&2psq*3LT&m&UT+ z!lQmPW9RyO98#2w0H+HMu}U|rpyxiG{_PO07>Xodd+Gmqm5yrClWT{$#9vHjue(WB zvddDAY1iqz_nK#|;-^o?7$IG&$u4iE-~c4Ay>1)wox1?5ot#ZyHKJ`HDZ|^L#(Z}! zoVx2gd--ndI7+6+hSJ4->z!($O(0=Hi&=ktfny>NDly?PjGB z*WxR^fwklCQoX`rV!?3wz3U??BEA#I+zmL2OSjIWO@+Kq+*Z3FaXfkCDD7;NiJPfR zzQ?ZNYPBc_UzNy~WHzocHp(Ke6{%KbvSz0k(Z4ah#-w!QL4RL4;pNKdL3s5>bFX(h zslU;0L*q4Wjvy89YwK8)UEVzU-t4mN7^F}BR3djR65J4MiSzfD;jTHUgM#xF7tl`l zw@1_ED*-Z7yk~vpk@eYDqA^GPp6a7sS89cW+uZ`}B|}DxG%n+DnvP>`;K+w9Mo~xV z+>Xvlwudi2np3#d%LkIiY6fFyL*!B56uy8%L8M6a*sbjId$JEkf95AaOrgx3xeYNf zktI(M??(#8li0WK7^dyd(c{|X*B&z%?FEi5*Q#W@{qb`2iB*kHQ*2vbU^@vn%&wo4 z`a@DEY3F2`#_&+{?F*gbxo{BSLxUM=e8F$j^U*@qRP|9GQ+`ZtIj!RlGfdnCxsU0k z)!s4n(-fUO5Mrs+Sh;9C4?dm05!{g|c|tBY_oq3OmR9$~k3Zd}_d!L-)$hmoeO`X} z%OOPGIi$wy@0#$^6+&b6qYOyh*RmKS#6<~$Bq~y3K`13@nb5QpJ6F=C%AfRZ?;kF% zU)?1{l$`nqLRJM)w_;TDkLwzXrzeggixlRj@88r_cn)_wYG=3JS&-_Ln3#tg>_lcL z@_+=b+N_>G%>Kxh>?Tp~G3Tua<+4BA`epqDvP$zo5xPt~7v(uC45C&iPR z?AGNyNLhepMq@CSn^?XSp)?rv3Cn7M=EFnGNR{OdNzbWex|sUS#(17HHNwhrV^KxH zhha)0y_^R*-E4j+OH9mlO*I2vsoPD+xV6dP)U#$CkNqO*O6x@8O%@f_=7s#rDj6dR zl96gGWwzZwCrD?zMTOAAe0fiyvZ0Bpgl0|xtMO*k;6-nX(a$#Xa^x3qx+ZYx5fUt5 z`%CneR>OF_N?!Jaw70)8Yr}M14%mu#8d(u=&9J$VU1W83I~eP%Pd&VuD@s03_#ByK z8NG#VH)NPRY}c1W^5DvPVZm{SAezu!8WvAzWVFl9 zB(V3n0f*6O6nC^Dx&7(f?GKGK_ie2H0U8|{Ba*n!T!wXl7iD_}tPOZ^hDGVk#=2&$ zx-(I$s`80+9t>H#fg57B@dc<}1|xDD-IsHXGv%2!A9%84&)bpAEX0asd78?9vKt#_ zv2^iNrmV3+$jM}-z3|irO`~ixKVR3#$0rKNDYw|+*u}KO*fk6!<~-yg_T|?s{gGnb zP5fF}y<zqSW_Gpu%`cn4x|@@g)TGrc zzn@e5A<}gVrOK!aN&WGARIU@lTkDK*8;)uTNquxlBX*O^VVoyqbOhpI=K?v`2EC#F z6TZ_hC#mDZHJ`W~gcSz=j zhSBHdJ6ycAb7CM_ahpRnKEc!bs9}?qcvH$(h??fiMG(d&=ys_cwCYB6MR4+kon(cc zz@mriTfYto35$tSf_rEN*~h^HDmYBK;f^*qDJr^m#n)Bi4UuQ#)mCM+PcOSC2jx`@ zR1|vECU;`QOC}sl28SW;rrITU!N4z3MggmN}0&qt!*d{I!j~jsG0) z#~;l;-qIq&CV%6ZpBi>qqjpn0NidqGn58TYy~&&4C)sSjX*->{l;CIMJuhl_X=pks zpA=0>GG2G_FjkEwPmjpvC3%wbLSd(VZSCOjFo&CIQrJ$7##Q8CH-FKy8R=PZzE}$4 zRy$TOTY==_R<$?h&J{fJ*?LpIgT@tHUC+46s2i2}elG*oYw3H&2W|5&wTU4kv!CTR ztLrmt>w5C9U-Gl*4=6P)G#zhMzh^k@nbg}7yh|$Fr+j1!KdAtIdnfqFiMmQbz{H=i zsjx;9o_KtONu6#=IV!UtLmRc}?#v&bcuFmFTTLw@t+Yp_QAV~NW#Xn6XEVW^Z?y?2G86|TIWCMK3@YMJzZuVtg^q7y) zwr6A_E@|_617*9F(rY_qf@8rl^TR38!QutRFHvegJrlThjQcv$r~Mp)>_i-zO+jam z-v9TajQ^98<;;K8Uiuu2TYl1_ct8oSgfMasltOR(3LixAL@*7tYhxPSyx$B?XUdom z=_te?={EA&Bo1tIyUYAuUa=Z&c; zUECyX1KI#W_!b0FP|2TDZ*s)#&CYqne930oiRZ%aq4mI0)71SD|8}kX5FEOrRAdsI zEj_{XGG2iZz1)$v{zANG>YPkIT=NIKaB#e^qsyeq{#%))^Zuu^1|+pi=QBp5j`-*0 zLi4nl^yrO79e+-&kJ=wV3BT%CS-XMjNy?3bq8#-jxIaNaWx&KQae;?>{cw5r^b5Lv zKiEG7kB>~S&0^RSy8oL`a%+&`6zW5Wc-25%7)(0>+t$otV5K$$9QsFRD8c6TGI*v8 z%8)J`@xaCP;uYt literal 0 HcmV?d00001 diff --git a/docs/src/assets/images/transformations.dot.svg b/docs/src/assets/images/transformations.dot.svg new file mode 100644 index 000000000..1e98f612d --- /dev/null +++ b/docs/src/assets/images/transformations.dot.svg @@ -0,0 +1,124 @@ + + + + + + +%3 + + + +tilde_node + +x ~ Normal() + + + +base_node + + vn = +@varname +(x) +dist = Normal() +x, vi = ... + + + +tilde_node->base_node + + +   +@model + + + +assume + +assume(vn, dist, vi) + + + +base_node->assume + + +  tilde-pipeline + + + +iflinked + + +if + istrans(vi, vn) + + + +assume->iflinked + + + + + +without_linking + +f = from_internal_transform(vi, vn, dist) + + + +iflinked->without_linking + + +   +false + + + +with_linking + +f = from_linked_internal_transform(vi, vn, dist) + + + +iflinked->with_linking + + +   +true + + + +with_logabsdetjac + +x, logjac = with_logabsdet_jacobian(f, getindex_internal(vi, vn, dist)) + + + +without_linking->with_logabsdetjac + + + + + +with_linking->with_logabsdetjac + + + + + +return + + +return + x, logpdf(dist, x) - logjac, vi + + + +with_logabsdetjac->return + + + + + diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 7ee800556..45134a593 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -316,7 +316,7 @@ While if `dist` is not provided, we have: Notice that `dist` is not present here, but otherwise the diagrams are the same. !!! warning - This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occcurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. ## Other functionalities From 95dc8e37a1195ba87e900c957e6d9e578d7015d8 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:51:34 +0000 Subject: [PATCH 134/182] temporarily removed `VarNameVector` completely --- docs/make.jl | 2 +- docs/src/internals/varinfo.md | 309 -------------- src/DynamicPPL.jl | 2 - src/threadsafe.jl | 2 - src/varinfo.jl | 216 ---------- src/varnamevector.jl | 756 ---------------------------------- test/runtests.jl | 1 - test/test_util.jl | 1 - test/varinfo.jl | 6 - test/varnamevector.jl | 472 --------------------- 10 files changed, 1 insertion(+), 1766 deletions(-) delete mode 100644 docs/src/internals/varinfo.md delete mode 100644 src/varnamevector.jl delete mode 100644 test/varnamevector.jl diff --git a/docs/make.jl b/docs/make.jl index de557ef9f..65d43d524 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,7 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], - "Internals" => ["internals/varinfo.md", "internals/transformations.md"], + "Internals" => ["internals/transformations.md"], ], checkdocs=:exports, doctest=false, diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md deleted file mode 100644 index 8d74fd2fa..000000000 --- a/docs/src/internals/varinfo.md +++ /dev/null @@ -1,309 +0,0 @@ -# Design of `VarInfo` - -[`VarInfo`](@ref) is a fairly simple structure. - -```@docs; canonical=false -VarInfo -``` - -It contains - - - a `logp` field for accumulation of the log-density evaluation, and - - a `metadata` field for storing information about the realizations of the different variables. - -Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. - -**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding to a variable `@varname(x)`. - -!!! note - - We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. - -To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo`, and hence the underlying `metadata`, to replicate the following functionality of `Dict`: - - - `keys(::Dict)`: return all the `VarName`s present in `metadata`. - - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. - - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. - - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. - - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. - - `empty!(::Dict)`: delete all realizations in `metadata`. - - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. - -*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: - - - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. - - + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. - - - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. - -We also want some additional methods that are *not* part of the `Dict` or `Vector` interface: - - - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. - - - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. - -In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - - - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. - - - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal representaton of a particular `VarName`. - -Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: - - 1. Type-stable when possible, but functional when not. - 2. Efficient storage and iteration when possible, but functional when not. - -The "but functional when not" is important as we want to support arbitrary models, which means that we can't always have these performance properties. - -In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). - -## Type-stability - -Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. - -Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form - -```@example varinfo-design -using DynamicPPL, Distributions, FillArrays - -@model function demo() - x ~ product_distribution(Fill(Bernoulli(0.5), 2)) - y ~ Normal(0, 1) - return nothing -end -``` - -then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where - - - `Vx` is a container with `eltype` `Bool`, and - - `Vy` is a container with `eltype` `Float64`. - -Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. - -For example, with the model above we have - -```@example varinfo-design -# Type-unstable `VarInfo` -varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) -typeof(varinfo_untyped.metadata) -``` - -```@example varinfo-design -# Type-stable `VarInfo` -varinfo_typed = DynamicPPL.typed_varinfo(demo()) -typeof(varinfo_typed.metadata) -``` - -But they both work as expected but one results in concrete typing and the other does not: - -```@example varinfo-design -varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] -``` - -```@example varinfo-design -varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] -``` - -Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entries while the typed uses `Vector{Bool}`. This is because the untyped version needs the underlying container to be able to handle both the `Bool` for `x` and the `Float64` for `y`, while the typed version can use a `Vector{Bool}` for `x` and a `Vector{Float64}` for `y` due to its usage of `NamedTuple`. - -!!! warning - - Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. - - ```julia - x[1] ~ Bernoulli(0.5) - x[2] ~ Normal(0, 1) - ``` - - In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. - - In practice, rarely observe such mixing of types, therefore in DynamicPPL, and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. - -!!! warning - - Another downside with such a `NamedTuple` approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. - - For these scenarios it can be useful to fall back to "untyped" representations. - -Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. - -## Efficient storage and iteration - -Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): - -```@docs -DynamicPPL.VarNameVector -``` - -In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. - -This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: - - - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. - - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - - `transforms::Vector`: the transforms associated with each `VarName`. - -Mutating functions, e.g. `setindex!(vnv::VarNameVector, val, vn::VarName)`, are then treated according to the following rules: - - 1. If `vn` is not already present: add it to the end of `vnv.varnames`, add the `val` to the underlying `vnv.vals`, etc. - - 2. If `vn` is already present in `vnv`: - - 1. If `val` has the *same length* as the existing value for `vn`: replace existing value. - 2. If `val` has a *smaller length* than the existing value for `vn`: replace existing value and mark the remaining indices as "inactive" by increasing the entry in `vnv.num_inactive` field. - 3. If `val` has a *larger length* than the existing value for `vn`: expand the underlying `vnv.vals` to accommodate the new value, update all `VarName`s occuring after `vn`, and update the `vnv.ranges` to point to the new range for `vn`. - -This means that `VarNameVector` is allowed to grow as needed, while "shrinking" (i.e. insertion of smaller elements) is handled by simply marking the redundant indices as "inactive". This turns out to be efficient for use-cases that we are generally interested in. - -For example, we want to optimize code-paths which effectively boil down to inner-loop in the following example: - -```julia -# Construct a `VarInfo` with types inferred from `model`. -varinfo = VarInfo(model) - -# Repeatedly sample from `model`. -for _ in 1:num_samples - rand!(rng, model, varinfo) - - # Do something with `varinfo`. - # ... -end -``` - -There are typically a few scenarios where we encounter changing representation sizes of a random variable `x`: - - 1. We're working with a transformed version `x` which is represented in a lower-dimensional space, e.g. transforming a `x ~ LKJ(2, 1)` to unconstrained `y = f(x)` takes us from 2-by-2 `Matrix{Float64}` to a 1-length `Vector{Float64}`. - 2. `x` has a random size, e.g. in a mixture model with a prior on the number of components. Here the size of `x` can vary widly between every realization of the `Model`. - -In scenario (1), we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". - -In scenario (2), we end up increasing the allocated memory for the randomly sized `x`, eventually leading to a vector that is large enough to hold realizations without needing to reallocate. But this can still lead to unnecessary memory usage, which might be undesirable. Hence one has to make a decision regarding the trade-off between memory usage and performance for the use-case at hand. - -To help with this, we have the following functions: - -```@docs -DynamicPPL.has_inactive -DynamicPPL.num_inactive -DynamicPPL.num_allocated -DynamicPPL.is_contiguous -DynamicPPL.contiguify! -``` - -For example, one might encounter the following scenario: - -```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") - -for i in 1:5 - x = fill(true, rand(1:100)) - DynamicPPL.update!(vnv, @varname(x), x) - println( - "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", - ) -end -``` - -We can then insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion whenever the allocation grows too large to reduce overall memory usage: - -```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") - -for i in 1:5 - x = fill(true, rand(1:100)) - DynamicPPL.update!(vnv, @varname(x), x) - if DynamicPPL.num_allocated(vnv) > 10 - DynamicPPL.contiguify!(vnv) - end - println( - "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", - ) -end -``` - -This does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. However, this also ensures that the the underlying `Vector{T}` is contiguous, which is important for performance. Hence, if we're about to do a lot of work with the `VarNameVector` without insertions, etc., it can be worth it to do a sweep to ensure that the underlying `Vector{T}` is contiguous. - -!!! note - - Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. - -Continuing from the example from the previous section, we can use a `VarInfo` with a `VarNameVector` as the `metadata` field: - -```@example varinfo-design -# Type-unstable -varinfo_untyped_vnv = DynamicPPL.VectorVarInfo(varinfo_untyped) -varinfo_untyped_vnv[@varname(x)], varinfo_untyped_vnv[@varname(y)] -``` - -```@example varinfo-design -# Type-stable -varinfo_typed_vnv = DynamicPPL.VectorVarInfo(varinfo_typed) -varinfo_typed_vnv[@varname(x)], varinfo_typed_vnv[@varname(y)] -``` - -If we now try to `delete!` `@varname(x)` - -```@example varinfo-design -haskey(varinfo_untyped_vnv, @varname(x)) -``` - -```@example varinfo-design -DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) -``` - -```@example varinfo-design -# `delete!` -DynamicPPL.delete!(varinfo_untyped_vnv.metadata, @varname(x)) -DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) -``` - -```@example varinfo-design -haskey(varinfo_untyped_vnv, @varname(x)) -``` - -Or insert a differently-sized value for `@varname(x)` - -```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 1)) -varinfo_untyped_vnv[@varname(x)] -``` - -```@example varinfo-design -DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) -``` - -```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 4)) -varinfo_untyped_vnv[@varname(x)] -``` - -```@example varinfo-design -DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) -``` - -### Performance summary - -In the end, we have the following "rough" performance characteristics for `VarNameVector`: - -| Method | Is blazingly fast? | -|:---------------------------------------:|:--------------------------------------------------------------------------------------------:| -| `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | -| `delete!` | ${\color{red} \times}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | -| `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | - -## Other methods - -```@docs -DynamicPPL.replace_values(::VarNameVector, vals::AbstractVector) -``` - -```@docs; canonical=false -DynamicPPL.values_as(::VarNameVector) -``` diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index ec0b086f7..c099a1fe7 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -44,7 +44,6 @@ export AbstractVarInfo, UntypedVarInfo, TypedVarInfo, SimpleVarInfo, - VarNameVector, push!!, empty!!, subset, @@ -159,7 +158,6 @@ include("sampler.jl") include("varname.jl") include("distribution_wrappers.jl") include("contexts.jl") -include("varnamevector.jl") include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 648a8ef1c..dd0357547 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -55,8 +55,6 @@ function setlogp!!(vi::ThreadSafeVarInfoWithRef, logp) return ThreadSafeVarInfo(setlogp!!(vi.varinfo, logp), vi.logps) end -has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) = has_varnamevector(vi.varinfo) - function BangBang.push!!( vi::ThreadSafeVarInfo, vn::VarName, r, dist::Distribution, gidset::Set{Selector} ) diff --git a/src/varinfo.jl b/src/varinfo.jl index a12a25cc3..eda8213b1 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -101,7 +101,6 @@ struct VarInfo{Tmeta,Tlogp} <: AbstractVarInfo logp::Base.RefValue{Tlogp} num_produce::Base.RefValue{Int} end -const VectorVarInfo = VarInfo{<:VarNameVector} const UntypedVarInfo = VarInfo{<:Metadata} const TypedVarInfo = VarInfo{<:NamedTuple} const VarInfoOrThreadSafeVarInfo{Tmeta} = Union{ @@ -132,46 +131,6 @@ function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) ) end -# No-op if we're already working with a `VarNameVector`. -metadata_to_varnamevector(vnv::VarNameVector) = vnv -function metadata_to_varnamevector(md::Metadata) - idcs = copy(md.idcs) - vns = copy(md.vns) - ranges = copy(md.ranges) - vals = copy(md.vals) - transforms = map(md.dists) do dist - # TODO: Handle linked distributions. - from_vec_transform(dist) - end - - return VarNameVector( - OrderedDict{eltype(keys(idcs)),Int}(idcs), vns, ranges, vals, transforms - ) -end - -function VectorVarInfo(vi::UntypedVarInfo) - md = metadata_to_varnamevector(vi.metadata) - lp = getlogp(vi) - return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) -end - -function VectorVarInfo(vi::TypedVarInfo) - md = map(metadata_to_varnamevector, vi.metadata) - lp = getlogp(vi) - return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) -end - -""" - has_varnamevector(varinfo::VarInfo) - -Returns `true` if `varinfo` uses `VarNameVector` as metadata. -""" -has_varnamevector(vi) = false -function has_varnamevector(vi::VarInfo) - return vi.metadata isa VarNameVector || - (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) -end - function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -373,10 +332,6 @@ function _merge(varinfo_left::VarInfo, varinfo_right::VarInfo) ) end -function merge_metadata(vnv_left::VarNameVector, vnv_right::VarNameVector) - return merge(vnv_left, vnv_right) -end - @generated function merge_metadata( metadata_left::NamedTuple{names_left}, metadata_right::NamedTuple{names_right} ) where {names_left,names_right} @@ -580,8 +535,6 @@ Return the distribution from which `vn` was sampled in `vi`. """ getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] -# HACK: we shouldn't need this -getdist(::VarNameVector, ::VarName) = nothing """ getindex_internal(vi::VarInfo, vn::VarName) @@ -592,8 +545,6 @@ The values may or may not be transformed to Euclidean space. """ getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) -# HACK: We shouldn't need this -getindex_internal(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) """ setval!(vi::VarInfo, val, vn::VarName) @@ -609,9 +560,6 @@ end function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) end -function setval!(vnv::VarNameVector, val, vn::VarName) - return setindex_raw!(vnv, tovec(val), vn) -end """ getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) @@ -639,7 +587,6 @@ function getall(md::Metadata) Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) ) end -getall(vnv::VarNameVector) = vnv.vals """ setall!(vi::VarInfo, val) @@ -655,12 +602,6 @@ function _setall!(metadata::Metadata, val) metadata.vals[r] .= val[r] end end -function _setall!(vnv::VarNameVector, val) - # TODO: Do something more efficient here. - for i in 1:length(vnv) - vnv[i] = val[i] - end -end @generated function _setall!(metadata::NamedTuple{names}, val) where {names} expr = Expr(:block) start = :(1) @@ -693,10 +634,6 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end -function settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) - settrans!(vnv, trans, vn) - return vnv -end function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) @@ -1044,8 +981,6 @@ end # X -> R for all variables associated with given sampler function link!!(t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) - # If we're working with a `VarNameVector`, we always use immutable. - has_varnamevector(vi) && return link(t, vi, spl, model) # Call `_link!` instead of `link!` to avoid deprecation warning. _link!(vi, spl) return vi @@ -1141,8 +1076,6 @@ end function invlink!!( t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model ) - # If we're working with a `VarNameVector`, we always use immutable. - has_varnamevector(vi) && return invlink(t, vi, spl, model) # Call `_invlink!` instead of `invlink!` to avoid deprecation warning. _invlink!(vi, spl) return vi @@ -1377,66 +1310,6 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar ) end -function _link_metadata!( - model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns -) - # HACK: We ignore `target_vns` here. - vns = keys(metadata) - # Need to extract the priors from the model. - dists = extract_priors(model, varinfo) - - is_transformed = copy(metadata.is_transformed) - - # Construct the linking transformations. - link_transforms = map(vns) do vn - # If `vn` is not part of `target_vns`, the `identity` transformation is used. - if (target_vns !== nothing && vn ∉ target_vns) - return identity - end - - # Otherwise, we derive the transformation from the distribution. - is_transformed[getidx(metadata, vn)] = true - internal_to_linked_internal_transform(varinfo, vn, dists[vn]) - end - # Compute the transformed values. - ys = map(vns, link_transforms) do vn, f - # TODO: Do we need to handle scenarios where `vn` is not in `dists`? - dist = dists[vn] - x = getindex_internal(metadata, vn) - y, logjac = with_logabsdet_jacobian(f, x) - # Accumulate the log-abs-det jacobian correction. - acclogp!!(varinfo, -logjac) - # Return the transformed value. - return y - end - # Extract the from-vec transformations. - fromvec_transforms = map(from_vec_transform, ys) - # Compose the transformations to form a full transformation from - # unconstrained vector representation to constrained space. - transforms = map(∘, map(inverse, link_transforms), fromvec_transforms) - # Convert to vector representation. - yvecs = map(tovec, ys) - - # Determine new ranges. - ranges_new = similar(metadata.ranges) - offset = 0 - for (i, v) in enumerate(yvecs) - r_start, r_end = offset + 1, length(v) + offset - offset = r_end - ranges_new[i] = r_start:r_end - end - - # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNameVector( - metadata.varname_to_index, - metadata.varnames, - ranges_new, - reduce(vcat, yvecs), - transforms, - is_transformed, - ) -end - function invlink( ::DynamicTransformation, varinfo::VarInfo, spl::AbstractSampler, model::Model ) @@ -1537,55 +1410,6 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe ) end -function _invlink_metadata!( - model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns -) - # HACK: We ignore `target_vns` here. - # TODO: Make use of `update!` to aovid copying values. - # => Only need to allocate for transformations. - - vns = keys(metadata) - is_transformed = copy(metadata.is_transformed) - - # Compute the transformed values. - xs = map(vns) do vn - f = gettransform(metadata, vn) - y = getindex_internal(metadata, vn) - # No need to use `with_reconstruct` as `f` will include this. - x, logjac = with_logabsdet_jacobian(f, y) - # Accumulate the log-abs-det jacobian correction. - acclogp!!(varinfo, -logjac) - # Mark as no longer transformed. - is_transformed[getidx(metadata, vn)] = false - # Return the transformed value. - return x - end - # Compose the transformations to form a full transformation from - # unconstrained vector representation to constrained space. - transforms = map(from_vec_transform, xs) - # Convert to vector representation. - xvecs = map(tovec, xs) - - # Determine new ranges. - ranges_new = similar(metadata.ranges) - offset = 0 - for (i, v) in enumerate(xvecs) - r_start, r_end = offset + 1, length(v) + offset - offset = r_end - ranges_new[i] = r_start:r_end - end - - # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNameVector( - metadata.varname_to_index, - metadata.varnames, - ranges_new, - reduce(vcat, xvecs), - transforms, - is_transformed, - ) -end - """ islinked(vi::VarInfo, spl::Union{Sampler, SampleFromPrior}) @@ -1655,14 +1479,6 @@ function getindex(vi::VarInfo, vn::VarName, dist::Distribution) val = getindex_internal(vi, vn) return from_maybe_linked_internal(vi, vn, dist, val) end -# HACK: Allows us to also work with `VarNameVector` where `dist` is not used, -# but we instead use a transformation stored with the variable. -function getindex(vi::VarInfo, vn::VarName, ::Nothing) - if !haskey(vi, vn) - throw(KeyError(vn)) - end - return getmetadata(vi, vn)[vn] -end function getindex(vi::VarInfo, vns::Vector{<:VarName}) vals_linked = mapreduce(vcat, vns) do vn @@ -1680,15 +1496,6 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) end getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) -function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) - # FIXME: This is too hacky. - # We know this will only be hit if we're working with `VarNameVector`, - # so we can just use the `getindex_raw` for `VarNameVector`. - # NOTE: This won't result in the same behavior as `getindex_raw` - # for the other `VarInfo`s since we don't have access to the `dist` - # and so can't call `reconstruct`. - return getindex_raw(getmetadata(vi, vn), vn) -end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) f = from_internal_transform(vi, vn, dist) return f(getindex_internal(vi, vn)) @@ -1876,11 +1683,6 @@ function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) return meta end -function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) - f = from_vec_transform(dist) - return push!(vnv, vn, r, f) -end - """ setorder!(vi::VarInfo, vn::VarName, index::Int) @@ -1895,7 +1697,6 @@ function setorder!(metadata::Metadata, vn::VarName, index::Int) metadata.orders[metadata.idcs[vn]] = index return metadata end -setorder!(vnv::VarNameVector, ::VarName, ::Int) = vnv """ getorder(vi::VarInfo, vn::VarName) @@ -1921,8 +1722,6 @@ end function is_flagged(metadata::Metadata, vn::VarName, flag::String) return metadata.flags[flag][getidx(metadata, vn)] end -# HACK: This is bad. Should we always return `true` here? -is_flagged(::VarNameVector, ::VarName, flag::String) = flag == "del" ? true : false """ unset_flag!(vi::VarInfo, vn::VarName, flag::String) @@ -1937,7 +1736,6 @@ function unset_flag!(metadata::Metadata, vn::VarName, flag::String) metadata.flags[flag][getidx(metadata, vn)] = false return metadata end -unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv """ set_retained_vns_del_by_spl!(vi::VarInfo, spl::Sampler) @@ -2295,8 +2093,6 @@ function values_from_metadata(md::Metadata) ) end -values_from_metadata(md::VarNameVector) = pairs(md) - # Transforming from internal representation to distribution representation. # Without `dist` argument: base on `dist` extracted from self. function from_internal_transform(vi::VarInfo, vn::VarName) @@ -2305,17 +2101,11 @@ end function from_internal_transform(md::Metadata, vn::VarName) return from_internal_transform(md, vn, getdist(md, vn)) end -function from_internal_transform(md::VarNameVector, vn::VarName) - return gettransform(md, vn) -end # With both `vn` and `dist` arguments: base on provided `dist`. function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) -function from_internal_transform(::VarNameVector, ::VarName, dist) - return from_vec_transform(dist) -end # Without `dist` argument: base on `dist` extracted from self. function from_linked_internal_transform(vi::VarInfo, vn::VarName) @@ -2324,9 +2114,6 @@ end function from_linked_internal_transform(md::Metadata, vn::VarName) return from_linked_internal_transform(md, vn, getdist(md, vn)) end -function from_linked_internal_transform(md::VarNameVector, vn::VarName) - return gettransform(md, vn) -end # With both `vn` and `dist` arguments: base on provided `dist`. function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) # Dispatch to metadata in case this alters the behavior. @@ -2335,6 +2122,3 @@ end function from_linked_internal_transform(::Metadata, ::VarName, dist) return from_linked_vec_transform(dist) end -function from_linked_internal_transform(::VarNameVector, ::VarName, dist) - return from_linked_vec_transform(dist) -end diff --git a/src/varnamevector.jl b/src/varnamevector.jl deleted file mode 100644 index ed919a20c..000000000 --- a/src/varnamevector.jl +++ /dev/null @@ -1,756 +0,0 @@ -""" - VarNameVector - -A container that works like a `Vector` and an `OrderedDict` but is neither. - -# Fields -$(FIELDS) -""" -struct VarNameVector{ - K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData -} - "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" - varname_to_index::OrderedDict{K,Int} - - "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" - varnames::TVN # AbstractVector{<:VarName} - - "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" - ranges::Vector{UnitRange{Int}} - - "vector of values of all variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" - vals::TVal # AbstractVector{<:Real} - - "vector of transformations whose inverse takes us back to the original space" - transforms::TTrans - - "specifies whether a variable is transformed or not " - is_transformed::BitVector - - "additional entries which are considered inactive" - num_inactive::OrderedDict{Int,Int} - - "metadata associated with the varnames" - metadata::MData -end - -function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) - return vnv_left.varname_to_index == vnv_right.varname_to_index && - vnv_left.varnames == vnv_right.varnames && - vnv_left.ranges == vnv_right.ranges && - vnv_left.vals == vnv_right.vals && - vnv_left.transforms == vnv_right.transforms && - vnv_left.is_transformed == vnv_right.is_transformed && - vnv_left.num_inactive == vnv_right.num_inactive && - vnv_left.metadata == vnv_right.metadata -end - -function VarNameVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_transformed=fill!(BitVector(undef, length(varnames)), 0), -) - return VarNameVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_transformed, - OrderedDict{Int,Int}(), - nothing, - ) -end -# TODO: Do we need this? -function VarNameVector{K,V}() where {K,V} - return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) -end - -istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] -function settrans!(vnv::VarNameVector, val::Bool, vn::VarName) - return vnv.is_transformed[vnv.varname_to_index[vn]] = val -end - -VarNameVector() = VarNameVector{VarName,Real}() -VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) -VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) -VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) -function VarNameVector( - varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) -) - # TODO: Check uniqueness of `varnames`? - - # Convert `vals` into a vector of vectors. - vals_vecs = map(tovec, vals) - - # TODO: Is this really the way to do this? - if !(eltype(varnames) <: VarName) - varnames = convert(Vector{VarName}, varnames) - end - varname_to_index = OrderedDict{eltype(varnames),Int}() - ranges = Vector{UnitRange{Int}}() - offset = 0 - for (i, (vn, x)) in enumerate(zip(varnames, vals_vecs)) - # Add the varname index. - push!(varname_to_index, vn => length(varname_to_index) + 1) - # Add the range. - r = (offset + 1):(offset + length(x)) - push!(ranges, r) - # Update the offset. - offset = r[end] - end - - return VarNameVector( - varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms - ) -end - -""" - replace_values(vnv::VarNameVector, vals::AbstractVector) - -Replace the values in `vnv` with `vals`. - -This is useful when we want to update the entire underlying vector of values -in one go or if we want to change the how the values are stored, e.g. alter the `eltype`. - -!!! warning - This replaces the raw underlying values, and so care should be taken when using this - function. For example, if `vnv` has any inactive entries, then the provided `vals` - should also contain the inactive entries to avoid unexpected behavior. - -# Example - -```jldoctest varnamevector-replace-values -julia> using DynamicPPL: VarNameVector, replace_values - -julia> vnv = VarNameVector(@varname(x) => [1.0]); - -julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] -true -``` - -This is also useful when we want to differentiate wrt. the values -using automatic differentiation, e.g. ForwardDiff.jl. - -```jldoctest varnamevector-replace-values -julia> using ForwardDiff: ForwardDiff - -julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) -f (generic function with 1 method) - -julia> ForwardDiff.gradient(f, [1.0]) -1-element Vector{Float64}: - 2.0 -``` -""" -replace_values(vnv::VarNameVector, vals) = Setfield.@set vnv.vals = vals - -# Some `VarNameVector` specific functions. -getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] - -getrange(vnv::VarNameVector, idx::Int) = vnv.ranges[idx] -getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) - -gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] - -""" - has_inactive(vnv::VarNameVector) - -Returns `true` if `vnv` has inactive ranges. -""" -has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) - -""" - num_inactive(vnv::VarNameVector) - -Return the number of inactive entries in `vnv`. -""" -num_inactive(vnv::VarNameVector) = sum(values(vnv.num_inactive)) - -""" - num_inactive(vnv::VarNameVector, vn::VarName) - -Returns the number of inactive entries for `vn` in `vnv`. -""" -num_inactive(vnv::VarNameVector, vn::VarName) = num_inactive(vnv, getidx(vnv, vn)) -num_inactive(vnv::VarNameVector, idx::Int) = get(vnv.num_inactive, idx, 0) - -""" - num_allocated(vnv::VarNameVector) - -Returns the number of allocated entries in `vnv`. -""" -num_allocated(vnv::VarNameVector) = length(vnv.vals) - -""" - num_allocated(vnv::VarNameVector, vn::VarName) - -Returns the number of allocated entries for `vn` in `vnv`. -""" -num_allocated(vnv::VarNameVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) -function num_allocated(vnv::VarNameVector, idx::Int) - return length(getrange(vnv, idx)) + num_inactive(vnv, idx) -end - -# Basic array interface. -Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) -Base.length(vnv::VarNameVector) = - if isempty(vnv.num_inactive) - length(vnv.vals) - else - sum(length, vnv.ranges) - end -Base.size(vnv::VarNameVector) = (length(vnv),) -Base.isempty(vnv::VarNameVector) = isempty(vnv.varnames) - -# TODO: We should probably remove this -Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() - -# Dictionary interface. -Base.keys(vnv::VarNameVector) = vnv.varnames -Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) -Base.pairs(vnv::VarNameVector) = (vn => vnv[vn] for vn in keys(vnv)) - -Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) - -# `getindex` & `setindex!` -Base.getindex(vnv::VarNameVector, i::Int) = getindex_raw(vnv, i) -function Base.getindex(vnv::VarNameVector, vn::VarName) - x = getindex_raw(vnv, vn) - f = gettransform(vnv, vn) - return f(x) -end - -function find_range_from_sorted(ranges::AbstractVector{<:AbstractRange}, x) - # TODO: Assume `ranges` to be sorted and contiguous, and use `searchsortedfirst` - # for a more efficient approach. - range_idx = findfirst(Base.Fix1(∈, x), ranges) - - # If we're out of bounds, we raise an error. - if range_idx === nothing - throw(ArgumentError("Value $x is not in any of the ranges.")) - end - - return range_idx -end - -function adjusted_ranges(vnv::VarNameVector) - # Every range following inactive entries needs to be shifted. - offset = 0 - ranges_adj = similar(vnv.ranges) - for (idx, r) in enumerate(vnv.ranges) - # Remove the `offset` in `r` due to inactive entries. - ranges_adj[idx] = r .- offset - # Update `offset`. - offset += get(vnv.num_inactive, idx, 0) - end - - return ranges_adj -end - -function index_to_raw_index(vnv::VarNameVector, i::Int) - # If we don't have any inactive entries, there's nothing to do. - has_inactive(vnv) || return i - - # Get the adjusted ranges. - ranges_adj = adjusted_ranges(vnv) - # Determine the adjusted range that the index corresponds to. - r_idx = find_range_from_sorted(ranges_adj, i) - r = vnv.ranges[r_idx] - # Determine how much of the index `i` is used to get to this range. - i_used = r_idx == 1 ? 0 : sum(length, ranges_adj[1:(r_idx - 1)]) - # Use remainder to index into `r`. - i_remainder = i - i_used - return r[i_remainder] -end - -getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] -getindex_raw(vnv::VarNameVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] - -# `getindex` for `Colon` -function Base.getindex(vnv::VarNameVector, ::Colon) - return if has_inactive(vnv) - mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) - else - vnv.vals - end -end - -function getindex_raw(vnv::VarNameVector, ::Colon) - return if has_inactive(vnv) - mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) - else - vnv.vals - end -end - -# HACK: remove this as soon as possible. -Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] - -Base.setindex!(vnv::VarNameVector, val, i::Int) = setindex_raw!(vnv, val, i) -function Base.setindex!(vnv::VarNameVector, val, vn::VarName) - f = inverse(gettransform(vnv, vn)) - return setindex_raw!(vnv, f(val), vn) -end - -setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] = val -function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) - return vnv.vals[getrange(vnv, vn)] = val -end - -# `empty!(!)` -function Base.empty!(vnv::VarNameVector) - # TODO: Or should the semantics be different, e.g. keeping `varnames`? - empty!(vnv.varname_to_index) - empty!(vnv.varnames) - empty!(vnv.ranges) - empty!(vnv.vals) - empty!(vnv.transforms) - empty!(vnv.num_inactive) - return nothing -end -BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) - -function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) - # Return early if possible. - isempty(left_vnv) && return deepcopy(right_vnv) - isempty(right_vnv) && return deepcopy(left_vnv) - - # Determine varnames. - vns_left = left_vnv.varnames - vns_right = right_vnv.varnames - vns_both = union(vns_left, vns_right) - - # Determine `eltype` of `vals`. - T_left = eltype(left_vnv.vals) - T_right = eltype(right_vnv.vals) - T = promote_type(T_left, T_right) - # TODO: Is this necessary? - if !(T <: Real) - T = Real - end - - # Determine `eltype` of `varnames`. - V_left = eltype(left_vnv.varnames) - V_right = eltype(right_vnv.varnames) - V = promote_type(V_left, V_right) - if !(V <: VarName) - V = VarName - end - - # Determine `eltype` of `transforms`. - F_left = eltype(left_vnv.transforms) - F_right = eltype(right_vnv.transforms) - F = promote_type(F_left, F_right) - - # Allocate. - varnames_to_index = OrderedDict{V,Int}() - ranges = UnitRange{Int}[] - vals = T[] - transforms = F[] - is_transformed = BitVector(undef, length(vns_both)) - - # Range offset. - offset = 0 - - for (idx, vn) in enumerate(vns_both) - # Extract the necessary information from `left` or `right`. - if vn in vns_left && !(vn in vns_right) - # `vn` is only in `left`. - varnames_to_index[vn] = idx - val = getindex_raw(left_vnv, vn) - n = length(val) - r = (offset + 1):(offset + n) - f = gettransform(left_vnv, vn) - is_transformed[idx] = istrans(left_vnv, vn) - else - # `vn` is either in both or just `right`. - varnames_to_index[vn] = idx - val = getindex_raw(right_vnv, vn) - n = length(val) - r = (offset + 1):(offset + n) - f = gettransform(right_vnv, vn) - is_transformed[idx] = istrans(right_vnv, vn) - end - # Update. - append!(vals, val) - push!(ranges, r) - push!(transforms, f) - # Increment `offset`. - offset += n - end - - return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) -end - -function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) - # NOTE: This does not specialize types when possible. - vnv_new = similar(vnv) - # Return early if possible. - isempty(vnv) && return vnv_new - - for vn in vns - push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) - end - - return vnv_new -end - -# `similar` -similar_metadata(::Nothing) = nothing -similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) -function Base.similar(vnv::VarNameVector) - # NOTE: Whether or not we should empty the underlying containers or note - # is somewhat ambiguous. For example, `similar(vnv.varname_to_index)` will - # result in an empty `AbstractDict`, while the vectors, e.g. `vnv.ranges`, - # will result in non-empty vectors but with entries as `undef`. But it's - # much easier to write the rest of the code assuming that `undef` is not - # present, and so for now we empty the underlying containers, thus differing - # from the behavior of `similar` for `AbstractArray`s. - return VarNameVector( - similar(vnv.varname_to_index), - similar(vnv.varnames, 0), - similar(vnv.ranges, 0), - similar(vnv.vals, 0), - similar(vnv.transforms, 0), - BitVector(), - similar(vnv.num_inactive), - similar_metadata(vnv.metadata), - ) -end - -""" - is_contiguous(vnv::VarNameVector) - -Returns `true` if the underlying data of `vnv` is stored in a contiguous array. - -This is equivalent to negating [`has_inactive(vnv)`](@ref). -""" -is_contiguous(vnv::VarNameVector) = !has_inactive(vnv) - -function nextrange(vnv::VarNameVector, x) - # If `vnv` is empty, return immediately. - isempty(vnv) && return 1:length(x) - - # The offset will be the last range's end + its number of inactive entries. - vn_last = vnv.varnames[end] - idx = getidx(vnv, vn_last) - offset = last(getrange(vnv, idx)) + num_inactive(vnv, idx) - - return (offset + 1):(offset + length(x)) -end - -# `push!` and `push!!`: add a variable to the varname vector. -function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) - # Error if we already have the variable. - haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) - # NOTE: We need to compute the `nextrange` BEFORE we start mutating - # the underlying; otherwise we might get some strange behaviors. - val_vec = tovec(val) - r_new = nextrange(vnv, val_vec) - vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 - push!(vnv.varnames, vn) - push!(vnv.ranges, r_new) - append!(vnv.vals, val_vec) - push!(vnv.transforms, transform) - push!(vnv.is_transformed, false) - return nothing -end - -""" - shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) - -Shifts the elements of `x` starting from index `start` by `n` to the right. -""" -function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) - x[(start + n):end] = x[start:(end - n)] - return x -end - -""" - shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) - -Shifts the ranges of variables in `vnv` starting from index `idx` by `n`. -""" -function shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) - for i in (idx + 1):length(vnv.ranges) - vnv.ranges[i] = vnv.ranges[i] .+ n - end - return nothing -end - -# `update!` and `update!!`: update a variable in the varname vector. -""" - update!(vnv::VarNameVector, vn::VarName, val[, transform]) - -Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. - -If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). -""" -function update!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) - if !haskey(vnv, vn) - # Here we just add a new entry. - return push!(vnv, vn, val, transform) - end - - # Here we update an existing entry. - val_vec = tovec(val) - idx = getidx(vnv, vn) - # Extract the old range. - r_old = getrange(vnv, idx) - start_old, end_old = first(r_old), last(r_old) - n_old = length(r_old) - # Compute the new range. - n_new = length(val_vec) - start_new = start_old - end_new = start_old + n_new - 1 - r_new = start_new:end_new - - #= - Suppose we currently have the following: - - | x | x | o | o | o | y | y | y | <- Current entries - - where 'O' denotes an inactive entry, and we're going to - update the variable `x` to be of size `k` instead of 2. - - We then have a few different scenarios: - 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. - E.g. if `k = 7`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | x | x | y | y | y | <- New entries - - 2. `k = 5`: All inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | y | y | y | <- New entries - - 3. `k < 5`: Some inactive entries become active, some remain inactive. - E.g. if `k = 3`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | o | o | y | y | y | <- New entries - - 4. `k = 2`: No inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | o | o | o | y | y | y | <- New entries - - 5. `k < 2`: More entries become inactive. - E.g. if `k = 1`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | o | o | o | o | y | y | y | <- New entries - =# - - # Compute the allocated space for `vn`. - had_inactive = haskey(vnv.num_inactive, idx) - n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old - - if n_new > n_allocated - # Then we need to grow the underlying vector. - n_extra = n_new - n_allocated - # Allocate. - resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift current values. - shift_right!(vnv.vals, end_old + 1, n_extra) - # No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - # Update the ranges for all variables after this one. - shift_subsequent_ranges_by!(vnv, idx, n_extra) - elseif n_new == n_allocated - # => No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - else - # `n_new < n_allocated` - # => Need to update the number of inactive entries. - vnv.num_inactive[idx] = n_allocated - n_new - end - - # Update the range for this variable. - vnv.ranges[idx] = r_new - # Update the value. - vnv.vals[r_new] = val_vec - # Update the transform. - vnv.transforms[idx] = transform - - # TODO: Should we maybe sweep over inactive ranges and re-contiguify - # if we the total number of inactive elements is "large" in some sense? - - return nothing -end - -function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) - offset = 0 - for i in 1:length(ranges) - r_old = ranges[i] - ranges[i] = (offset + 1):(offset + length(r_old)) - offset += length(r_old) - end - - return ranges -end - -""" - contiguify!(vnv::VarNameVector) - -Re-contiguify the underlying vector and shrink if possible. -""" -function contiguify!(vnv::VarNameVector) - # Extract the re-contiguified values. - # NOTE: We need to do this before we update the ranges. - vals = vnv[:] - # And then we re-contiguify the ranges. - recontiguify_ranges!(vnv.ranges) - # Clear the inactive ranges. - empty!(vnv.num_inactive) - # Now we update the values. - for (i, r) in enumerate(vnv.ranges) - vnv.vals[r] = vals[r] - end - # And (potentially) shrink the underlying vector. - resize!(vnv.vals, vnv.ranges[end][end]) - # The rest should be left as is. - return vnv -end - -""" - group_by_symbol(vnv::VarNameVector) - -Return a dictionary mapping symbols to `VarNameVector`s with -varnames containing that symbol. -""" -function group_by_symbol(vnv::VarNameVector) - # Group varnames in `vnv` by the symbol. - d = OrderedDict{Symbol,Vector{VarName}}() - for vn in vnv.varnames - push!(get!(d, getsym(vn), Vector{VarName}()), vn) - end - - # Create a `NamedTuple` from the grouped varnames. - nt_vals = map(values(d)) do varnames - # TODO: Do we need to specialize the inputs here? - VarNameVector( - map(identity, varnames), - map(Base.Fix1(getindex, vnv), varnames), - map(Base.Fix1(gettransform, vnv), varnames), - ) - end - - return OrderedDict(zip(keys(d), nt_vals)) -end - -""" - shift_index_left!(vnv::VarNameVector, idx::Int) - -Shift the index `idx` to the left by one and update the relevant fields. - -!!! warning - This does not check if index we're shifting to is already occupied. -""" -function shift_index_left!(vnv::VarNameVector, idx::Int) - # Shift the index in the lookup table. - vn = vnv.varnames[idx] - vnv.varname_to_index[vn] = idx - 1 - # Shift the index in the inactive ranges. - if haskey(vnv.num_inactive, idx) - # Done in increasing order => don't need to worry about - # potentially shifting the same index twice. - vnv.num_inactive[idx - 1] = pop!(vnv.num_inactive, idx) - end -end - -""" - shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) - -Shift the indices for all variables after `idx` to the left by one and update -the relevant fields. - -This just -""" -function shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) - # Shift the indices for all variables after `idx`. - for idx_to_shift in (idx + 1):length(vnv.varnames) - shift_index_left!(vnv, idx_to_shift) - end -end - -function Base.delete!(vnv::VarNameVector, vn::VarName) - # Error if we don't have the variable. - !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) - - # Get the index of the variable. - idx = getidx(vnv, vn) - - # Delete the values. - r_start = first(getrange(vnv, idx)) - n_allocated = num_allocated(vnv, idx) - # NOTE: `deleteat!` also results in a `resize!` so we don't need to do that. - deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) - - # Delete `vn` from the lookup table. - delete!(vnv.varname_to_index, vn) - - # Delete any inactive ranges corresponding to `vn`. - haskey(vnv.num_inactive, idx) && delete!(vnv.num_inactive, idx) - - # Re-adjust the indices for varnames occuring after `vn` so - # that they point to the correct indices after the deletions below. - shift_subsequent_indices_left!(vnv, idx) - - # Re-adjust the ranges for varnames occuring after `vn`. - shift_subsequent_ranges_by!(vnv, idx, -n_allocated) - - # Delete references from vector fields, thus shifting the indices of - # varnames occuring after `vn` by one to the left, as we adjusted for above. - deleteat!(vnv.varnames, idx) - deleteat!(vnv.ranges, idx) - deleteat!(vnv.transforms, idx) - - return vnv -end - -""" - values_as(vnv::VarNameVector[, T]) - -Return the values/realizations in `vnv` as type `T`, if implemented. - -If no type `T` is provided, return values as stored in `vnv`. - -# Examples - -```jldoctest -julia> using DynamicPPL: VarNameVector - -julia> vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2.0]); - -julia> values_as(vnv) == [1.0, 2.0] -true - -julia> values_as(vnv, Vector{Float32}) == Vector{Float32}([1.0, 2.0]) -true - -julia> values_as(vnv, OrderedDict) == OrderedDict(@varname(x) => 1.0, @varname(y) => [2.0]) -true - -julia> values_as(vnv, NamedTuple) == (x = 1.0, y = [2.0]) -true -``` -""" -values_as(vnv::VarNameVector) = values_as(vnv, Vector) -values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] -function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} - return convert(Vector{T}, values_as(vnv, Vector)) -end -function values_as(vnv::VarNameVector, ::Type{NamedTuple}) - return NamedTuple(zip(map(Symbol, keys(vnv)), values(vnv))) -end -function values_as(vnv::VarNameVector, ::Type{D}) where {D<:AbstractDict} - return ConstructionBase.constructorof(D)(pairs(vnv)) -end diff --git a/test/runtests.jl b/test/runtests.jl index ef7672db8..d15b17b42 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -37,7 +37,6 @@ include("test_util.jl") @testset "interface" begin include("utils.jl") include("compiler.jl") - include("varnamevector.jl") include("varinfo.jl") include("simple_varinfo.jl") include("model.jl") diff --git a/test/test_util.jl b/test/test_util.jl index 563e13df1..5267d6d2b 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -85,7 +85,6 @@ Return string representing a short description of `vi`. short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" function short_varinfo_name(vi::TypedVarInfo) - DynamicPPL.has_varnamevector(vi) && return "TypedVarInfo with VarNameVector" return "TypedVarInfo" end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" diff --git a/test/varinfo.jl b/test/varinfo.jl index 570f9623e..4237b291a 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -405,12 +405,6 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end - if DynamicPPL.has_varnamevector(varinfo) && mutating - # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. - @test_broken false - continue - end - # Evaluate the model once to update the logp of the varinfo. varinfo = last(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())) diff --git a/test/varnamevector.jl b/test/varnamevector.jl deleted file mode 100644 index 6b7acfbeb..000000000 --- a/test/varnamevector.jl +++ /dev/null @@ -1,472 +0,0 @@ -replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) - -increase_size_for_test(x::Real) = [x] -increase_size_for_test(x::AbstractArray) = repeat(x, 2) - -decrease_size_for_test(x::Real) = x -decrease_size_for_test(x::AbstractVector) = first(x) -decrease_size_for_test(x::AbstractArray) = first(eachslice(x; dims=1)) - -function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) - if isconcretetype(eltype(vnv.varnames)) - # If the container is concrete, we need to make sure that the varname types match. - # E.g. if `vnv.varnames` has `eltype` `VarName{:x, IndexLens{Tuple{Int64}}}` then - # we need `vn` to also be of this type. - # => If the varname types don't match, we need to relax the container type. - return any(keys(vnv)) do vn_present - typeof(vn_present) !== typeof(val) - end - end - - return false -end -function need_varnames_relaxation(vnv::VarNameVector, vns, vals) - return any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) -end - -function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) - if isconcretetype(eltype(vnv.vals)) - return promote_type(eltype(vnv.vals), eltype(val)) != eltype(vnv.vals) - end - - return false -end -function need_values_relaxation(vnv::VarNameVector, vns, vals) - return any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) -end - -function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) - return if isconcretetype(eltype(vnv.transforms)) - # If the container is concrete, we need to make sure that the sizes match. - # => If the sizes don't match, we need to relax the container type. - any(keys(vnv)) do vn_present - size(vnv[vn_present]) != size(val) - end - elseif eltype(vnv.transforms) !== Any - # If it's not concrete AND it's not `Any`, then we should just make it `Any`. - true - else - # Otherwise, it's `Any`, so we don't need to relax the container type. - false - end -end -function need_transforms_relaxation(vnv::VarNameVector, vns, vals) - return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) -end - -""" - relax_container_types(vnv::VarNameVector, vn::VarName, val) - relax_container_types(vnv::VarNameVector, vns, val) - -Relax the container types of `vnv` if necessary to accommodate `vn` and `val`. - -This attempts to avoid unnecessary container type relaxations by checking whether -the container types of `vnv` are already compatible with `vn` and `val`. - -# Notes -For example, if `vn` is not compatible with the current keys in `vnv`, then -the underlying types will be changed to `VarName` to accommodate `vn`. - -Similarly: -- If `val` is not compatible with the current values in `vnv`, then - the underlying value type will be changed to `Real`. -- If `val` requires a transformation that is not compatible with the current - transformations type in `vnv`, then the underlying transformation type will - be changed to `Any`. -""" -function relax_container_types(vnv::VarNameVector, vn::VarName, val) - return relax_container_types(vnv, [vn], [val]) -end -function relax_container_types(vnv::VarNameVector, vns, vals) - if need_varnames_relaxation(vnv, vns, vals) - varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) - varnames_new = convert(Vector{VarName}, vnv.varnames) - else - varname_to_index_new = vnv.varname_to_index - varnames_new = vnv.varnames - end - - transforms_new = if need_transforms_relaxation(vnv, vns, vals) - convert(Vector{Any}, vnv.transforms) - else - vnv.transforms - end - - vals_new = if need_values_relaxation(vnv, vns, vals) - convert(Vector{Real}, vnv.vals) - else - vnv.vals - end - - return VarNameVector( - varname_to_index_new, - varnames_new, - vnv.ranges, - vals_new, - transforms_new, - vnv.is_transformed, - vnv.num_inactive, - vnv.metadata, - ) -end - -@testset "VarNameVector" begin - # Need to test element-related operations: - # - `getindex` - # - `setindex!` - # - `push!` - # - `update!` - # - # And these should all be tested for different types of values: - # - scalar - # - vector - # - matrix - - # Need to test operations on `VarNameVector`: - # - `empty!` - # - `iterate` - # - `convert` to - # - `AbstractDict` - test_pairs = OrderedDict( - @varname(x[1]) => rand(), - @varname(x[2]) => rand(2), - @varname(x[3]) => rand(2, 3), - @varname(y[1]) => rand(), - @varname(y[2]) => rand(2), - @varname(y[3]) => rand(2, 3), - @varname(z[1]) => rand(1:10), - @varname(z[2]) => rand(1:10, 2), - @varname(z[3]) => rand(1:10, 2, 3), - ) - test_vns = collect(keys(test_pairs)) - test_vals = collect(values(test_pairs)) - - @testset "constructor: no args" begin - # Empty. - vnv = VarNameVector() - @test isempty(vnv) - @test eltype(vnv) == Real - - # Empty with types. - vnv = VarNameVector{VarName,Float64}() - @test isempty(vnv) - @test eltype(vnv) == Float64 - end - - test_varnames_iter = combinations(test_vns, 2) - @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter - val_left = test_pairs[vn_left] - val_right = test_pairs[vn_right] - vnv_base = VarNameVector([vn_left, vn_right], [val_left, val_right]) - - # We'll need the transformations later. - # TODO: Should we test other transformations than just `FromVec`? - from_vec_left = DynamicPPL.from_vec_transform(val_left) - from_vec_right = DynamicPPL.from_vec_transform(val_right) - to_vec_left = inverse(from_vec_left) - to_vec_right = inverse(from_vec_right) - - # Compare to alternative constructors. - vnv_from_dict = VarNameVector( - OrderedDict(vn_left => val_left, vn_right => val_right) - ) - @test vnv_base == vnv_from_dict - - # We want the types of fields such as `varnames` and `transforms` to specialize - # whenever possible + some functionality, e.g. `push!`, is only sensible - # if the underlying containers can support it. - # Expected behavior - should_have_restricted_varname_type = typeof(vn_left) == typeof(vn_right) - should_have_restricted_transform_type = size(val_left) == size(val_right) - # Actual behavior - has_restricted_transform_type = isconcretetype(eltype(vnv_base.transforms)) - has_restricted_varname_type = isconcretetype(eltype(vnv_base.varnames)) - - @testset "type specialization" begin - @test !should_have_restricted_varname_type || has_restricted_varname_type - @test !should_have_restricted_transform_type || has_restricted_transform_type - end - - # `eltype` - @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) - # `length` - @test length(vnv_base) == length(val_left) + length(val_right) - - # `isempty` - @test !isempty(vnv_base) - - # `empty!` - @testset "empty!" begin - vnv = deepcopy(vnv_base) - empty!(vnv) - @test isempty(vnv) - end - - # `similar` - @testset "similar" begin - vnv = similar(vnv_base) - @test isempty(vnv) - @test typeof(vnv) == typeof(vnv_base) - end - - # `getindex` - @testset "getindex" begin - # With `VarName` index. - @test vnv_base[vn_left] == val_left - @test vnv_base[vn_right] == val_right - - # With `Int` index. - val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) - @test all(vnv_base[i] == val_vec[i] for i in 1:length(val_vec)) - end - - # `setindex!` - @testset "setindex!" begin - vnv = deepcopy(vnv_base) - vnv[vn_left] = val_left .+ 100 - @test vnv[vn_left] == val_left .+ 100 - vnv[vn_right] = val_right .+ 100 - @test vnv[vn_right] == val_right .+ 100 - end - - # `getindex_raw` - @testset "getindex_raw" begin - # With `VarName` index. - @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) - @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) - # With `Int` index. - val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) - @test all( - DynamicPPL.getindex_raw(vnv_base, i) == val_vec[i] for - i in 1:length(val_vec) - ) - end - - # `setindex_raw!` - @testset "setindex_raw!" begin - vnv = deepcopy(vnv_base) - DynamicPPL.setindex_raw!(vnv, to_vec_left(val_left .+ 100), vn_left) - @test vnv[vn_left] == val_left .+ 100 - DynamicPPL.setindex_raw!(vnv, to_vec_right(val_right .+ 100), vn_right) - @test vnv[vn_right] == val_right .+ 100 - end - - # `delete!` - @testset "delete!" begin - vnv = deepcopy(vnv_base) - delete!(vnv, vn_left) - @test !haskey(vnv, vn_left) - @test haskey(vnv, vn_right) - delete!(vnv, vn_right) - @test !haskey(vnv, vn_right) - end - - # `merge` - @testset "merge" begin - # When there are no inactive entries, `merge` on itself result in the same. - @test merge(vnv_base, vnv_base) == vnv_base - - # Merging with empty should result in the same. - @test merge(vnv_base, similar(vnv_base)) == vnv_base - @test merge(similar(vnv_base), vnv_base) == vnv_base - - # With differences. - vnv_left_only = deepcopy(vnv_base) - delete!(vnv_left_only, vn_right) - vnv_right_only = deepcopy(vnv_base) - delete!(vnv_right_only, vn_left) - - # `(x,)` and `(x, y)` should be `(x, y)`. - @test merge(vnv_left_only, vnv_base) == vnv_base - # `(x, y)` and `(x,)` should be `(x, y)`. - @test merge(vnv_base, vnv_left_only) == vnv_base - # `(x, y)` and `(y,)` should be `(x, y)`. - @test merge(vnv_base, vnv_right_only) == vnv_base - # `(y,)` and `(x, y)` should be `(y, x)`. - vnv_merged = merge(vnv_right_only, vnv_base) - @test vnv_merged != vnv_base - @test collect(keys(vnv_merged)) == [vn_right, vn_left] - end - - # `push!` & `update!` - @testset "push!" begin - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn" for vn in test_vns - val = test_pairs[vn] - if vn == vn_left || vn == vn_right - # Should not be possible to `push!` existing varname. - @test_throws ArgumentError push!(vnv, vn, val) - else - push!(vnv, vn, val) - @test vnv[vn] == val - end - end - end - @testset "update!" begin - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn" for vn in test_vns - val = test_pairs[vn] - expected_length = if haskey(vnv, vn) - # If it's already present, the resulting length will be unchanged. - length(vnv) - else - length(vnv) + length(val) - end - - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] - @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) - - # There should be no redundant values in the underlying vector. - @test !DynamicPPL.has_inactive(vnv) - - # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) - end - - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn (increased size)" for vn in test_vns - val_original = test_pairs[vn] - val = increase_size_for_test(val_original) - vn_already_present = haskey(vnv, vn) - expected_length = if vn_already_present - # If it's already present, the resulting length will be altered. - length(vnv) + length(val) - length(val_original) - else - length(vnv) + length(val) - end - - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] - @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) - - # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) - end - - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn (decreased size)" for vn in test_vns - val_original = test_pairs[vn] - val = decrease_size_for_test(val_original) - vn_already_present = haskey(vnv, vn) - expected_length = if vn_already_present - # If it's already present, the resulting length will be altered. - length(vnv) + length(val) - length(val_original) - else - length(vnv) + length(val) - end - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] - @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) - - # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) - end - end - end - - @testset "growing and shrinking" begin - @testset "deterministic" begin - n = 5 - vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) - @test !DynamicPPL.has_inactive(vnv) - # Growing should not create inactive ranges. - for i in 1:n - x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive(vnv) - end - - # Same size should not create inactive ranges. - x = fill(true, n) - DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive(vnv) - - # Shrinking should create inactive ranges. - for i in (n - 1):-1:1 - x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) - @test DynamicPPL.has_inactive(vnv) - @test DynamicPPL.num_inactive(vnv, vn) == n - i - end - end - - @testset "random" begin - n = 5 - vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) - @test !DynamicPPL.has_inactive(vnv) - - # Insert a bunch of random-length vectors. - for i in 1:100 - x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, vn, x) - end - # Should never be allocating more than `n` elements. - @test DynamicPPL.num_allocated(vnv, vn) ≤ n - - # If we compaticfy, then it should always be the same size as just inserted. - for i in 1:10 - x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, vn, x) - DynamicPPL.contiguify!(vnv) - @test DynamicPPL.num_allocated(vnv, vn) == length(x) - end - end - end -end - -@testset "VarInfo + VarNameVector" begin - models = DynamicPPL.TestUtils.DEMO_MODELS - @testset "$(model.f)" for model in models - # NOTE: Need to set random seed explicitly to avoid using the same seed - # for initialization as for sampling in the inner testset below. - Random.seed!(42) - value_true = DynamicPPL.TestUtils.rand_prior_true(model) - vns = DynamicPPL.TestUtils.varnames(model) - varnames = DynamicPPL.TestUtils.varnames(model) - varinfos = DynamicPPL.TestUtils.setup_varinfos( - model, value_true, varnames; include_threadsafe=false - ) - # Filter out those which are not based on `VarNameVector`. - varinfos = filter(DynamicPPL.has_varnamevector, varinfos) - # Get the true log joint. - logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) - - @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos - # Need to make sure we're using a different random seed from the - # one used in the above call to `rand_prior_true`. - Random.seed!(43) - - # Are values correct? - DynamicPPL.TestUtils.test_values(varinfo, value_true, vns) - - # Is evaluation correct? - varinfo_eval = last( - DynamicPPL.evaluate!!(model, deepcopy(varinfo), DefaultContext()) - ) - # Log density should be the same. - @test getlogp(varinfo_eval) ≈ logp_true - # Values should be the same. - DynamicPPL.TestUtils.test_values(varinfo_eval, value_true, vns) - - # Is sampling correct? - varinfo_sample = last( - DynamicPPL.evaluate!!(model, deepcopy(varinfo), SamplingContext()) - ) - # Log density should be different. - @test getlogp(varinfo_sample) != getlogp(varinfo) - # Values should be different. - DynamicPPL.TestUtils.test_values( - varinfo_sample, value_true, vns; compare=!isequal - ) - end - end -end From 8930f9c2ea69a4ce106df2263bf3264315c3e5e7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:11:07 +0000 Subject: [PATCH 135/182] formatting --- docs/src/internals/transformations.md | 88 +++++++++++++-------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 45134a593..8173a4abc 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -9,7 +9,7 @@ For example, consider the following model: ```julia @model function demo() s ~ InverseGamma(2, 3) - m ~ Normal(0, √s) + return m ~ Normal(0, √s) end ``` @@ -18,6 +18,7 @@ Here we have two variables `s` and `m`, where `s` is constrained to be positive, For certain inference methods, it's necessary / much more convenient to work with an equivalent model to `demo` but where all the variables can take any real values (they're "unconstrained"). !!! note + We write "unconstrained" with quotes because there are many ways to transform a constrained variable to an unconstrained one, *and* DynamicPPL can work with a much broader class of bijective transformations of variables, not just ones that go to the entire real line. But for MCMC, unconstraining is the most common transformation so we'll stick with that terminology. For a large family of constraints encoucntered in practice, it is indeed possible to transform a (partially) contrained model to a completely unconstrained one in such a way that sampling in the unconstrained space is equivalent to sampling in the constrained space. @@ -30,7 +31,7 @@ For example, the above model could be transformed into (the following psuedo-cod @model function demo() log_s ~ log(InverseGamma(2, 3)) s = exp(log_s) - m ~ Normal(0, √s) + return m ~ Normal(0, √s) end ``` @@ -51,12 +52,12 @@ Below we'll see how this is done. ## What do we need? There are two aspects to transforming from the internal representation of a variable in a `varinfo` to the representation wanted in the model: -1. Different implementations of [`AbstractVarInfo`](@ref) represent realizations of a model in different ways internally, so we need to transform from this internal representation to the desired representation in the model. For example, - - [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. - - [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). -2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. - + 1. Different implementations of [`AbstractVarInfo`](@ref) represent realizations of a model in different ways internally, so we need to transform from this internal representation to the desired representation in the model. For example, + + + [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. + + [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). + 2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. ## Working example @@ -70,11 +71,13 @@ using DynamicPPL, Distributions `LKJCholesky` is a `LKJ(2, 1.0)` distribution, a distribution over correlation matrices (covariance matrices but with unit diagonal), but working directly with the Cholesky factorization of the correlation matrix rather than the correlation matrix itself (this is more numerically stable and computationally efficient). !!! note + This is a particularly "annoying" case because the return-value is not a simple `Real` or `AbstractArray{<:Real}`, but rather a `LineraAlgebra.Cholesky` object which wraps a triangular matrix (whether it's upper- or lower-triangular depends on the instance). As mentioned, some implementations of `AbstractVarInfo`, e.g. [`VarInfo`](@ref), works with a "flattened" / vector representation of a variable, and so in this case we need two transformations: -1. From the `Cholesky` object to a vector representation. -2. From the `Cholesky` object to an "unconstrained" / linked vector representation. + + 1. From the `Cholesky` object to a vector representation. + 2. From the `Cholesky` object to an "unconstrained" / linked vector representation. And similarly, we'll need the inverses of these transformations. @@ -88,10 +91,12 @@ DynamicPPL.from_internal_transform ``` These methods allows us to extract the internal-to-model transformation function depending on the `varinfo`, the variable, and the distribution of the variable: -- `varinfo` + `vn` defines the internal representation of the variable. -- `dist` defines the representation expected within the model scope. + + - `varinfo` + `vn` defines the internal representation of the variable. + - `dist` defines the representation expected within the model scope. !!! note + If `vn` is not present in `varinfo`, then the internal representation is fully determined by `varinfo` alone. This is used when we're about to add a new variable to the `varinfo` and need to know how to represent it internally. Continuing from the example above, we can inspect the internal representation of `x` in `demo_lkj` with [`VarInfo`](@ref) using [`DynamicPPL.getindex_internal`](@ref): @@ -104,9 +109,7 @@ x_internal = DynamicPPL.getindex_internal(varinfo, @varname(x)) ```@example transformations-internal f_from_internal = DynamicPPL.from_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) + varinfo, @varname(x), LKJCholesky(2, 1.0) ) f_from_internal(x_internal) ``` @@ -120,11 +123,7 @@ x_model = varinfo[@varname(x)] Similarly, we can go from the model representation to the internal representation: ```@example transformations-internal -f_to_internal = DynamicPPL.to_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) -) +f_to_internal = DynamicPPL.to_internal_transform(varinfo, @varname(x), LKJCholesky(2, 1.0)) f_to_internal(x_model) ``` @@ -139,11 +138,7 @@ DynamicPPL.getindex_internal(simple_varinfo, @varname(x)) Here see that the internal representation is exactly the same as the model representation, and so we'd expect `from_internal_transform` to be the `identity` function: ```@example transformations-internal -DynamicPPL.from_internal_transform( - simple_varinfo, - @varname(x), - LKJCholesky(2, 1.0) -) +DynamicPPL.from_internal_transform(simple_varinfo, @varname(x), LKJCholesky(2, 1.0)) ``` Great! @@ -165,9 +160,7 @@ Continuing from the example above: ```@example transformations-internal f_to_linked_internal = DynamicPPL.to_linked_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) + varinfo, @varname(x), LKJCholesky(2, 1.0) ) x_linked_internal = f_to_linked_internal(x_model) @@ -175,9 +168,7 @@ x_linked_internal = f_to_linked_internal(x_model) ```@example transformations-internal f_from_linked_internal = DynamicPPL.from_linked_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) + varinfo, @varname(x), LKJCholesky(2, 1.0) ) f_from_linked_internal(x_linked_internal) @@ -216,7 +207,7 @@ Unfortunately, this is not possible in general. Consider for example the followi ```@example transformations-internal @model function demo_dynamic_constraint() m ~ Normal() - x ~ truncated(Normal(), lower=m) + x ~ truncated(Normal(); lower=m) return (m=m, x=x) end @@ -266,6 +257,7 @@ first(DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext())) we see that we indeed satisfy the constraint `m < x`, as desired. !!! warning + One shouldn't be setting variables in a linked `varinfo` willy-nilly directly like this unless one knows that the value will be compatible with the constraints of the model. The reason for this is that internally in a model evaluation, we construct the transformation from the internal to the model representation based on the *current* realizations in the model! That is, we take the `dist` in a `x ~ dist` expression _at model evaluation time_ and use that to construct the transformation, thus allowing it to change between model evaluations without invalidating the transformation. @@ -291,11 +283,13 @@ And so the earlier diagram becomes: ``` !!! note - If the support of `dist` was constant, this would not be necessary since we could just determine the transformation at the time of `varinfo_linked = link(varinfo, model)` and define this as the `from_internal_transform` for all subsequent evaluations. However, since the support of `dist` is *not* constant in general, we need to be able to determine the transformation at the time of the evaluation *and* thus whether we should construct the transformation from the linked internal representation or the non-linked internal representation. This is annoying, but necessary. + If the support of `dist` was constant, this would not be necessary since we could just determine the transformation at the time of `varinfo_linked = link(varinfo, model)` and define this as the `from_internal_transform` for all subsequent evaluations. However, since the support of `dist` is *not* constant in general, we need to be able to determine the transformation at the time of the evaluation *and* thus whether we should construct the transformation from the linked internal representation or the non-linked internal representation. This is annoying, but necessary. + This is also the reason why we have two definitions of `getindex`: -- [`getindex(::AbstractVarInfo, ::VarName, ::Distribution)`](@ref): used internally in model evaluations with the `dist` in a `x ~ dist` expression. -- [`getindex(::AbstractVarInfo, ::VarName)`](@ref): used externally by the user to get the realization of a variable. + + - [`getindex(::AbstractVarInfo, ::VarName, ::Distribution)`](@ref): used internally in model evaluations with the `dist` in a `x ~ dist` expression. + - [`getindex(::AbstractVarInfo, ::VarName)`](@ref): used externally by the user to get the realization of a variable. For `getindex` we have the following diagram: @@ -316,8 +310,9 @@ While if `dist` is not provided, we have: Notice that `dist` is not present here, but otherwise the diagrams are the same. !!! warning - This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + ## Other functionalities There are also some additional methods for transforming between representations that are all automatically implemented from [`DynamicPPL.from_internal_transform`](@ref), [`DynamicPPL.from_linked_internal_transform`](@ref) and their siblings, and thus don't need to be implemented manually. @@ -341,8 +336,9 @@ DynamicPPL.from_maybe_linked_internal # Supporting a new distribution To support a new distribution, one needs to implement for the desired `AbstractVarInfo` the following methods: -- [`DynamicPPL.from_internal_transform`](@ref) -- [`DynamicPPL.from_linked_internal_transform`](@ref) + + - [`DynamicPPL.from_internal_transform`](@ref) + - [`DynamicPPL.from_linked_internal_transform`](@ref) At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implemente the following methods instead: @@ -354,8 +350,9 @@ DynamicPPL.from_linked_vec_transform(::Distribution) These are used internally by [`VarInfo`](@ref). Optionally, if `inverse` of the above is expensive to compute, one can also implement: -- [`DynamicPPL.to_internal_transform`](@ref) -- [`DynamicPPL.to_linked_internal_transform`](@ref) + + - [`DynamicPPL.to_internal_transform`](@ref) + - [`DynamicPPL.to_linked_internal_transform`](@ref) And similarly, there are corresponding to-methods for the `from_*_vec_transform` variants too @@ -365,13 +362,14 @@ DynamicPPL.to_linked_vec_transform ``` !!! warning + Whatever the resulting transformation is, it should be invertible, i.e. implement `InverseFunctions.inverse`, and have a well-defined log-abs-det Jacobian, i.e. implement `ChangesOfVariables.with_logabsdet_jacobian`. # TL;DR -- DynamicPPL.jl has three representations of a variable: the **model representation**, the **internal representation**, and the **linked internal representation**. - - The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. - - The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). - - The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). -- Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. - + - DynamicPPL.jl has three representations of a variable: the **model representation**, the **internal representation**, and the **linked internal representation**. + + + The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. + + The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). + + The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). + - Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. From e45b6681d28be4c866d94ccd25dcff9ddb7d281c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:14:49 +0000 Subject: [PATCH 136/182] Update docs/src/internals/transformations.md Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/src/internals/transformations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 8173a4abc..ee2c652bf 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -57,6 +57,7 @@ There are two aspects to transforming from the internal representation of a vari + [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. + [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). + 2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. ## Working example From 0e71092394ac0c6d9a228e6dd108b72812854928 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:14:55 +0000 Subject: [PATCH 137/182] Update docs/src/internals/transformations.md Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/src/internals/transformations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index ee2c652bf..19fe9c1e0 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -373,4 +373,5 @@ DynamicPPL.to_linked_vec_transform + The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. + The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). + The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). + - Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. From 2de9ac98eb92f7bb14354e7fb9617e33b2c64ad2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:18:34 +0000 Subject: [PATCH 138/182] removed refs to VectorVarInfo --- src/test_utils.jl | 2 -- src/varinfo.jl | 37 ------------------------------------- test/test_util.jl | 1 - 3 files changed, 40 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index e79aad9d1..ba40d10bd 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -52,8 +52,6 @@ function setup_varinfos( vi_untyped = VarInfo() model(vi_untyped) vi_typed = DynamicPPL.TypedVarInfo(vi_untyped) - vi_vnv = DynamicPPL.VectorVarInfo(vi_untyped) - vi_vnv_typed = DynamicPPL.VectorVarInfo(vi_typed) # SimpleVarInfo svi_typed = SimpleVarInfo(example_values) svi_untyped = SimpleVarInfo(OrderedDict()) diff --git a/src/varinfo.jl b/src/varinfo.jl index eda8213b1..725dbdc86 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -118,19 +118,6 @@ function VarInfo(old_vi::UntypedVarInfo, spl, x::AbstractVector) return new_vi end -function VarInfo(old_vi::VectorVarInfo, spl, x::AbstractVector) - new_vi = deepcopy(old_vi) - new_vi[spl] = x - return new_vi -end - -function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) - md = newmetadata(old_vi.metadata, Val(getspace(spl)), x) - return VarInfo( - md, Base.RefValue{eltype(x)}(getlogp(old_vi)), Ref(get_num_produce(old_vi)) - ) -end - function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -252,11 +239,6 @@ function subset(varinfo::UntypedVarInfo, vns::AbstractVector{<:VarName}) return VarInfo(metadata, varinfo.logp, varinfo.num_produce) end -function subset(varinfo::VectorVarInfo, vns::AbstractVector{<:VarName}) - metadata = subset(varinfo.metadata, vns) - return VarInfo(metadata, varinfo.logp, varinfo.num_produce) -end - function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName{sym}}) where {sym} # If all the variables are using the same symbol, then we can just extract that field from the metadata. metadata = subset(getfield(varinfo.metadata, sym), vns) @@ -875,12 +857,6 @@ function TypedVarInfo(vi::UntypedVarInfo) return VarInfo(nt, Ref(logp), Ref(num_produce)) end TypedVarInfo(vi::TypedVarInfo) = vi -function TypedVarInfo(vi::VectorVarInfo) - logp = getlogp(vi) - num_produce = get_num_produce(vi) - nt = NamedTuple(group_by_symbol(vi.metadata)) - return VarInfo(nt, Ref(logp), Ref(num_produce)) -end function BangBang.empty!!(vi::VarInfo) _empty!(vi.metadata) @@ -1227,15 +1203,6 @@ function _link(model::Model, varinfo::UntypedVarInfo, spl::AbstractSampler) ) end -function _link(model::Model, varinfo::VectorVarInfo, spl::AbstractSampler) - varinfo = deepcopy(varinfo) - return VarInfo( - _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), - Base.Ref(getlogp(varinfo)), - Ref(get_num_produce(varinfo)), - ) -end - function _link(model::Model, varinfo::TypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) md = _link_metadata_namedtuple!( @@ -2067,10 +2034,6 @@ end function values_as(vi::UntypedVarInfo, ::Type{D}) where {D<:AbstractDict} return ConstructionBase.constructorof(D)(values_from_metadata(vi.metadata)) end -values_as(vi::VectorVarInfo, ::Type{NamedTuple}) = values_as(vi.metadata, NamedTuple) -function values_as(vi::VectorVarInfo, ::Type{D}) where {D<:AbstractDict} - return values_as(vi.metadata, D) -end function values_as(vi::VarInfo{<:NamedTuple{names}}, ::Type{NamedTuple}) where {names} iter = Iterators.flatten(values_from_metadata(getfield(vi.metadata, n)) for n in names) diff --git a/test/test_util.jl b/test/test_util.jl index 5267d6d2b..1d23567da 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -88,7 +88,6 @@ function short_varinfo_name(vi::TypedVarInfo) return "TypedVarInfo" end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" -short_varinfo_name(::DynamicPPL.VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" From 9b7142850dedab31e25e66216e36c3239a3d7607 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:18:52 +0000 Subject: [PATCH 139/182] added impls of `from_internal_transform` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index dd0357547..e3095b791 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -243,12 +243,16 @@ function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) return invlink_with_logpdf(vi.varinfo, vn, dist, y) end -function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) +function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) + return from_linked_internal_transform(varinfo.varinfo, vn) +end +function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) return from_linked_internal_transform(varinfo.varinfo, vn, dist) end -function from_linked_internal_transform( - varinfo::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist -) - return from_linked_internal_transform(varinfo.varinfo, vns, dist) +function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) + return from_linked_internal_transform(varinfo.varinfo, vn) +end +function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) + return from_linked_internal_transform(varinfo.varinfo, vn, dist) end From 786e9bf8f32c777cb66bd83e5a7243d2ef5ae791 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:21:58 +0000 Subject: [PATCH 140/182] reverted accidental removal of old `VarInfo` constructor --- src/varinfo.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index 725dbdc86..0d0bb1bad 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -118,6 +118,13 @@ function VarInfo(old_vi::UntypedVarInfo, spl, x::AbstractVector) return new_vi end +function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) + md = newmetadata(old_vi.metadata, Val(getspace(spl)), x) + return VarInfo( + md, Base.RefValue{eltype(x)}(getlogp(old_vi)), Ref(get_num_produce(old_vi)) + ) +end + function untyped_varinfo( rng::Random.AbstractRNG, model::Model, From f1fe42cc23b8f20aba6f7ccac80a2d77399a9afd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:26:18 +0000 Subject: [PATCH 141/182] fixed incorrect `recombine` call --- src/varinfo.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 0d0bb1bad..c7bec1829 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1458,7 +1458,9 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}) vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn) end - return recombine(vi, vals_linked, length(vns)) + # HACK: I don't like this. + dist = getdist(vi, vns[1]) + return recombine(dist, vals_linked, length(vns)) end function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) @assert haskey(vi, vns[1]) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" From 2273954f12fb37f7fbad45903a1bb0927f4825e1 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:32:04 +0000 Subject: [PATCH 142/182] removed undefined refs to `VarNameVector` stuff in `setup_varinfos` --- src/test_utils.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index ba40d10bd..172d68d4e 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -62,14 +62,7 @@ function setup_varinfos( lp = getlogp(vi_typed) varinfos = map(( - vi_untyped, - vi_typed, - vi_vnv, - vi_vnv_typed, - svi_typed, - svi_untyped, - svi_typed_ref, - svi_untyped_ref, + vi_untyped, vi_typed, svi_typed, svi_untyped, svi_typed_ref, svi_untyped_ref )) do vi # Set them all to the same values. DynamicPPL.setlogp!!(update_values!!(vi, example_values, varnames), lp) From ab7c18984bbf9d3ea32f3b9dd676853d85928291 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:40:13 +0000 Subject: [PATCH 143/182] bump minior version because Turing breaks --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c6d662c44..218a1af72 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DynamicPPL" uuid = "366bfd00-2699-11ea-058f-f148b4cae6d8" -version = "0.24.5" +version = "0.25.0" [deps] AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" From 3a86601f199f844161a3975d5a883a52f926b4b4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:24:57 +0000 Subject: [PATCH 144/182] fix: was using `from_linked_internal_transform` in `from_internal_transform` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index e3095b791..8769c7008 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -244,10 +244,10 @@ function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) end function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) - return from_linked_internal_transform(varinfo.varinfo, vn) + return from_internal_transform(varinfo.varinfo, vn) end function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) - return from_linked_internal_transform(varinfo.varinfo, vn, dist) + return from_internal_transform(varinfo.varinfo, vn, dist) end function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) From 28c7d85ea829feeb034d185ea86cebbaf849ff6f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:48:09 +0000 Subject: [PATCH 145/182] removed `getindex_raw` --- docs/src/api.md | 1 - src/abstract_varinfo.jl | 20 +++++--------------- src/simple_varinfo.jl | 26 +++++--------------------- src/threadsafe.jl | 15 --------------- src/varinfo.jl | 32 ++++---------------------------- test/simple_varinfo.jl | 21 +++++++++++---------- test/varinfo.jl | 12 ++++++++---- 7 files changed, 33 insertions(+), 94 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index e07d762da..9d1d1501f 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -269,7 +269,6 @@ resetlogp!! ```@docs keys getindex -DynamicPPL.getindex_raw DynamicPPL.getindex_internal push!! empty!! diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index c048014b5..eb3c222b3 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -144,9 +144,7 @@ Return an iterator over all `vns` in `vi`. Return the current value(s) of `vn` (`vns`) in `vi` in the support of its (their) distribution(s). -If `dist` is specified, the value(s) will be reshaped accordingly. - -See also: [`getindex_raw(vi::AbstractVarInfo, vn::VarName, dist::Distribution)`](@ref) +If `dist` is specified, the value(s) will be massaged into the representation expected by `dist`. """ Base.getindex """ @@ -164,22 +162,14 @@ Base.getindex(vi::AbstractVarInfo, ::Colon) = values_as(vi, Vector) Base.getindex(vi::AbstractVarInfo, ::AbstractSampler) = vi[:] """ - getindex_raw(vi::AbstractVarInfo, vn::VarName[, dist::Distribution]) - getindex_raw(vi::AbstractVarInfo, vns::Vector{<:VarName}[, dist::Distribution]) - -Return the current value(s) of `vn` (`vns`) in `vi`. + getindex_internal(vi::AbstractVarInfo, vn::VarName) + getindex_internal(vi::AbstractVarInfo, vns::Vector{<:VarName}) -If `dist` is specified, the value(s) will be reshaped accordingly. +Return the current value(s) of `vn` (`vns`) in `vi` as represented internally in `vi`. See also: [`getindex(vi::AbstractVarInfo, vn::VarName, dist::Distribution)`](@ref) - -!!! note - The difference between `getindex(vi, vn, dist)` and `getindex_raw` is that - `getindex` will also transform the value(s) to the support of the distribution(s). - This is _not_ the case for `getindex_raw`. - """ -function getindex_raw end +function getindex_internal end """ push!!(vi::AbstractVarInfo, vn::VarName, r, dist::Distribution) diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index dc2780aae..a0d9e5925 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -290,7 +290,6 @@ function Base.show(io::IO, ::MIME"text/plain", svi::SimpleVarInfo) return print(io, "SimpleVarInfo(", svi.values, ", ", svi.logp, ")") end -# `NamedTuple` function Base.getindex(vi::SimpleVarInfo, vn::VarName, dist::Distribution) return from_maybe_linked_internal(vi, vn, dist, getindex(vi, vn)) end @@ -301,12 +300,7 @@ function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribu return recombine(dist, vals_linked, length(vns)) end -Base.getindex(vi::SimpleVarInfo, vn::VarName) = get(vi.values, vn) - -# `AbstractDict` -function Base.getindex(vi::SimpleVarInfo{<:AbstractDict}, vn::VarName) - return nested_getindex(vi.values, vn) -end +Base.getindex(vi::SimpleVarInfo, vn::VarName) = getindex_internal(vi, vn) # `SimpleVarInfo` doesn't necessarily vectorize, so we can have arrays other than # just `Vector`. @@ -318,22 +312,12 @@ Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}) = map(Base.Fix1(getinde Base.getindex(svi::SimpleVarInfo, ::Colon) = values_as(svi, Vector) -# Since we don't perform any transformations in `getindex` for `SimpleVarInfo` -# we simply call `getindex` in `getindex_raw`. -getindex_raw(vi::SimpleVarInfo, vn::VarName) = vi[vn] -function getindex_raw(vi::SimpleVarInfo, vn::VarName, dist::Distribution) - f = from_internal_transform(vi, vn, dist) - return f(getindex_raw(vi, vn)) -end -getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}) = vi[vns] -function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) - vals = mapreduce(Base.Fix1(getindex_raw, vi), vcat, vns) - return recombine(dist, vals, length(vns)) +getindex_internal(vi::SimpleVarInfo, vn::VarName) = get(vi.values, vn) +# `AbstractDict` +function getindex_internal(vi::SimpleVarInfo{<:AbstractDict}, vn::VarName) + return nested_getindex(vi.values, vn) end -# HACK: because `VarInfo` isn't ready to implement a proper `getindex_raw`. -getindex_internal(vi::SimpleVarInfo, vn::VarName) = getindex_raw(vi, vn) - Base.haskey(vi::SimpleVarInfo, vn::VarName) = hasvalue(vi.values, vn) function BangBang.setindex!!(vi::SimpleVarInfo, val, vn::VarName) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 8769c7008..eb2d0246c 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -159,21 +159,6 @@ function getindex(vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist::D end getindex(vi::ThreadSafeVarInfo, spl::AbstractSampler) = getindex(vi.varinfo, spl) -getindex_raw(vi::ThreadSafeVarInfo, ::Colon) = getindex_raw(vi.varinfo, Colon()) -getindex_raw(vi::ThreadSafeVarInfo, vn::VarName) = getindex_raw(vi.varinfo, vn) -function getindex_raw(vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}) - return getindex_raw(vi.varinfo, vns) -end -function getindex_raw(vi::ThreadSafeVarInfo, vn::VarName, dist::Distribution) - return getindex_raw(vi.varinfo, vn, dist) -end -function getindex_raw( - vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist::Distribution -) - return getindex_raw(vi.varinfo, vns, dist) -end -getindex_raw(vi::ThreadSafeVarInfo, spl::AbstractSampler) = getindex_raw(vi.varinfo, spl) - function BangBang.setindex!!(vi::ThreadSafeVarInfo, val, spl::AbstractSampler) return Setfield.@set vi.varinfo = BangBang.setindex!!(vi.varinfo, val, spl) end diff --git a/src/varinfo.jl b/src/varinfo.jl index c7bec1829..4e6779463 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -456,11 +456,6 @@ end const VarView = Union{Int,UnitRange,Vector{Int}} -""" - getindex_internal(vi::UntypedVarInfo, vview::Union{Int, UnitRange, Vector{Int}}) - -Return a view `vi.vals[vview]`. -""" getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) """ @@ -535,6 +530,10 @@ The values may or may not be transformed to Euclidean space. getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) + return mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) +end + """ setval!(vi::VarInfo, val, vn::VarName) @@ -550,16 +549,6 @@ function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) end -""" - getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) - -Return the value(s) of `vns`. - -The values may or may not be transformed to Euclidean space. -""" -getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) = - mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) - """ getall(vi::VarInfo) @@ -1471,19 +1460,6 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) return recombine(dist, vals_linked, length(vns)) end -getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) -function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) - f = from_internal_transform(vi, vn, dist) - return f(getindex_internal(vi, vn)) -end -function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) - return getindex_raw(vi, vns, getdist(vi, first(vns))) -end -function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) - # TODO: Replace when we have better dispatch for multiple vals. - return recombine(dist, getindex_internal(vi, vns), length(vns)) -end - """ getindex(vi::VarInfo, spl::Union{SampleFromPrior, Sampler}) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 223d0dd49..e2d87cbac 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -100,7 +100,8 @@ # Should result in same values. @test all( - DynamicPPL.getindex_raw(vi_invlinked, vn) ≈ get(values_constrained, vn) for + DynamicPPL.getindex_internal(vi_invlinked, vn) ≈ + vec(get(values_constrained, vn)) for vn in DynamicPPL.TestUtils.varnames(model) ) end @@ -252,11 +253,9 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≠ - retval.s # `s` is unconstrained in original - @test DynamicPPL.getindex_raw( - vi_linked_result, @varname(s), priors[@varname(s)] - ) == retval.s # `s` is constrained in result + @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ vec(retval.s) # `s` is unconstrained in original + @test DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) == + vec(retval.s) # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -267,10 +266,12 @@ model, retval.s, retval.m ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≈ - retval_unconstrained.s - @test DynamicPPL.getindex_raw(vi_linked, @varname(m), priors[@varname(m)]) ≈ - retval_unconstrained.m + @test DynamicPPL.getindex_internal( + vi_linked, @varname(s), priors[@varname(s)] + ) ≈ vec(retval_unconstrained.s) + @test DynamicPPL.getindex_internal( + vi_linked, @varname(m), priors[@varname(m)] + ) ≈ vec(retval_unconstrained.m) # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) diff --git a/test/varinfo.jl b/test/varinfo.jl index 4237b291a..5c23f75a6 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -309,7 +309,8 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) vi = DynamicPPL.settrans!!(vi, true, vn) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) ## `TypedVarInfo` @@ -317,7 +318,8 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) vi = DynamicPPL.settrans!!(vi, true, vn) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) ### `SimpleVarInfo` @@ -325,14 +327,16 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) vi = DynamicPPL.settrans!!(SimpleVarInfo(), true) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) ## `SimpleVarInfo{<:Dict}` vi = DynamicPPL.settrans!!(SimpleVarInfo(Dict()), true) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) end From 59514d6c5378a2402a562e41a575e8ee5a950949 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:48:19 +0000 Subject: [PATCH 146/182] removed redundant docstrings --- src/varinfo.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 4e6779463..369c68ca9 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -520,13 +520,6 @@ Return the distribution from which `vn` was sampled in `vi`. getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] -""" - getindex_internal(vi::VarInfo, vn::VarName) - -Return the value(s) of `vn`. - -The values may or may not be transformed to Euclidean space. -""" getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) From cdc882b086f663b237a8b539b89328e5327ff6b3 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:49:25 +0000 Subject: [PATCH 147/182] fixed tests --- test/simple_varinfo.jl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index e2d87cbac..98fe0d8a9 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -101,7 +101,7 @@ # Should result in same values. @test all( DynamicPPL.getindex_internal(vi_invlinked, vn) ≈ - vec(get(values_constrained, vn)) for + DynamicPPLL.tovec(get(values_constrained, vn)) for vn in DynamicPPL.TestUtils.varnames(model) ) end @@ -253,9 +253,10 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ vec(retval.s) # `s` is unconstrained in original + @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ + DynamicPPL.tovec(retval.s) # `s` is unconstrained in original @test DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) == - vec(retval.s) # `s` is constrained in result + DynamicPPL.tovec(retval.s) # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -268,10 +269,10 @@ @test DynamicPPL.getindex_internal( vi_linked, @varname(s), priors[@varname(s)] - ) ≈ vec(retval_unconstrained.s) + ) ≈ DynamicPPL.tovec(retval_unconstrained.s) @test DynamicPPL.getindex_internal( vi_linked, @varname(m), priors[@varname(m)] - ) ≈ vec(retval_unconstrained.m) + ) ≈ DynamicPPL.tovec(retval_unconstrained.m) # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) From 57ba7c0245999c9f055f95af577bfc73fd10aa75 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 11:16:18 +0000 Subject: [PATCH 148/182] fixed comparisons in tests --- test/simple_varinfo.jl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 98fe0d8a9..74dcdd842 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -100,8 +100,8 @@ # Should result in same values. @test all( - DynamicPPL.getindex_internal(vi_invlinked, vn) ≈ - DynamicPPLL.tovec(get(values_constrained, vn)) for + DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_invlinked, vn)) ≈ + DynamicPPL.tovec(get(values_constrained, vn)) for vn in DynamicPPL.TestUtils.varnames(model) ) end @@ -253,10 +253,11 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ + @test DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_linked, @varname(s))) ≠ DynamicPPL.tovec(retval.s) # `s` is unconstrained in original - @test DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) == - DynamicPPL.tovec(retval.s) # `s` is constrained in result + @test DynamicPPL.tovec( + DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) + ) == DynamicPPL.tovec(retval.s) # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -267,12 +268,10 @@ model, retval.s, retval.m ) - @test DynamicPPL.getindex_internal( - vi_linked, @varname(s), priors[@varname(s)] - ) ≈ DynamicPPL.tovec(retval_unconstrained.s) - @test DynamicPPL.getindex_internal( - vi_linked, @varname(m), priors[@varname(m)] - ) ≈ DynamicPPL.tovec(retval_unconstrained.m) + @test DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_linked, @varname(s))) ≈ + DynamicPPL.tovec(retval_unconstrained.s) + @test DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_linked, @varname(m))) ≈ + DynamicPPL.tovec(retval_unconstrained.m) # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) From 902e59cb65599b67f8cf7c8b320c8df68eb4cbf4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 11:22:49 +0000 Subject: [PATCH 149/182] try relative references for images in transformation docs --- docs/src/internals/transformations.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 19fe9c1e0..6fb18a392 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -43,7 +43,7 @@ In the end, we'll end up with something that looks like this: ```@raw html

- +
``` @@ -199,7 +199,7 @@ That is, why can't we just do ```@raw html
- +
``` @@ -279,7 +279,7 @@ And so the earlier diagram becomes: ```@raw html
- +
``` @@ -296,7 +296,7 @@ For `getindex` we have the following diagram: ```@raw html
- +
``` @@ -304,7 +304,7 @@ While if `dist` is not provided, we have: ```@raw html
- +
``` From d7aba554290d6cd8ea9b3884dea8ec29c5010d45 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 10:33:11 +0000 Subject: [PATCH 150/182] another attempt at fixing asset-references --- docs/src/internals/transformations.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 6fb18a392..d4721bddb 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -43,7 +43,7 @@ In the end, we'll end up with something that looks like this: ```@raw html
- +
``` @@ -199,7 +199,7 @@ That is, why can't we just do ```@raw html
- +
``` @@ -279,7 +279,7 @@ And so the earlier diagram becomes: ```@raw html
- +
``` @@ -296,7 +296,7 @@ For `getindex` we have the following diagram: ```@raw html
- +
``` @@ -304,7 +304,7 @@ While if `dist` is not provided, we have: ```@raw html
- +
``` @@ -341,7 +341,7 @@ To support a new distribution, one needs to implement for the desired `AbstractV - [`DynamicPPL.from_internal_transform`](@ref) - [`DynamicPPL.from_linked_internal_transform`](@ref) -At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implemente the following methods instead: +At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implement the following methods instead: ```@docs DynamicPPL.from_vec_transform(::Distribution) From 1f51203120523473a73c603dc618253ed11ad59e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:13:17 +0000 Subject: [PATCH 151/182] fixed getindex diagrams in docs --- .../transformations-getindex-with-dist.dot | 2 +- ...transformations-getindex-with-dist.dot.png | Bin 42305 -> 43833 bytes .../transformations-getindex-without-dist.dot | 2 +- ...nsformations-getindex-without-dist.dot.png | Bin 40798 -> 41849 bytes 4 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot b/docs/src/assets/images/transformations-getindex-with-dist.dot index 8c6826e3f..e44fc0ce6 100644 --- a/docs/src/assets/images/transformations-getindex-with-dist.dot +++ b/docs/src/assets/images/transformations-getindex-with-dist.dot @@ -7,7 +7,7 @@ digraph { getindex [shape=box, label=< x = getindex(varinfo, @varname(x), Normal()) >, fontname="Courier"]; iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; - with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_linked_internal_transform(varinfo, varname, dist)", fontname="Courier"]; return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; getindex -> iflinked_getindex; diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot.png b/docs/src/assets/images/transformations-getindex-with-dist.dot.png index 621b10a51498ec87050960725b6806bab2913416..381ba45a2c9af4ee4a9728c25f31bf225a865fb1 100644 GIT binary patch literal 43833 zcmd431z4Ng*Dpvd1b$Auf6=Y_W7i#u1HKkO@N1oN38VxnKmBY zbv8V_EAX2)z?ol<{k_504GR^;XLuKv|1ui#zT)9M#8Y}EtLvMxf$|9;gPgbTVx3e< znctIWlihq65-D&+o=i5HLiX9V#FFmgynNrZ^B=-aAD3#H|1i9L{WkOAbMlW~*&Gj< zWgf~vXEyq_qrS*Ie8bG_^m=rEcPw;kV`Iy9S)9M8aX)qYey)AeUx0+mKhht*b^jdU z;U)0=rTnjBIysqJ{~Z3ZyvcUupF>}wD=h!I9>tJx?VrP0-fOY{Iw=wI`tCo6uiI~H z{_71p_x~RaTD9-D|Nhp!3Md902D|>xsK4ZMb@J4q8$a8-&d&D<*bGX|iUl1<@-5pV zSN3`TMFjU#$B`n@?S7Hc*V-{+-W&Oe{I+R64FA5${SKd0YP(f_@f{u*EEL3F$f zuZOA)$@(k>d`LVha5)rODKaqD?1Wvt?~mW1!&6Egst|@|X`}M;oDDx8_Le7mc~nA3 zf4jBLvW-} z^$riUnml+3k#wz7GNSjc{xkOYUe&=pR-?{#eS<-B+TjA0q)&tx86BSS^{9c-P!4-M zH87hLO72}C-T4lDmItS5!su2ERaeyJp(8T(MDU^Y7j#jG~$TMA^X$d81I`KQ#w#;BkSR$?G2nh6h1XHjPPFWI> zS86tsVrVt)A}W<}+HG^KU{b{#9fvW+9IZy9$M*0gr=}Qc7Y>#JebyJoY;f&wmFgNO zf|Kz9VPPUFSHYL|RM^R>q1xCN_KSRc0cW{-J`X-Gwq;Aqn20yLq_#tV7IL@U+<~C zO`65bHiQ41t^e9Gd;5}*TH?4T#P*AOWS!}Rj7RL(8ae*0@Yv>BQ$(Jt z$l&B)aiC5b(%-m@d~8mu_moT4P}Dxm81tsi`NGVx3+@!N_;L6fkJQlZizO)Z$P>pR zs5m(TS0U}`)Q&R9UHee-Mr`y_@qY3Dt*WrQ{Dvya7n!P#I+8ns*W}H0L&lFf-tCEe;DMcQ+9pMSu)8&6+nM@<~F ztI>@rM!ci-crSp=SZU8sZZ5tGKi3RlPW)t4_fTPcWEpBVkeea#FLJT6o8g|yLC(Vk z)`Ks4rC|+>8nhYh^WPuq%ICk=uFU{TV z6V}5aL~^X$3_K~!RJLYLfo0QgZ!MWrrASw}&8aVai{bPxSawy`yHz~{#*-nJy|kg4 zaW5PSWliC+Kg|k*ySetXak%^BwMLWr8U{jp(JU`tH;ODv-MN-_=$3~Lu7;)_(oz@_ z$=Rf-ln^FP8#)vO(I&?>YB^R@s!{NBSd8~ranVv1+v*=V?zHP*+!mw{4pXxP%WI6I zz)Ui_9$uQ|eC5wCO?`B1k88Xg=Fg%WCfb;~JZ#oFaO}|4wNgm^^Njo1G?}jYFWP*l z#TV_>On2dT`BJ*Nm50xrn^q@D(}#Wp_1m|uH4qge$tWy3YF*-UU7l+me(2i%d>gZh zkex2sOgj1{(mU>LBHE`)`}z1;R{D|ht(8hAmsYhV4W{WcQv->e0;zhR8nZ-Py;x4q zKp@lf;7AHT?Ek=+oEN%2i-fp#!N#t!CrZs(d#*7$S*V$cdZaUY;|udbOU`fzj5t7t zKGLCRk)?F4&&7cK&kO_HRZ!4wP`>*7Ot5##SGN!0uQV~4uhH+ewu*&jJHHPo3L$FF zg_3gV$xbI6BOe~$2fx3}q0G&*zjd7*UqSv%`?kEAE$ov3aZ(9KM^TrC8bR5uG0D_p zjH=Tl(#W*~A1=pf4cc%b6~ij)?rA<*CWYYQSnqxZbYTOT#q((c=@OplP1aKKem3Zo zHj8JmFj(!lXMe9Y^2mF5e?NG@IIAvSD?h6K9l8*aPbzql!9JTQJi^PBp~t6a0D@^I?VlTH^=Pt^|P zg&94bi=zfJnsNDv!s8C{A8BF{I8D!)21ACt1g$P>{;w44ovXnPISoEZel ze-DXT1!FS@rpelwOV6bX3+upNlf9JYrEI$qrfT${(thQ!W?hP3qFB9sNSl<9VKA%; zdKk%^0_5)(%R5hFJiON&?YeE#*P$yYS4`j4JXN7081Jm4#Iq5o50$DN(xM@{7vZBVKtwpx->`sAK8vUfTp;6}fu{!&1zVYYqew?}(* z78Mk^W;;AQ$;ys|)$^1M`M)>O>|@;=;jcQT15!qo^cH<)=XIk`6ibB8%wq!>Z#Wv& z8{TP+lP#0l-ezikV&#g6RHSTb^&!;X?RC!U7 z?%99V7qH^6P*#!5I(Z-OHDNdiNnh_r^O{zl+}~TfsEv^{RU!XU?VOp0!@h#}oho5S z4!i`@UbO{(Kf#yID;;fRXBjPqJ>ayW70u|+;~AB_z(%+nwX2$5^wS>=(1$-PCje8v@mas<`DO=xNOQC(JHf#H8*kBv*EH^JXY4Y5x zdUfBb+JoZ4w34SX_(LyQZ}6F<^UZoSO9g>^94l4aFa;H7<7huSMP-2R8xpHJtEAkN zNppQ;#Wc;MY87btebzg%LC8JW^wmQPom(fN zcc9^P55{Fqqyy#6TlBOOHp+iqW9!)dXmjRY5^2hpSf!@L;}nMwu&z5?Eq0{M33gtT!)S3@3Z7%Z^oYDduSx?-Dg z70hm|bKNYZ?qtcxk5^o=y9-u<u%UWNmoPQ~db{k@+^ zy#oV$2)bX3-~BtC9h$}e><$j6iNt+JtG_6y-d@Xn^eSkN*d@d9g3mD7B9Ba3Jxg3` zU7EH~HZ>_9`ntOQjLAz6s|DT6tNHpW!oT#9*KQYmde8qBFK4^;@Fk-qD-L4aSQTex zOx5Cb+5|Z5r<&_A{cJ-n2I01&>tm(#t6SW}|`MI9clY{X?bM28s0H! zq}FWnD7A)JOcCEeeX!gIqXo~cBwyV8VqSS84DFvjZFIkoKlEMq;FS0~tKF^QC=zS? z?WthPrqmM&#z{*iw;6Rcm7Sk!;wPZ`usWvOX%ULx8r zU)UXYhhXP0#{7j`C4AF8u!^^KkLQ*T%UYr9A;(FQr)*{tPZPj=8xt(!{pSW_oEWgX9qrYD_M}F zM*i||zYxW7<~{nO!H zAZ<9^@*jn=JwU`hJHjj7oD9xJ#=JFstQYq#@RG6}OriauGCkUU+`%E^5&1b6M% z+y3b(h)ecmNt<|RQ{vs~G$K87Zg$qzHw@7TC|t!_95k6Gik;lfS{ak>TNrb2bpMJI z5~zeM6FJw;E~HEhJ1v?c^*9BO%RS%QT2Cyi{FGkl8Kk`;WXg+UQhP9=`dtK3oQQY?4gRQD!*pQm^iT zotI={?~TiaN+hVif2-D z(??mmmduRxwbQ=lMf1|jQwAT-z$raUeWiq2??6{sYoAC|Dx2TkDK?k8tSKP z)VWV`Jbi~QR6UYpV?0lZkc0yhEBx(Ku)}FPjbN~A$WRJ*4xP$Cihbe1ctJvA{Lhf= z2;+m_xw<*lwhqmIFwnz&NKz$_%DAW@&UyS%ub8ie3%@a^?cpaxDu9q z)vLDp*mJW#H?r@39;?lkuC_x;e7%T~T*yr6C&&4?=pfCe`jhvMT*RVR<4(S58?D$a zAmVaR5>x+nHLuIupE>kU(%XN$@Q{=|6+Med%G%sU<=`({6|{3A4xWzwI?c#bcH`XI zw-tOPmwTE;H&8*m+ttxI@cmm9(sI(In0qL-TX(OX6>_m?%9lhu3MceoV?eOmLhFTk zsq?mep9n%W){musaS>z2x*jCvvh$y*5Ff;#`EW0TI%y8y)``ABrD!(>5X&GNS{<6s`=?}J9p~83{@1| zufArzl+Esf#79@1`4_WqmnkhdTCR{dkHxU4Jkt>*=5jf9atKtcFfyDMIO$%JhZfgW z^$!>%Pr)SJ-b0VmO>6}x71zf#C>__|SXr+uv9k)AO)g7*KQzNXzyz;Q(u>p`+I1C% z4_i$=c9!@COWCkGtFIaX?C{g(_tpEc z1^ed1O+gZmR8r5OmV+)VHY;p`-j$D{9Jsd+?l+wG=+fp_5d9f$oVgc!pX~dgAhD9m zS*=%28~5{3!FR3$KBWf;N7G`QwPlP+E_*eRpuMqq#Aj8m$uF9Vq}R5Z-HieUbjOew zWLx1&nBFT4+J1P44rg&H!{z7yF}_|(M(1Kgx7R7ri29y6vzx^6tF@%2)sJYC;NslR zO{OZ-)cJ{|)d*+IMxK*nE{T&ETe-imS}^2CeEpCCj|mNPvbbKjFmvkAyPuB*zj@pc z7@*-H9@>b;kfGHFa}VmECpof7{AFoRd%ZS4c1N?b2d>*R{1nqucorymifzPg^)eV8 zw&4XrH9RX1JBQ!aidR-xY(vIXzDHOdfG%-%oezOV?66I~RK{Xwfvh zNFPpLQ!zDpDvGt$5xv8N349u2aUrZLXfj!`v3i?JwPWCoeY0+JgQDXo%-qxwgwW= z)w?H#ZBMm&1g^|n3$hKXc6n|ToU@IAy<~`&(h|OL24v+Evq`D0O!YToH!}6onf(nDtk7p`|Or2 z<%rE7_4IV^k!!BK`msx+8dqM08hH~mJC z+^?G^tn*2n53`wKIHABEN!P?Sf8g1vvx6z83`yxj!PPy=2Jn+Z&qmV(wc-%Y*NLpV z9`2?4yAFf7K+`u+v$u4Ae_cn5Ng!oio92Anb+i$6|r=!`GmvT zGy}wHyi%@T<48S_KD{8#vI+T7!`s2XWKp?Ws@$iPxt8Dh(?&)DZba+r!{iMixPy>W zK5yXqB6ja_{!&;j6dghhTd<5q32REaIAxCL>)g6!&&pHFqeC2WN;9mSiJJ-=ZkUtjzl1XsHXBEjx_oc1yWSn+ z`+_%A<#~|BlY$V1l2j@nIU)DY$tmkP&Ro^M+D5$}al1jN{j^*B!R>3CcwR~K1;4O(W( zz1A5N=>ne*(z#DNBaf*=r_=p4e}TJ!nTYXb{=xUN5<`ubiygJKPb0KIKBz z@wbO-r&^~3jR3%%Nta@-ApjS_R+6`*OHCa6@haY-vv>zrZpCzhHPqKY93Rhtf3 zG+wMNSWvH4^~tq}sR$JYG|R&3 zs03ym!$iY4=@5JO4XwJ9w5p_?4J)-t{%6Et^}cj}+bOy8D?!JbOY1z+&6+Z=FFvgV z?Fp*3+Ad*Vb73zf0QHXqOkYYs8SGH!E}q-Zj$=7O$}QbU_F9Vshx6QFj|)54FfW@* zw^!#99}6P3d#Nk92IT?Y_jg`O&yC5FL9!?;LsFUYH>@x?jY4F%-v zZ_r0Ol*g7MOLcn*l2}_w%@rrp0SB0RZjcu?aoRKZgViSIA5BHAT+2}p-I>rihrEU2=^Hd?VpCp-wUx@g3DEU{DDyWRxyw6 z=wp6)8dy{t_SuEMy=5cn_=Mn)q4~?w@Cxfl^rQewqDNw`m9k59;>8oSTD8)+O)t`bhR1y?7z1?;v441_^71$0hL6)u%c#Q7*(9mcQdzwlU+TEWP+!X^4Ow>=q5I!z5g6;MD zm?H}e)Z*)DjT*Kp;FPwdqv(j*n5-)FNdEijuQ66*j#F}E2|gXp z(!lqFS2@mKT@bLHbYBwThdf!n7ZEhi$&xZ!7MB&V*-M7(Qruz^uh@6;NqX`Wg7|1m z7Qb7btCkA82&CaDbzu^y`CTxoiO(i6WvqN*cya9Z!2+?j*dhp+IE`dEQ-as?-&H)X0PI~=`P3KjasPiHMMX5Do>{~;OeGKKXJj# zIc3CBiW>Z4Hjsixb(*xMwXzrEY2SP5P0nhAvZN+_Cs0+yp< zE`x6+?UlPA-Ag^lx#so?izNz4Y^T06_aP1v=W)Td{Zmj8UbU&8CaJB()gSb}=$mss zvuxe&v>j6XpfZ?SkgABcZTS$62>*6x`cS} z@F-aU{40}a<(imrC#y+c8Ez|xL7EZpGH(~sk5XT|{6ra3o*LpK_c!x{3F zC>(ZFz;j`cpU1SKS}Q$mZfI~D^L+B=@e5+A5H$>)k&$oc${)>NPpo{gqm=Bq9>GB} zkRsMxQL(jN6blOt3y<-c)&fRTg(s&dNaw2Y`_9R^Jh9aC66@ZR0?X)E?A=vPOO6Y8 z5O`x&eJTy6yEM_ZSYj3@F8D*gDkvzZ$c7eUuzlyM9=wC%ohD1%gA5jwW?{ZRe+<|Lz4q zouwCd?Ku{Y#inb{BywrC42N zZ9*(`?|C`vepKJ?U-s9#3L*8dKO;@96hP%xZK6}Wv)Gd-AVsA~T|+#cK_i=%w)v9_ zo?h*t$rZ(BOc;q3S_J3QCFE@{h@<56Z}YX_jRC^#-cc@%9$jqviroNbIOd>-*2v@G z%45s>Y5o^%*q_K+`KFfau8HhCsF27>qZaR^W@o+lg}hH?hrR=>@cA#(Y3IiISnRj? zoD!zc!cUm|#$XX;c%|4!k{02c-IQ0iyhj@PDB`Q*G zj($W>wYPUDHty4=ZG4rrbOhF)uH}VYhLJCFg}MqNqw4jmBm5LSDe5y^@@AArZ;rga z#gD9#O-x@%dck&1h2reba+Nl;^CrpG5doZnhL7`no3c@ zoQlx(kdYuh&y+4qL4j1XnQcY#j9#(eT!~FxRaF##%IZ(ABEKuLHb~ZZQ@K9U4T{r$ zHsCYCw%{i4{VOw9&T*hyFG+P?3Jt?xDvwaWm%@Q-!i z1UMA)*?+{LW>0&*@yf}Cl1T0=u(L5@6N2;ou-HXkkBF-J;oPC^?m)ligHu{+kdsrh zq}_I16vg5il$*gX`M^Y1yusJ^b>kSHj?b`2Q$;_-D=H1@^EFZdGV*@XfufdQh(pDC zgTB@X<4Xv+$ZwpUY?R4bPtDnuz3wW&F@L>OKx&FalN`Uz9Jn@qXoe zXWWy8f-r;IojnQDQ4IAO3UHQ+<+e4;C2>vq{xeE#Bpp_q*1&X&l~eVL3)JFs=1}9{ ziTSA^1UhK;*VGBcTAnEhg!&{iL|5jA6|rmf%EgGbV0Fom`6HKs6lW!|n6=Da4!zcG zdYgF@wNs09G)hvPssE0N-ht5j`Ph<(?jaZJL`uBZ;Z%}rS7z_Zk#-?i*7wf^A*1yh znO}(u9f*#iMtbU0Fvqvy=P&W%6$$wGFBnX0QFoORTfH^?O|59xIf3Y`A zNP`m0?RD3kd-t+>lZ7c0t9-W2l!jQ-1GKH60A(HiY1vMwPXq4Je0|xGHhVYj%a@Fk zEx3k$p&o_G`3L@7d*v9qfE`(-n90rf?xmrrT2D!RR`39~>))VfLOzXaz1EvIZ1>3R{qkz0q#m^xqpdVC$Pa$C9j%8I_KO{AZe?stL1&52?I~}(PhWm zYn2yXUKJjmo+<&TH7|6e_*Ttp@=J00SD)CQI>>tN$B!THnv;@}8jO~h^1PG>S7o&R zA>c`PYvvs$1v39z)y;DQramY=->U$X!~1nlga)+3)wINUCHlCylw@XQNow8!7u@b% z0SNgomJ+pOA&24oK`O-hXlZGFJ{zg-Wv5=s-|Y!*RQGu;-fwvO&v}M6e}`{3PQLq_ zrsKamak#HCDSZBdSbH4Mw}b-EWhaFuIPhAEGl14T2DkjfUk_f7 z#pi~_`ryBa2ckoWC2$2V=AVaY1_uW{;3$t;6KXN9{Jic<7BDOQvxII8*4EbS;^M}+ z_8g+3wJr9EiHTjEo$TI8lbia=%F1)L%ZrO_{QTuRer9HALqkJh0j}EG+R-Zo1qEQh z5B>ane6h*N{t69wLBWREd3lsno_k)9^$|(SB$P`+Q&Vta6Q_vCi;!0-yr?<1j{g4A zom!-W-piLSqy3&feHtb~N<B?C$PXX>4qi&-L^7?{06;Q$+xJ zXZQq3!>v>{?nH3&Cfi;b;$1Zkvrc(y`t@n*b-u6$yiufIVB}csi>$vTv@11v$e8flay@e*pKUzxICxKVP@~&Kq1TpIHR7YV{dQ&;k%=eQPSStUWMySV|mEJ%KYXgm$Y=# zpI>DUPbDz5O-)T;?qaK*dZpE@i7m)!N49Qf_vlV`Fs}{oD~6 zXTjBzrW-4`qa(1^8bE&mH5^2iU=Y1(7ZhW^MsPz0c4Dnz&`C($kfST-zAD8I2E#>^ zBqt}wGlr!&AmX|yrK7D4gsQNjLLetQn=E86Pm1T}=ED4Zm;}#FjK-!PlmqDI?CeWG z4kt77GX@5R5B(Z}WWtin%!@HZs4JiZ7*!q!Y{?ZcMKS0>ek7eFY@t1Bee9>@hYLyX zjWH_3*RNmWzJB#=TM>B>X(1QU7D1DK2TFDtEW2Iu`t|ESbf11LwE%$yhQ%G^m?Ds| zcb`8K<>cg$pR~VwD^1p@|(kv^D2>qYO8z?9!e3JYgMK5jUOEg|?8)ac( zK@i27hn@Bkd0_iMgW#{8j~{O$OH7TCg$4%2MwMiD?r1IVY6I>6d+A=^y?Zwn7-jJG zwVkaNtEN z9aO&)#h4e8cpXvgJedpnfBzmN=+rkFi;s_AXw#bnGi!iG@02M-P{Xvc6y_WO8KoVq z*Fi$cdug3Hp6lz!&o3-g0?)CDf*XXA&~gL>tqw43PF8CGEA(OC-_MU-KtLC)E?{ar z8d;bZNh=P!92xv{7%3|F^5r(N)ZG01{G85&4=>^C+gCK`(Nc5Rv8SgeWMhn%g@uJ( zN~&>Z%wT7x(s7i7on6V@-CbElg)<->=pk4wz}lP;Vj2p%6KWx+7f1DAa6Y1}G756w zL+)H!oY=R(H=UoITvFNE+R82@r2q5h&#>@t#G9-56~PzhjB}Qi4oKZRby_e#4Fgk1 z9O#$M5rd$~*5thyT<5Rg#kJ14u@UpTp=B23Z7X!^d!Mno3!Z?Ob@|3cN<0RDt_K8}NhC zVB*VjEWjNnBqSKT>Ua_JHu;3?L6gt+bn#5kiJIqf&){H|BFoZ@T|N*Q@PyXfF5GW_ zIqdW2$C#ycVEq4FHdUIGMLrPCEH*ImZUyRZKQyc*TB;d1ULM!I=JcjK4{rrxO=aqJYPq~aL zG_9<%jjEhFBmD93c5VX!{UzVO~gcctT;EoFP9UMga z#XOenty=H`kv5r!=LbGWshNJb(`6GH8%m_KdfEA0Q&@K?`YK0p1S-t@psY>e>{HN$ zhh{;oi(#^4G1Ayf-?;A?k4NpqK`<@SUYqI67D0s4+AqS-=P3Tin1n#FTc#phJUkl4 z)h@^+K}RwWhPpN$E0i|kV5$h+GaQESYIlh#oP&cynSu~s0S~V$9Pt!%h?d*-({a7{ zArnr?GXfy&Sm3opM#ju9a0b9$XaUP(psD#KPo-G9XZQ?i*`;6H!zXE~yk@03y7_7_ zDfrAbEL~hXVg5SSZ}JhkAedf1K{#D^8}>k5pOOEZ5G8#i+NJC!tCglo1f(ku&TJkCL*I68pw)dO&`Hd5|H{EFC+${1W(< z7*KeXK-_`oI&lyPdlHEd0^MhvHb7-#vcYZ&{TH%$ue+n{lZ@cr#f@i&#(^iB)t4#( zf{Nh=e;>>i${47EHV`9fE3uT@0iEKh`E}Inn_( z8C-8>esNxV!z9-aiu=p}6LpF#xxe2KQZ;0 zi_wY)D4!X2US6$m3NGy^I!OTtTDQ`5#>~LXFQ+YvzO=y~3#~H&dE?{0yI1fir!wWk zy%r*bVSq~9(L$8q%U^iTV?Sj5v0F&-?fNV*i7p#>iXwx%BeFsijoUFmd(69|@Zpe( zZ;o^(?oIYBS;zZRV*4Lca&@H7{cYw>>YD21#k&Koog*8Y;7N}|7#3QZ3rCr0AFu86 z`Ph$?%ORIw)C326+A_VI$dZ|NNj1JrWdxMYe>ciqNL<+U_QjX2pmM(a+5S;5EvK?1 zQ5;$8bE9(}WlT@oH8fp&G4w0Hj^I{N2`4;a$lvj*)DeIVTqK(XhqfAdfL6h z5gDz(ZG;GgUN`mcpnA`fmRB6Ry{$~v*km!D@Lhl?pN2{-!U91&i*dFXw zAUwRj7%AXnqn(&gT(n+P;cOBQiU050;6fXYMT)z{A!q+dE1Y4R)Js`%`|vX7uQ^2w zwvk&mf2@TK(%a7u*borI^2`#(Pp_i$yBGE$N5hBStPW>+mV)^@_w-12 z;;Vz-m!PGTOV7xBCW8E8id)stlLNyeiTo8TrSB8D?}30E=);izc0*p^$)P*czcV*^XX-}WmnA-s78+uI zv)F9eLjuGZ2L$S&NjzsOX64Z+wmrJi#tYu4IyFR3nmx%O_qivrvFU!Fp<>Tly;T$8 zJ|~y*#9jp{yXpHj>|&)^Imi7FQET%m#+V-at7mG>>o@innp*`@)&xc)99<@gh0KZA$4XRULo(JN5S3 zj-=ZW8Pp$d&LkyCwN`#OBiJu7+_l*a{i?b@DK4khJlGlF=~GK8k|2ULe@VUeb(W&aGdzYP?@ET@KhNjdox2XJ=%~9*V(ObUUn`yfdQ>6W) z^h%;#N>ksdu<~7VQzdH%;gk(7{RwN%GAVDn=J>~kj7cziJ9qvufI{0s1uW=b%1w zBrH~9OR7<$xrPhFlDhC@O&R!1~l3rf`)mn^x=+I(gi_)cec|^>VU|N$m@P8U;aMd zC|`bLf)trqZUb5zyKAnh-yrJJhTeSb5b@31gGW%f=;q=pw)-w~xP-_Gq za3YJeFLa`->%V^^((zNAha6O7bpK~PNt$iPkBz5gTif8)pYjVn-tDU@>dh3aZ&2Hv zQAN!E31HMn&JE5f-r^LndN%qsAtb?aT)DEZ>$~oH(XdCQ%Wp!#j~Ymr8+D@IA74QY`0Fgfup@%P8Ayx#|2t+-ijS+B|I@lt|4`}HYh%LZxwRS zHNub$$&8yWw30CotA`E{6|Ru62_p^l^i1ASVZE9tHK8WzI*@LvJwO=6pAljfs^&}P zsyH%j-WSCUuyq2YLVlO}D1r8%n;&1! z4N|(t%}*-4H;w+@Si$nWY;0m}u$q2kPpcHB7|w!5Hl!K}!nF-5w{q73a`+`|mCezy zXfjj0v-EvJ>g4nYmudPyS@zxeXoo8Vz+%v zT#P~Q4}B)1E5@a;Z&c*qeCl18_9-xa+UhA2TL^6d-Rp9)G*^C;zlkXx1i!Uf6q6l{mfnGbsEd*Z9a7Z;bP zcf1f{#)!jimpYczDgdv81iMKp0**b#H(L+s@xkU(MX>o)CMQwUGq=iVJQL*Fcht@H z9>d%b$wrmIQ4~$mM#2IZy{A1Z)cz{j)DWn4c%+_;)vJO6F+mkENax59c6vB0Rt6Gq zM0mER_;$$C^f-ZR<-7o3CsLbHXH7B6qDxu2mAK1Rj%C8@u?NTOS{%baBn8i;f$P#}FMK;e2bj zc(_455V2rpqW@ElFV4i@ipMDBY-TIpljC_U8rS z1YxjRo#@l;+ANM+3BLHUevruZ@11!=*Nj8t^;K>Sh9+i9`roy_ae&Y%J(u?Pp`+Am zxNtueZ07D@ym_;R!R4jCQPQW0w_MY1#A=Anx}42n=bw&{g(~0T$>MyMoKY9i_yQ|# z-6rKi|Eq%C!^WA3@UolSvu`&%gM!(<*6=x9JqnVd)8PYY>Wb6ManDLX|8e9>SN@1E zoL0kAS@=BRU~0nL3q4R)t1wU!L1pYBw1TtK+0ks9nQfmdm12aLGhC#e-10X$;oFKY z89N~yu^I2H?4$lb@vMTd#ia3*o})U|Pxuk5elh2DtlLD$$VG!ga!*x$X&|bS)>O>z zm5b;Qv>2N$jZPe_ z!8*%oA$bNGJWjRAN0`q{xw>trFouPNTO6k$we$hMLRCC=?u(?oCt)>TFplb8_S!P> zKC7&FlrA525k1<7>ia&#M^GL#dv>zDU@q=dQL}gBM(UhYdi(Os=Y-DposI(ZqpdZM z)vS$&jfsLxgbzutkq31xWL@;&+c!lUyio8Pl=ZeQdf7Rt;nuCz|HZqFE|$fqZ%nz? zn5cnUOy`PDUrv7hN06x8rJ`Ef(0umnS&G7S;`&J&YD;eEha-CdlZ0SMeGKo0 zT(_A1EVX!Uo%#B$e``C|IHefacexajf$e54k$ zjB_+q{4qzI%SDt$aCk_4sG?&V5}TE{l?JrD6v85!#YzeTOoH- z^59(Mb3cbNrwXgtS8Uv$mU4`GBUx+jgmk~SoXnoYc6I13w51R0yT|qXj=;j))#t=ub z)s+uSPC_(Tln=d7as(*&iHbkCbLUQ;)(r+mM$97#i4;%-+5e4C{{#RPpeCF`8#7GV zq*cG$`ZzZ-D$79krqq^;Z}rx*^f^-g{mFj6%oVB|K~IkPH{!_5l5VT$I=W6d_#GE+ z&n6ttXSIK75+R}&HmEmJ^gL^(-$P0-+=BHMIhHwnyG0`wBFjZ^ghP1s8D<_(|0=$< z$s73~x8p*gtsOsg1ixfDzL?i-vu801@nUE3+d={-96m=rkY=_^%DXOYzNeZ_Ve6|y zb#byA96R@Pu|@n-VWGOen$@KQF1so88MP{}fBj5LruHqJb&KmSdwN9r9n!|OOrCVh zjFUB2&Y2z2i|XcQ2fbNq1q>pEpqiczV5BD?Dk;+EeD&&8ABobXf~qQT)m1PvGvl@6 z%ig|oM>IK((}2%mm<=$Ir+()zpDA~IoASPBB~PD^Irrea=zT}SA4^vijzq=NY_bC+ z`w!|;RH^#B&OtqGC|XNDo-F3vu38L!kv;s3O1eojLY`=8 zyx`rAx3@^dmsj@z`owQP#5_@9-z&{h6ey*klau`UEMG7@ohsnDuxwycf@_R z7U#ox+AMy$O*&WKxqp9k<2D!)t!p}r4;eB9W`&WB?V}={-}}oCW_^_3by@<%<$nNk zpT2zgPj}=Taosz2q94g!+DCicI|Wb!TmD?UGiIc1;Ig1cD0qE*6^yGUtQM z$pooD+~q6ZzI~fQTlev!fZWO9rX}!2ryvYJ%)A5@b`B1*!T&6Th+ZlMFc?HdQJ@%H z-b@Z&6mvPdXaxZKh5vqs&+ohQ60fVP>jkKRo`udU(g(Jb_e&AV=<1Sz7-}g-g?lbD zoSX5{BZY(zu$Xe73@xXiaHHvTj}V}(w&0~slU2kbL4Or%q)SO$obDfP(+CR-vnoDj zWE_&nmtkjQR0~N?rUAvZ@8FKZ6_N)cFSiHF0AI}fmdHV2{PhA;D0X%59Vm?Y5AK1N z4dGp9WTdPCWvzX{?k~~NzX4=0_T$x6fX|VUk>TB@pilrSNd+*GhU&&fao}UnQ3SEz zec9J<+~Bk7xE(@7Wf8nJ`}^G?eOlaw`Hz=Zj*pM=-iL$?Whs&Z{CI7umOoVt!pOl9 zc3H3${`~Pdzm~p!8z@(gRXNjuDkrF8aXC%ZL=)2peMwFxyxhsWI+%qo>3_I5V(iND z=urrrBy<3#$H}V5!^r6IO6v{Kx1>}61@PJpJiI|nt-EwwR8+(v-wlL-AK0S9?HSV7 zg@v2|(qZD@=rXI2IKD_rOY3_9thQ+Q{Jce#3(A4#Y1-+9DcoC2OY4e^rsi$%{Y+7j zAIu1Puqq4E^P7Z(o_luz0D1Fo&QgBa*>U^R++4d=w+@pq*VJ^Ih?)ej8PJ2L-rhoZ zkG_5aw*qkF8Qgo*ZnT6Jl+ali8Q%hE0RgIo0fB+3KVL91_KwUohXM^H1$S8h&RW^n zl=)*_yf-IWtDGl0y1M>6A)582G56D6gB6y*JYqWg`};dpfDvI_$^0wl<42R7)3dWR z6kHe-_M|o|K_y#{-OOdODgyB92S`wW&@=;~+(6)tkM-DJC;;u%?>X4nvFnh(cgMa0 z;NCDYT7XZe+1S{`yeg~OB2NuxE*0vlV24&^tot>SpfIa}Ri~!2xy*=Fv(2h-nKNNZw2oA7~8yBQEZ;EWz z0(7Rw##4?f_v7e*$U(?frCp-a?J?12YH-$poA|hgb{P;7#E&wTy z{FCofQ7PrAr|w{}K6^2M!OsK#Uj1we2}wy105bFO?G+(5Q1ApW`!8{E^7{G|etv$7 zi;F)1sMUCJ?te*ScXw7r#qc?ukB`sWckgZi$d8%1#jL^!RVN5Q+zJ=e#QAnc#zPdn z&NH+cFbDV~4_*S;dtCkV8W1a5g zY_8hakjGCUF8tW9auvjcewJN57vGcz;z%kt^Y`ueZ2u^2$&hfa4kH{(EO?CtHn zHoswrNp*@D1eYa$m>48U}=umEX(Cij76nUpgmuX^k zRT72^xz!y(b907QCDth$YBN*Q#JweB8L;6wA`uPu5C;th2L}-tbo9*5!w&i1JhBsy z+1Yn}KZfNpMz-zk?@uf(`IMG&LsHY;-v0NGmp34XF}t|v9TP*c zva(|DxzJ1)-7S8v_QkQv~Y{myv*J{r&(E~^!Y| zvzoPqOy#keZ@>*HD8?HDNp~L3f@0vXoW2UFenKwmPhDLy@Ho`V7|3rTvyuew~piJEuV;7(ko=ih8IUUZ8yz_-;GWUprV% zA)`nYMi>?vDv!54s-J5CdGfd=ffEfb87E!dh=*A6tWjO1A5FiXFtLb;yD5T0iz*z`&R0<`vuA{S z*_1bR=6FWosP^(e_Re{Q=N5Up<4x_`LxXZV4A!+xAIlfH@{{I^)wwn3yAF05Ndu(h z*T)lgZX5clZzuApW3>jzDX=gK-3^0IlC7>Y&$+*KQaV8;#Y~YSlD`q}={P-he zqyxjk1XoAP9af6xie|p3UcGkBLd7Hrds2cfQKV*lW8>k22X8-oxCjd^EH19s9>wX4 z$9x%RGl-rBx&)wN`$`dY- za)Yh;PG16cf4WyC9Ed}BCs4n^`2=E?7ccNVL6b6A&5FT87k+=g(7|^|0|2}xa2J+` z@_nYKb+O13`rs{uMMShg?9-KVJrJKxr~W3mqatErH|Z<(9&&PWwxi1;LN+Lc=)V{E z`H-U4%2)9U2uOlBglFsb^A8_CX6rWH-B}s&={0&xN7v5Rl`HgW5llLmQ<#dNFJA;q zOlZ0j#Yw?`7#L6r3JxBubgRs*L2Qg+*O?Lz6iAAX|IpJTM}vI>cr;>PS+2PQ%$997 z#)nO#^u*cuNLerdJH;5R0?>JwV3a@>LKETxsM4X%DfnQq--3vqcX|NC}n2=C6 zXbzz?8OrklGiDC)W$@RpA|QmHyu2PzQhI^=$H^^*h?v0Bvw2Nn2gewKN5GQBz-Iz6 zddS2Slpq?GV>Kt?^h+k5GmSz&p^K?`2`qq~2Xqm(G>@3V5$Xi3Nd|~yM8$N?3WNd6-K!Cv) zQZ6vTgZJXu?vVvCWQ2eO0(U`wLJQkuWO8qb{2gk(xA*lg^cwzk+7BK*`T!yRXIN*L zh^Ol6#6U@b(0&|@+OpJ3d_;Z0Xn(3?OwY`$cM6hBEG#quq4Xsf*7f!E`1p7!6_rJ} z66SdE+nHUJAq5l#^67UmF;V4}Y;XQFT6ORWg69T82XgEMMjj+GP@oOm22cXPi5x7o zqi5D{jfF^%QK$YkKE5{$4Bj0^Y2Zl(fnoPm0pZu-c)NdL zw46ImFhKuLk%jd_SKZ)uL_PL+)2(B}1DJskg~n{q{s`6vT$qjJ+>Wj!NfMx;E|Z$4 zaZq+K{M9v+g=4NdH6FKa5^;s#G3)=%H&z7r!G66C2egHSZ8)-QtY?wIrvc#pH1aju zzJ5)C!GFX6TZ=|Hr`FZ140MPXS#a)itYS@|;V=L|0hB5sQBhKOwqbW72&opyV`p%A z|IuATXDX(+Dl{bI^VhG)WP?u*gn@HwE2XPJVPOsLF5UP90@l{n))eU7*wm`e4AC~! zRlEl=#imym)75VXK&9QBk3ka+ktI-bt*v6fHWC&SdqhFeIH=_@67nL9?iCW3iiXkYZA>?rjFv^fBNN3Uo1K-NsW18d^jO4z z1dq;j@2SK(%xnF$bT9G6Ko?R{Qi6u|1bu}`L?r&~*?CMdu~=B0c63QtkguOhk}*=R zfN>RK4vdKj=rz&h(!V0ggS^Mcup2=X>-~^&U znG?<|iMZIymzP??=o_@NZxa%>0$c{{-Z|h+;XouApO_f*`LmQkoiGChul0xPJXcG~ zRB63~A9jTo7Zu&c!SMtWPEQ{IEb2SAZ&&?i#TOqTjR8J+AX$}uZnou&2>3bS>}JGp z`>mCc$hjD|XecSTOGv1;8?(Q3US>r}#IMf*3}{IYWLhfV(+2oR2j z(kHuMf@GxQ8DPS05Z9P_f(`Bh5x4DaBBC}xJ-hcV=TN+AXg)N^K~4M1Cmaq|E)wjV zuJNOntr7OCp_cZX{hrw#ADht}F5lnLI!0gC8}lw-K3qpzIo%~M*^9Ctwf*#szM^X5 z@nFcXS+H{bcN@cW*FDAS(UOy?X~$Bhl6M*)-N1@a|=Otx4)W zt(m~|lW96&sc_H_USCof&AZRWDEos2;8XJ zqy5bYZaX@}Nd^<535xuWMFY`ao7H#AihuB1#=ryK|BHs);Q&K-1Bb5GC_e{;UnSqD4w$8T z{go##!QwDR;5$^}?usrjC-|PWskd(6m2xHhFS!;KJqGai?=@UkDT z`|rl@!rkog5i zqb?MO44A8rnGY>|>5@J*Y_!AMiuHn3X2`m<3o*`duUUQ3meAYHG+S@PK z^bqH0EWTKmgt49H>B~z74l{q$9xm*!ZBO$=Hg{!5UR}i!*7TG*Dzzq^>|(PYSZ#yV zdbg zrHsP6_wG$EOA+{sUM0%%Nj=7(%Qpl3?%Y`*5`PX02PfBLkO`EJQjz%sP`fvYdBW|+ zT*43rI|_wbzHJ=qySyJ0C_DeMF?@KQ{ovudXR4EnM(;NK;sf>w2#k5mFljRh8$LZ| z&v84}-!W~Od?LVA*=`@u<^LS_Xc70}k+|jngNeJACawYX$1gh6{4JsnWyt?}2M~p9 zz$RGlRn!O!3aW+(xd|o=m&cv~jG%Z?xt04M$!e_C%~v{1TIVHcI1xy7Zp_`Ue2B<7qrB z>(8y%ji&-#|#dmVzoJacBj0C~+2;)HQe`u*9I)3|Yy%E>mqy@Ks} zxjkkA98`S#uSRCP%PB)G@9E#D^LXkY-Q-2Gqy9q8jxxG# zY0q`>ysK(u*pDCAO&$w!<{Y!z(nWRT@Y6mLu!8H$3quF6hoxTOj*;^ z_rH+VH=z=T*db^=>3NSJqv)o9^Iy7wjbsdg81ig)Wdw1$rt1BS`QnD6>w`aiSDZX- z>tA+6aoU#KP8tT^pfxKBlz1GIyY0Jb71#FpWY6WmZP{^#a@6Tk)eE0MSzP{vhV|TJ z0lbd{3ZJuWFY$jnu+5UNt{|JN9aO-KR`k;uC$ZhwPD*jkCEL!EWWH2)XXKMH$wM6P zzusDHym0V9xf46kXSD%4M>Yjux=h-&SKpMP*r4EHW?3rD5+yX;;xf49wxnz^T%L|r z0z^JFfFBfrXbFpo>cHy*2*WxxIc>4xQorswXZFKo4(*THEAv}dlTZUr#S81A z9h{bhOLgp4$K4hm1oy_Z`D#A!g<(cIlb63)ek%2W1Q6z;%O7s@%nMvD-^L9S_{MVo z1E;&cBpKsTgw~7g>kY#Vw{nv(Xh}QqbgeD{`oRaH~xSsEhj+`O71JXA- zr0`e)kNsoaEe^_z0kSY^YAivipM=EaMhn^h2SODAs6pW$%Nu|;x1&qLeI#UXwx4bz zX~Yyj2p@uigV=-|;#P?D2_z*YXFFm#M$4T5>!@vLfXM2h%#Bdex8&sH!4TBu=(T); zC=s|{NFE=&NB#DwCO}-H&TyxmJ7*z4`}ZF|-a#f)CtK+%I=VjKalkrZHtwSVOa;)e z#}6Kyv$NaX>=1kLf)Ik?XKHGM;JJYB3ndOOl$3BWc-GK<{rUxu4-N`?W@wlKp&0ms zDrcH``W{#&igdqznew`}BSRWuRfmwi<97g&cEZj=@t~yvbfPT)Leu=AB zuU2VigP#NWrpc)QelnO~fQAvc2TZe&kWegC|AdR>6ShM72*P$mej6{DLK#Nc-lUK9 z>gYGf$gPb!t==Ia>455#CP4b2BBV|jzz!OiMsO=2>KZJvAVt(BVj&?0x({dvd9?jI zC=cO214cNnAd@@&4wgY2a`F&CS}zX-R-PWO-X!Gsj6g0ce}v(SFm=b2Gwkb%4Llhv zI3##mAw0lk*1rkU43G(w>OKc32=dt0X92P0^}lzPl9JLS(JO`ID$iys^9uoi9KhWU z%Uc2Bv!RRyzYTs^1E5Y*YwIIgXk$~vakAgx)|Dblen>F^7>(80-QJz$JAJ`^Lg0x6 z!|;ax6%2=l>d`Belq`N}1@9nq(4+w85i>LMKR_GL{&aAj=`f@!kRTXPDw{Ky1&0P1 zUc7Qja`Js%-fvC88T&*}PT-u&ywG zD*!Bz0R)nh!$d9!5XisMq90>i54>Rh!Gp&APhl_v!*2+K!B?G_oAdmOq5vSL?u23m zbO-_606GVJ1Ob1va1qf1B(s~<6o#Rzr>pxJEH@M}Gy-mhfE&T&lCzMaO#pW4nP7eW z^na&QK?)&ndG=lp+k(y}HXhh>o|1g~9{`C#!d^gA0Q7$Za5L;i15a`Y1=PTi7J?X_ zk+IYD;qrS7zHuN|kaKZ;1$YQ*93-JE>1;OaN~U6`FM*H84mh z3)t7+P*xcv^-hWvBO@c|-E5mBg}!v<=dfv4%Qn%!aof%QgSnjruYkFAO`l~#{bV3F z2+=fbV+3)7EP)@;3^wOF3}8nBum?MzA#z(Dto?QtBv>DTf(Z?+`;Q6+vaa6%G591D z9Is&m4|omPp8>%^^#7q;AOQq0#b4!OcZPG}Q2aaEmi@Z(VRd!2{rKA}fYxUR+;xTg z(b;ufx@ReC38%O>uigu>lqjf3l6RfWRtp`9**I$yuJ-pv&fGZx|6yyRN2qa7#Vt~3#$TebfCQM0rv25)u-Aqfkuc4xv8;&VVz5a_P8@hX+h< z(Sg5E<$Wlo3@h>it%>j#;9X_+P7fByXOxUK=3MB+>=Z6W&|2ZR}fL4)h@ z7Kj=oE?b?3-N*mYGuG1!1nLGjX^=RzsYdu1EU$o(a3AyoWEvoe(qf!dE&vP+z(`41 zSnfd*U}L6*@n&4};cWXiAGo`R)=&L5e8-*TCaAOhmUjcP>LYVq_jts@i2rmfeF$sV``x0?~fr`^ksPXeabn^5&@HY4n zt_$D6JSp%l#OK<-L8hw3uscGG>}Jh4)Y(l!4zP9vC>5Pxyg$a$Ext$gn&r9SN;m#~ zbtl@>y#{`Un^V4Z!oZ6FP6^_P!d=8-_+BCE@eVnVw8*=YBn{vn;}a5)Tz6#Tuwl2n zYf>eU-w+!Os+`RlNgVVd>AA*BMk6IQYR>C*=_>i&puKV>6jfA2U}=Cw(V)x3s`&vf zwPXXP45V{{p|e8!IcrF=qQ>@1=wWBEC?H{;ntQu-Ws_ACz`jr$e(QDwthii%b0{?! zX`WTL?IE{EB|F4}}XW{35;}H8QKO?mlNP z_lUxk%%aKb#IAc29tRw-N9&+a4zlv*doGY+gLL`%-w-u0kABj4K?s?OaK}?$Ph!VU zuEz(xhjX7kS%rj#W}f?fxKcLZ1e`RD!pZT`0&T5<24w~ zCzi*z0@r|QCs`bZ{Iy5uISz#lf+GI&vA{>0FReJs9?%T-;7VOt+J#p`S{Fqj=BSWeU+Zc!?%qeJK%!*Bm} zO+@KQX9St~;;t0s(&^BRtj`x`XK`;GPuxmQA!-q)9koh%i7p-1AW%n^_-fW)^f!_7 zDB6CDLx7VYg<{6#nbiFLd&5LCsp3{}&p5UG6Be)Y34zR>rI^tZQ>;!Laz=64S+j}Q zS>0IeIw4`#vc4+-y;oGWv$bDYmdB%+d34fS}Zld^W7iyMFLwfC_2Me|s3o4Ls= z(YlXB_geB!r3M%}S&I^FY1s4a6RsJ}Iw+REo?S|(EZ}aIkDRr|JVzI$ zROEAgI+3HGRvj!L8VuGOs?9DgH@2sz)Ggf(uus3lCQ?xmwnmn{%kiGNJbL&^gqEh9 z$rQ=)$TuRX;O=Sv2UE9WDz~_dT?E(0_t5<+wK9^XikeK>4ZlVm@#-33bsT>|Uv@kN zwre*J^vcWEDNd<#dXwMyAS8|`XyrE2{ijH4}>G}6HKr-^i z2vg_&WS&(|Y$@xb7@uwE*zh?pxOzBdnbdyIK6BH`mZiOGs~}RSt;lLWLuWz#h(y0E z?CHncniMN>l+qicb#I~W{;eUH?cetb;WCyHwfs)v&>4PSVc*DW!A)X&c3l76RA_u-JC_#&lg$$YgvwX5ywNa<*GKx zaa1`gRczSWAnI+nWQk=N>fSkKIQz~+(r6Gou!)` z1rZ-510EC@Q&bQW=b3bWY%wc|VXqvOpNS?yCwbn>Nk^!R!OG`hqD%j+!!6*_Z97>S z^&}T-F1N*LZ{x5AC7U4$ZJookZ2OXnmZBmRHN(}NrfPEy7UgDzTzx_XFU#||SqI&S zjAWMtU#{4dSvK4+$=gZ%=yv7c)B^jI+cA;%`@tc-RJm=Efa#(|Ca;INPW&&AkV{8@ zK3N71Nsab~Z_o+$NDjN|Ul#1EsQ=8sUKV%S_>PF2hzLE8yqAB4=(}3JWpuNAXd0EG zd}gXz{tAVnsipw!ALq@sdN53&1iq!4w8@bUVO?lHGDn>lOf@hyO!AKL1n`Rqq@& zxsxVKzk2826ps}>XjEfW-}UiL>I)%^`KcYb9&oZUH-JtjFnn0aDnf0b7N|oL(bKlZ z)$M4$^!iT$LqjsY#CD%q{uJxY8VU)vh zX^KmyezbcBt%jl5msK|ImSPxs{e@G`1UtRq)7mUF%y^jSCXdbWV=;Ssl$>S~VOLb(%{hk`>UU`AtiuD?07J7e;EK zmyPVD*D~C!(O*7Y?G`Yh#;M8bh^r6dBzJKqMR`4MC?^d`BaTV1l=m#i)}m14B{qp` z3sy5zHsvL_`aou=Rr&B_9%J01sQEskbAbkHg;!JIhE9^Cq~l3 zSsC#=;?f)znlKvk6lZRUJfm^`qS&~C(QsQO@R!@J-)vghGp4!GXUV1n{3q5j7;lX> z$1zrks`PPj>qH~Q4Y)eCuuNY4#8sV~Gijp#K0Qu`E$08L+=O&MN7ck5!u-r#hMl7@$L`wDwzHLBJUD_&w^MLCkurZRLoi!IPD-8UzWGhKrt^yx|I z-uQZ$lJexAS>M`-{V`n%XHYKE?n;NJvO+TI_x;w?yH5vb^DNv^|d#XBcTk5?| zvI(ZK8(lSFm*1O=c59O&%ci32Etb`Z;DCxWOR@^>sq?d0g?IQYdHN6J_*83-)A5Ik z+5@IMhThmDW}DndjfzGSnRAldJm{#~+i?jG(b%A%;;aUax9)g-$KFYv z>@W1?%3f1Ds70NW=ntn*Yhl`3m~(t3eM{g>Y6wAOoUjBgDW_oE-Sw=@PUzosUY`+F z!%-Rh%Gnjo;rNH!ftNzjCdyKNCKU`%3Orm2R7EePk?cr%2XRPc8Ux5f{=}Q>3^XbJ z(HeFhi}g;E%a<|*`aLU|{i7|u-kn-?V4|l%YiZUZmfTx^XoN#&9rUGdVh>0FTd0q z6Z_1tiCKKB=E3U4bPd6&d^}HPY+T`~EJivb0!={i6*-?z+77blOr=m4(shW$ESD{% z%WU_(CvBPi%|(4UYG3A)HeqWRv|h;^GsO9Rlju1*A)*dI8#KuHc8-$Opf)P;Dx*>3 zsRKqwag~C-%+Wi{@zV9A)Dokdgd-iPrGApGqP(C3az+-PlK#~Jd8d8x;u$&xF7-}s zYfNI9yuL}UsrWS$359qMX05j3xjIf%<7|_LOcz%-3>pZciwG_R+x5-g zjm%laQRloEt@9dN#OuMXx!SSOJO;e!d!ap>5zc7LrcS$LTk$8wiSAO*c&a#TLv$|h z#jJi9XrkKr>9EA;yRpsVl!DW79{2L)%O2-EwA@p$lsU1gC`-3E`?GY`zvz{T9Jx2_ znVl$0M23cSzBXGmkKP{N%aOMtC{=H-$s*ersa;U(ol?ytvgYJ1wfnyG_yQi!PwI1Y ztjf4o{BNAf@{N}rKELJvJxJ%CJZ?A-?>fh@tLchRbw)OiiyIY*i_ND%B*6WUo?%3ALD=c5EpoQWfr-}(5wMfH;m zIyVZmHeC?q=!#ZT3uq{>2}xt)YL*X>&0Iu7wp3_Zw5V*WJOS$Q==Q`e!F2SJ7TMZP z$LpUGKWEuqD+tIEPi>fvZ3Iq15w`k};AYf};~_ z@Yf(7mYf`}CNA)83Ec)v222`C*+>)x79!|wag8Agt?p=UJ4~2+x&_P2N-6tsKwJV= zg9iPv$KgZ#{qNT8+fu=LX)A1Om|W9(3M)hP{bqk^Y(`W*7Y;Q~tutCLvVWX8G#-%S zvR50B*Ksr7!`*oOHq7T*Tv7OBE}ehGr@*P=dhVExIoI~57eo}=bSBW5!+Z%E>ULIp zRhA8Uhx&=68yXmUoJ)IC>T_rplSWedPE;e=^64Vbx)+?@OUHd%(%tS^iTY8LjLR_5 zTiw!hXg!rsmi8U|7>>RIf<3DlS&bamUoVFAT^gskPwKt|yP>#-qI-rdRw-su{roL^ zF()8m;6vuMBlju@dHyff4YSl6?oGZv`zEELHy#dM(xw}$O=V5;#aB4OOzR7M=fq+9 zd};lr#B+v7#z-;T87#S|1tn`!!7vH+a8;;pbB#Q9eIQR*$*XI5N}k^fB+Q3I7wGHR~&J^AsO#_6cU3mzL$`R4`c83l^Cj zK3Xi;=oBqUy3P1G7#;;#a*rBoOr^M)eXpJSz@1)N z<}hx0{5cj4Q_mQCPxJp+^J~`pBhuxTj&TgP3l{dyu2L~ohGlx)(4qJ(8!WVLK)MB0$y7jF# zgz0J1hhE~~+9&nlQ*Yvn^sA0*F#0`G7D$Bi zW@Mqa`-bWYnJaJqw>j|)TqeHg*lM%z!(UgNiymDGcsnILZ@yuJxm`pXl{q_%8aRqv zN2+lBW~WN#^?T*OX|I#oWXo?a_(zm%oTs)V(KixZ9 zYau>F1Pm(9Q=;Sa5cIu5J9?{`L-QF+nr(C0lLn!?U?v`s{~>i3Bebm(ZvqgPKn zagvE?aB%^>!2z<jOKREeOn&HjilI{&c1nx^}V0$1tsRBo;M#gYhy2(_!F2Am1`*XF?3 zKKPP>@2j0tar)VSTwOH4oS#Q+WZiVwbNHk@unW5!&(2EBZw0>#*WYI>|KAIcotl4P z;^HZ^vI_^~n#W*RObN=^Z1;^7B=1fY;Tx5Vn2^)SGZmNg4!+WlDyB`%s?bh$h*Yr~ zt`e#4yNld7+%|z;SabjUb{6(eHzS)+znnknu2Xs^=tDzmCscg%?)frQS8cE>y z@3!n)eY$Kq%u7Ejy+r+IzVIVnq2z2FFB_q;Bq28vSz(2w^?kPNO_fDrtHs+hsiE)> zeuEVO%^P%s>2&$@JiexZr=1`69VSN~e%@g`##N>Ac-x|E}G_vXOd+6T94wL{TumIxdsWIJuq6>WO$5s_aN$y&fkw zoQuz2UHZXZQR&1;38sq&e`~85|D0De>k(^Lvi}Wct>+C4$uG}oxP5Ks?B>fl_vhS*pn=tF=aZz z>~5&xT<`Nm>Bsr$clq7hWzLA{P1o|h_pO{(u5&zdQJkBRa}KU9 z+tt~fj&6$Y-k41-NLCR078J!kNy#mtUi9<-y4xlqzA9Bb<|s}U43*N!UYO(xR?hjG zCunGr)wxD~6Z^?4v9xr_(FXDw3m(OlNZ=&K!YVVw{ zf8q&)T|`39k%?(kZ~oBi*3+5a*gX)ti9vuIsJANrv_ zp`5S3<}7Ci*Cp|%vHh{V;BM5b^nUI*_Q*k?;au$OUUCETaOk0Lrv)@{mgY%2za3-1 zpB5Br^4gi4g0E0O^y-hY*U-syk)3ae+FwTj?u6#9QhVDBnwugYj|a#}#mJ|M;SuzS zJ^SE#TzvfZ|IlhaLkp*~k0$P2q>nT5x%vDZbc#kk?pi|7?ELJ*aPu1U3O)P$zq#q^ z(2x}LlM^G8H8n*?LyLXtMkVsD@+?qH~!1Q*$(M{=r9(A-O!pt&T}FF^yq zz?umuBcoI(9K*uHqfF~f1^WK{!BQ4Gml({jA5`4Do@ow+3p>5BsI6E2Z~{>rS_-!q z z50{7x8k5&7SQ|mw!jgj!c4-&90@801N?xExp&vAr1?q@^=QXq^*LnjghG^i z5*N@|l%V^XG&Fez(#95ad1{%lOaTH$-Sfi3tpy$HPcQ=n&&$rC*{DlM_$y?oX8;F+ z_Em`-l*A#8vss+?%m-1nNXIZ}HN%KZt&tcs?}qH@Bk02o)MThzC13g!p&WK@y=3c4^wSw71gjbW)oK z?k(VX6>Uuu3V!89M7#;nJt0w;05z#lzn^CBKMm>XMqMU6DRz+_?HwV*ECa^>l6<6LQ*%b2Kx9({18G%i~VH`30bda+>ooJkx;1E z?o-{^-R*})Ot5G+(8qxiik#;^gVa<3!2~*Ua={wVD(Ad_%KpVa#knSIk?PLtFKsxa;$7qrhQ*A3+mev{426>F??h;?tLW&>f&6su|FVf+K$k3ZL^3IofHz<|l!xpX zr2i-s#$mx~!G93JfQAN8`o;=^`GmE(y}b{*ngZVe>K+~uaaz8B{xu5F%N|*CU)=9d zvv;=U#D1^{Xd%JSq&r<94gNqfHx2TQ*cg{8p^R|ave^3u3P=~wHopOR%TRN}w~Ipc^rQZWD&gad4Y{_fM@F zC>1ua4o7pBfl^RIDDp$J2VA%X`QKEikpx0`G7JT1nmvi)1ZQh8s{ILOuebHD%Bw4< zE;JjXF0HMOX5eZsD^CjgC5tpF@j%@cnE? ze2sF^@R^djQ-IxrjWIF`O{F2@4II1&ppGE14!yX0kokY|xbr<|01W6hkp?dbD>Ia% zC%P`R28*BazXbs1jta_KeL*L`Qj=TWTb}|hZ!l2P691==vFPD{Nl;KUDjlAI;3p0vsz2>KSSJ ziB8Ou4Cek8F^>|6BCws17RZR8xZ~F&{>)+^qV}J02j#(HO@1Fo^nRVTG$>0L2V}xC zpv^EoA)~v8J`nX0dkg9eh+&z)L<2i*2t`nE&H%7@K7(0Btj@R=qW0adaM&z{^ku%H z_N;jT_6Il%UnugJq5ros4L0lz0(U;sw_ncq4eTv(G*Ja`JEFPZVnpv|r#u9rP7 zJw5$P0Rbu|CYc~0W$-~ksR(E&NYMWV)*mweAT2O>*sSi)xY05f=O`b84Teq(T*5M7 z{Z&qwS<#+!;0tv#wC=>8P`x$GXHq_@B&$DAcijFN9NH z81BKjJZP`)OT_SHup0N>!zl&gIDNEYyM8WX79q?ip9TsXlkhBT*k;^_thrnMV;;g3 zW?i7WVM^aaxDC#<7IXMW?O`*#=w$Dr8mm(XQ*D((&qKy#0C6+kN z|L8vA=1M*LZ;u3;nJz)Y7&uj+5B`H3?EpRoC@WAizcqoeWelbBh~|a%L^gYCT1Z!E zs9A;8{x@BL7W-Bc(GAJAz@&J59R)o=5El!|gh<0$5`bUQa^^;+LLVFf&zn6?Vc12& zAi>hofk5d_a9SxAMEd4B?Iu>;f~M$7NZ(a7IMl8c4`~<<+EOJ)*9(+XqQo~YL+FRo zEAtxR!7ReQW)--J8<#p~3AaLe8zR0KSXJoup7E94EE8z7%|`iDVDi8mrqqMeghu6w zXum3kFq|PE_x>-%llNzJ&axQkC=Iq)%B*}FyQXm>usxbL4Y+lR(DM+9rJ+OI0;uG846PRMeTiYdClUMM&NT(92(d9iK|#n*KV@{d(roN(gkXVM zd@O8i{GaSk_Te2D1`T46;d=)MgQ!6f=5bvP^ke{4SF&1hc<=7rABaZ?Y)nNsivwP# zdeYbciV7c*!@hn06xvauNlnqw=H)_Thrd2TuKm0b5I1{s4J3gApH}M#zG4Y5@{!Yg zpg#caJzjG?=(8dPp%pS&#cG*%vqpfQegN@Hi_M8FqG#->5ce4^AUHZY6PypkDThkJ ze(-zm!I1<|^ch-G2KDmJ&dyuAm8Y*Nk5OjEoGRnZK!db~c5>AG{HNZ$E^d5<@87>i zY^@(Ys|biX8~~wAD+ocT6f^E5$S03Pu6`R(&Xb^YVjkLO!MQ=b;!ufFr~MF2RC8Nf zFYx=au3?geaPbf(i$fV{$&^1gGQF^yG#EJz*&=(P8x!cPMfkPKPtCW$RTYweI8^N{ zaAoj?j#W@NJA*>K($5E(%7E?%Vo);G54OyR@>y&=nEPi?1{lY;KaD9o*U>`F7$H_S zR|JP1n0~wpn&09qs(p@i?;Zu5zXF5Oqf>=A3eF`lhCt47(u8mcxC5iHr!$L#>6FU?Uwb}OQN4I_33?kiZgmPO$80aP9>8W$ zgibtI{l!t;3n7o%BA5jU*x&R3BRLuNDn2&X<1X(db>Y9K7(vWJ z9Wdh7XK=icj)C?>{wzrM0JW{0;>I4?2D^ZYE@*0MDh`UzQ-L-KUHoLg93tl^$fqeR zK-n{$)h7-qyf=la4FM#Iz+kk2ChChNOPR88R>^uL=ut{I)dmDoF)CmK%7o$cjnH$Q z@G|z7QxOq)4&jYB%nO`ZAqVl~R~Ew`(Cp`~|C$h-Faa((Y-c$*Gz99hr%|X37&|Tz zRAfByO9uT3y1(i<^lYJIWtE3oLKd!ju_8i$V)>v64M+-JXo>VHR1~E`$awwQHAb#` zXis=w^j}0~f*A3D4q z3mPo7;+dxleg);IO*;1Rtc%i?qaa58?GU7j|1eHt5rj3jMtLJ1fM-TM6%QA z{Eb8L3eUcuiVdlLzf};jr)1a376z>q;IcpN`KEW zhM`6&c9adJ+x7(`jk)(3PYFFO*A!Og?9zUtz6|dNtK(@>o~Ekd1X0=3zinV-EnNC; z*gs!Y>ehManK`;$@UZ1P+BoIn5i{6^=$4Gu?j-rxC)@ZCTAwJM4%m>0|2~qGOzjO% z@kUQgxc!#Pf9YB3JWNsJj|y3`C66+jgMMzO>GZR)-{ctqha66rkp*=|Ytoe9x9XOv z>Q*V?!yLtx<5U>!-1PlU0DWJ)gKFr{%JE65=XP%^#t!mNmdr%;oOqtTCpi=go-^(B z9GgqF|H@+bg)bs9wlU%c*8YILi#i3?@@VQEW}N8L;Z}P;Q6A~Cvd!P8hKr=@B|6HE zP4rqX1-FhcZ?Nvg@Gt6mW&6+g>~`(8ajd;R;b9jhHF&&OFxS7>o5du*lCmx$orj{d z*<@_eWlB4~U@SMzmv)KKwa#ial+P?mRPYN;&m3o)ZvM;W)9b4!u54+yQm5F=TW{js z6Z>CRnT__fzh4dElZ;#PEhs-MBKx3`rr>uxjWeLqFF2CInTDS7(>|iyk+o~r+2n2T zwqbP0++O}ytPZUC40MdF(xS4B{H;uECJm;r@zB{JZZ|^bp5p~muI)O08%13MR*b7s zKE_v;zGGC%j_S2=2W0i9*$FO`e8Jg8Eu~Z6)TQmHC}z$($&Q>}B}#3&fw9zed&D4U zTT@YE-#RL_mNazXt=xr%@71Q#?NjE$t)eSeg9bZN+5M-DR)Y*!GVPMTvKl5VT^>zw zP;GB~6=+XiHaFq7I=`TSp*WU_!kRXaj>GOx-fcYG6504g^-2)Oc5Si$<%t0A0kxTf z#%?&R2`Ys7&Rx%(H`!hjK5cXvY}SQkH&H?*>JkzV2LZ@-EBF$3NUpl{Pon zCgK)dMRBNZKCfO9QtfHnkE|3bWZzLpS}>c>FCysSNq+CaSNPPxpJS_Z&Gxzp;g3e; z7VG7%oy_0)4$*eI!z?+us->8d`DwnzFR61(Y@k_#oQjGH%H$^6?{P#=V<1POL1QtM z5&hZ1;qAuLmCl1aLcN(eYE;s(qsxqzUdiL)%{)feeB!-5?u_A>gxq~PEI79Fx8&h= zeP!S4_I`&{)Z5pJe!J&)BU~I;G`RCj%nI7ExI!HkKVZ9rr0yEI45%gcWexq{i5bN= zZ^7yc=2YEYHYMFUv}2oZ=;my0_N$v^nv4vd7%JG&`M!x6Lcxj1F&&~2u!!7fD{rT#&75pwKF}^#LdEwcywGXzmmewz8xKKjf63j-DD$#WAa1*ESPMyr&^@Q?z3CC}5KR z;K1#l7^Ct<6O&3sv@+GJQFJ z!6o?emhr9U?GuqyDyxrcl;Xoq)zCB_fe?15m##9y7Ix!l4Rk#o4}Bz4^r{ zA$B$kzuzToFYXXt+HsX4G38HPJ~AO1JfOcdl=~u5)Vj%HDSZl~i+5av9O3JQ9d;&N+T2`7ms! zax-rh)hrwA8bC2|DnY@18ik&4B731yIh8hagk3nty4!bm?00R|_Q6!h;7~|Jmf7sF z3I<(sZ)~M={G6HTkDQ%YYgThYsvc}C9`wi0_Jvo=!haaY92MFUw#&JWDx-$i3KGY< zmV>U;Y~&5^PyOEc(&TtB(mV9cR>1-M)DE6hUjGJbAG5D_PwkWwFzthyEneBD&7p#A zg7-i39oxHj5!bexP3D^?iJ}_Bdw*9^)vL}jrg<&>(B2Jd)uW$}z)I3Z_v#d-(>HAA(L`^Yc5={jMI^10N8uG5+o0dBQMjQh+1-UX$hD^xHN>$yzZr}f>ikgE zkVIUDpkMO9au}~T&C;~_$5N?Hpxp$q>9G9b`A~2L+)Dxga*1_T<7Bd1O5*ps*_r=z zbYp!-dbs@?tB&rcui@9XX9{Y#blxOVR#v>s?q`>lB?&I0ab4_+Dl&t2w8eqd2Negvs|^T&fP#RG zabiX$2?zwxW{?B{1(|1$fWTo41Ob^r2!TXF2uXl6C_{(=MM5-;${dDZNWd^8w@}aN z{&WA_yVk8$|8}iid+(}MRp0wP^?pt~ZotBuq(JiVN3inb!wVBOxo_)+Yl7g#eaj6_ zqU-q&e(T{j@_QfEPPAuIWRG^;#+$6j-m`47F$6D z<#XZS$oN^Zyf2+5$ZGN;K_6~aTiVMh+7AVtp)4d&@&i)vX(|E}OhdB6V4u}71RjHw zw`}EodAHKr<=p3P{~JBR;K&>%M^Oj;{NyS_NM|Ps%xIBdgcQ?_oF^nMFL~InJ)d?e zb|l=QVpJLpdBtRTQQm26&xA<)LFuw(122vA(NMQZu&V8$HEkC*d{s<6~Mw+ z7Vn%zDMOy{Q>@4upTVe){UP5d=Ii>!yiV~?D_W~u$dydai0y4~JZ4L3T4z*^&!^xA z4`(Mf3av{AtJ6FDmGkTmEJmde!^oH~aUp%zHU!;Lgp^Y_0{3S>1@6>Z91EIU`s)HUESg+LjlwhG|uAm~l}X=z{eK z=!h2b(cZR_^rBDI&P7g@zcUPKr()6^z^RLEB~sm`XJ?(D%#Wm$F3ZCX<%lQv`~Bk{ zWZs3SqUd<8iOrqLh_mLPO=5;@;JHrAq`?p3+S~^L3HK%+RXkGVKG<5fWk6VK&MK~| zIca!UckS64ex!BUz~Mp^l>|%;ZeQ>4Vw{#d=lLFjk{W7vw?v~Ic#BuzEDPtsH+&!4 z+ml9Wz$1B2O(q2>P_$chi=4U3!cUq;%OxeRyFp^yO#P;3rcm1YR}E33A|=?J2%)EiLbce$&VvK@e>9I-#+ zOAX(nY}FFxq{a61?eQ)XsE^pI*TMME-+}kD^dS0jn;&~;H5xYb1Y^>*L@P`xl zY%*VBJEGW8-=FNFpB#7@`sCjWnvv{(tC3e=HqSD19(>F9QFHrN@JYEr7Ws+0dO*!I zfCz)O31_BfOw}lmKgn;KYQNvSK|3l|b~;5iP;tMkX%-<pUmE>+ddbT} zjiO9h3ZTy0qsMD+7>se$O}o`rr?R<$);!K~w7z)NGLs$@kw3tUR8LIcKqeB^H9GKCV3xK{Fq zkeAOCOv-F8rOmTQ{M|S1xf#b`z!)%SxGT#;jY@?bN?ovISV0OiNe^m^LLbzeRhh@a zR%o-#&=EP&+@>nvpML!!EI96W&#^ zkitHZlkPwdd5f@!y{hvngD~gpD`3?|GYI_7oAUkznrwv(<)x#{=X)cpTqiF0+TMP{comj@QLLKYUN5qkMxF|qTMu=6)4@Gy`V88_Hr=Bb&QTD(dv5+~Qq17KQ-X4dPw}8${qo%rn)3~OEAK4dCQaDbKf5vUj|%ea zzR}6N_BDxKOr0ssf_Bk66vEdA-ZtU}Xu}9gCGvUWM0^{1J{>%m1X?OEPc$G{-JBH| z7s^CBtg;Ks$dN0x9qM0##s|qhAue@b8!ux1-i<$9%*B)pXFjF|R zt{50JTS1;DDS0;2+JS-|1d9_&Je*d@+*^9Ix55HZA+3_&SAcN1zjiW;+rDpOrJKKU&fROt}b z*Rh|YmFK}Ju71O6meH#OBa{BvNXa4z^)&}Zj;g9QP48;6@X&B|Vtka9~ z>8jR_kVwqzEEu`{a*rX+sbYVOV;w|364MChlb>0=9f56Qc{%lI-pkZtcn|v+D8Q?w zORlK7xz@_(5Q*Z}t6hp0e$L=!?n%vWFf14jwi7aI@7qqE^qF`e;3$oh-f)}B<`5lA zKNK<85uyqan+y2^n~|B%G^1iLJgrpdW5X;9$u`@9!9PZ*`7w7T3te=LLk_mM*6jv! z_IsE55RDOUcL!#_n5+;}?O*@KeZO)BT!}R2kqVrt<5XEB#aUX$)d9w9x|DZEjWQ)! zzOFt3y0uzzwMAu%_77}sObhel=D<_>*S=ZqN@`p0$F2IVAaPW2W#-k6)9+^QX!2Aq zfIU0c!2S$OzK*`liKybCeX56{>c}aEJ8G=h2gy#Ty2z&l={P5Sex(s8rSkK_LrW4y z^$iOxjHg2%mgr~#C8lFu5RLi=X4=->Dnn{;eubKbjPSz2fxK3FM>jA~^QGri)f19b zRFDg-e(HtpMXXEFkltpqAWF@82t8@1_89m8XgUcw-CIiW6;=vmWRw~0o#79AaD#)4 zvOhDTi3>5oO8ED*d|rn%$am@$HOOGom$Qa$Duo&~8S?3N2cgZH2SVTT`92oS*Kps^k-a{2WRn4@P02vvX^YU<(1vq*B8yd$q`@Vq1GqRk6(ZlvFVh_A1G~a~`V3 z5(}4Wgtw197ZSQ5+Q965qX?R)wLj-;hekh*6@&+W3TO?uTMAxV3yPT@s@xPqZon$+`;)ES)(QEfX)V#;Z z&*4I@*O*g{IB43Lx;Q+mUkQcidu0*?|Ex-ump%e?49?Rvqp66OKiizKbgP3{2vmK! zC$PEWF;`ntsQV)3^WDgmcs2sZp0T7y6-(7b9KBEda4ZKti5T+uNOk)9b+UflyXx|3 ze?I=;hZSQ3I}T(?tgUx`@-)BYSnF!Z-VnITIHu_CbhAqGIM@D0(vreD!D3od8jW6? zaO1{d?zPCYZ#s?UHedN_)u~u!Zmv=Pt%vMr%xe-_`s19@pYVIK>upQh1RS5Ez2=4N z7ULln>&fgHUmN!eb{wWMv(pNQ z)IR>j-6AD@zYh4O)2!KnDZKRc^_&5W>NTNY^te7}LoF&w);!)&Tc=6gH660iw1#d? zxJAl*c3t0YC@8Ks;BUm7tTtR>Jx?xtX?vS>pMlJ`H|lu#L$CXG9XQ9~ughy0t}XaF z%~lS;ormx-ex7U>G#PTqOn+Dl)mfFvTuc5iqS`?36^w@~_u^jRuRC%iG~Nq4U~!&; zRr;TVP17uTJ;qJ4S~)P+(TvtV)aB^HfAHEmp2=etH=0}sU#2K%)cg=8^-_RrL)FEh zi7<#lj4K?c0bOj7^$ZS$vQ>n(<1&5rmrcYEwQgU~U+;d&>gi9b#$$tBM2h;1^@~1X zNAA`{01KqIZ%0;jhFF0Ztz^2#F1j>@@ec2DF+jc0{@VHMZObrq1AxP zOJ{fYDQO^=;w=K$BITe^UJs80#6U6-hJ{uGfA|Al=_h_P16nhn1R?&@{G+G;J#Qxi z0xEC=t}{l7z}Qa4ex2-A`i=nr`gk_}{Z8WU+>c7u2Aa+k3Q^TJ=m6eVtct5Ub# P8J?-3<#p`U+mHSmlqq^^ literal 42305 zcmd432UwF^_$L_kDvAOZrAU`19i%q_l_tFgBs7s8dhZ|>kgoJ1ARPh;y@sM9(rXBz zg(5xl4gtb^+&i=LpJ#VxXZG3M+0Vx(C;9T7^Pab#-}{CLZB1noVp?Jl2t=atQb894 z`fCCNx`w=c3;2Yc{P6=D+r)>M9ltiSvyrq+Y>-V>n#FR)ir?=i8Yl{Y0N{EnRPBrWcL@5P z&A?3e?|6>+h1kEN@A4lU{~f&zzYT2RpNovV>e{~}Mi#3Z|Bjw2bY1NYxO_i-Ci!=? z^Y{P!gjN1~^C}6C?Jw|$OJJ&h=lSliJdkO;Jm^0f?*x2~KV8X|Ilba7aD6W&Ws)r` zZ^ifIZSNGbXRvu~PNMS?$^N~&Q&?Qg8S_x^XWsT}_5YbI72m6(a&SylYz->JeJXtu z{4@GeU;K<_kBO=vI`Z@mz`0(ktIa)Ae3=2CIwv_L!<$6R@ZSZN<^GDEx(U3#Wt$Df zKOHU@Y%wJT-X`AR`)~O7wl|r_89vj93{N+>tN-kkyytN0x2 zSAVKruj51WzL8ISjqPKD2j{}i;fEs70rE^^1>xY5Etf4G)knk-Wp78sfMD4f~#$EK+;yL4c@)eKC%*FW02A*6!Sk_H zj;+S5waG)x65Gt0-pHaE#I?{qMFSs<&J3V9Y&+#lV4m5=>%q*FCWFDF^>XpLf)(TX z&?2IgAZ)bup|4-E8P*fd$RuZAI(4o^R$rrrc-7yu!XHGBdJ%FN40A5+pLM+k`o0uV z`UnJarE%vv+Fuc|_h@>@vW@BM?@UZiD$E;ZD#Z*aLbfG*jPxR+vPWA`M?H3ekc5;i z1Cu8C*ip+#sco9Wd(!zd4`zm%cvX^>{2iT=)lrFQQfl(&Y;B?0Tt7rSR?;LR{n}C- zx3-Va{V1(o>vSPu>a|$yh3iY+Q1elYyG*k}8;2DaPjG$imOVStGR0^w@7#Ydu)ugSFGnjyo9;>w&X z=DLZwOg8+ZZrR@c?nj<7oZ6I_liluiC9P9w9y*a*9*27AN9uB- zcdC0CXH4YHeP7CK@H5PuAxGG{qcJ_(c}Z(1T{6ueQ+Z?Mu0h89Ce!U%-kIadC=NQ5x{dn z&`(#02X2>_(6+{>Wx+i4;@4B(kgmK8-5f1R;+rjcFXhPqZ-3Kn0*w)yr1^0_x{sY=3m`2?^F{vn{?y%sAjLXS$yeuVzn%> z)SP-Jv7nB2v{tpmxW>Ye;K>VrI@Yf~+GDG%=Ds9i$|w#Ki|fe;;R+O z|6Nh=rzcj!uy?jmS#K*yU%39Ob{MNXhnNQDy;ev8>hwjWHuee52CR0b^dL+QPU8@) zXOb>2NwX_ev(LeIzQYwmztcQM;kdb^2S)!SkrW$1Mfbb)G{2%rBuAKCG<6L&s_AWo zDBbnWGDv1MRnC)7MFMZo^_^)ZU+lBQksnNI|~lp#5)$APVP~!G5A7)* zuR>b(u=V~yjy|F#IDxj$_^Y}3(gUl1-lfX>ux|617n6`j$f>Oh_Fs=GqFBp5z9|d! zU1<~DL`Y2v`$cc1G4M#mUI)nhVv!|&-MO2DPE0*s#Aj>To#$`BeTZ9M9{AL5oMvrk z5z00Nd_EukWCDWv_iPh={Y0hchi-)TxvHVbK=7XvO3L?(o6c1(;vcTdD{)`UUq2b+ z)bMpoCulPqZcRnLy$bj2ziU~qSd*qIz0)$2*{r=i6Efh~t}IH_R>a|t_k&vjUpm>( zVZU-(>L2aL17=q*&RlfEmB@*ANVIEzV@3WjdDKFeGojzX_kr*J=f$g*W-Hu@;{v7l-2{`KLyzyPUe7Z@|xiX@}ZN2COobbT<>h^LVW|o z9{?m%bJpu)aMUMq)@IXtAkYqnsOjvt0>xLpP3~u#}UVOFqWivO9;3(_Q=PI$zdR3?VsJ2m-JR-FH(c% zwU(Q-8Bz*onvFU^Z+qSXKE~Bgg-Ovnx1!2#{6`Oy_AtBsGqPm@Neaj(efs4?;gt z;j+ui^dXXo<-G2}**W%dh5l<8p6t6AH+vl1eN+(iJ>V)m=8WBb7L>KBJGOVGzoL6) zh*BSk7oR~be>OfGN)1c8%>DKb$Kupqj~DFD6(b8NZ(q|)E9B}#7>*7@g4^aLQ%GG- zjC?dE%QS+$ye?bBaqF0{)za|EQU|=O%;}5(#B{=#g)pK2eb+*trkLor*l3QHIhwkX za#W;FBKh^lTUkTRAp#>MYL8QDx%WC4+*1qHSAtuDVpb6m{}(XR1Q19pn#bC zCbe@J*M7K$Z_W7iu z=M=B^b?gq)X=Um%!d_X5cy-uIHk%Hqz3iZ8^7f*YQRNdd#4W5zdkst5Ldr+FzQB`? zup<2t%ph*Re<(2dhk~z$@u0VR7LV-@A2=9P|Vppe)x&NDNbE#>WHZ0zVAEm#71`(SpsPJu3kP_zO4}Pmn5!f?L)uJh zh_in%QHXlB#P#^0tN}X2bxHD?VsdLsDU5d|dRK2$;3QI_d7Sh&pR*! zv_rn#7ns=qkJg)3{0WvujBn`qhrCJjZ?4f;AuG4Wt@bQ6ZGE}vAM9|H=8p8-`_6{9uYc8J0%5#AnQ9JgspSrrt-9tMl3t#jq%!-c4l#iO z5D&8D;Bw(3!x%piRUcjz?U9_>!=6Q#(aYU(PgE3-h`ptI=ta&4>n_PM>c_T#M@lXzxtUXg8;|?YURu1P#mBja zOEaF7a*H!JW68({;A3OrB_XVO(#uNrXNpY6gD@(~OPE8g0ZJD645SYWYW=IzW9&MX z(Gg9#AUBdo&Zo~_(~4I&$ver5I(%6_Nlc6=-HHF$xThl*){V_?*9LXZ}>E6fVToO%|N3%sD>OneNSn z`2>}gjU-FTKm?{e^9Zs*F-Mu2hDsyM@QmQ{TXIJ&@x(I|jj;~HCk~#QSQqHb;F+h8 zsG5mbpdB?Z zOCh>WEy;4rUj5RQCO)zDRh(H!y%|2RL~Kh2w#XYkvOs3v>yy12k%3$<-(QHlou}Rr za$Mp35FbzFC|UL^F##XcZ=3#P*CCVN(jT1L#2NE72ygYuha||YE*LMPSz|{%n7D9$ z0+U1_E^cVnnRG06kWlKSnE9=@dC+{-SIkQYM780-wp@`)28xoxug%6;wy>s5gdv9Ts>Pds1RvNBRcGf9 z);-po;>qi<(=cl?c3UHr)Qz)WZ?+v3-uP^z?#p`ocx2>d9K1i?K3z!9*8WrWqWF$5 zd#cUWQ(&6!*V_pe82A#3k$z~b(WZHgo zv#Q0qEf_haTmok|*tdlbVEl&~c~m{>n)GeE9v;L46LH5cGg<7?Y{C)b`> zr)zd-HfnmanpjfXW=$~d#dRk)FS1|kBlW6A8S=$bHuVB+lH)DM93~r*_tav>=mrOd z=82JUVqQD-5*^`V+AEMSwiLpvMuP{fTHrGa@zo4NqSZr(paWF$Uy>Sn{4~(X%1~Om zHAt-_zj@cD6l3Xzx_vTe7B=CTd00ksi80|TTZ7O9 z@tWItANA_iT20(Rw-9JGvVAiQnq?F`n6~nb*QW+1FzO65{t7mOjOiDXPvE|_ZD!4n z^$iTGl3g?ruOJ;oIxwE;F(a+nQr=M1 zk?tkUPY)$9$;rAlIy$YuewI@9vw7O_>0QRb!YXJqy57w`^3?qURLTd6?w*IxrWCdd zE5iyM+Qb}t96BJsJycba@?-b8G+O)`W3PeCW^HE3|5b4=d~YEljRj ziJ4Vyc!jJ^pDXN>w=|tL#j*R^!9m7Szghf3lWZRLEH{0EejR(LVu1(-_w>hBIQH-NDFA1Gh~-$ zkJQ-WOH@)#k9L-6@jvBp7Cvh`GqWDqPi%iXEFBN`9N3o-6}2LA;H-Orpc{d-@I#n(V}Jf@obXPq`Z`{H?M!ONgOp2 zj5oh2>-oK)-OXP2P#azS(*2kHR-`X@ow1~?mRR>&KjL@hq(fkUoX4Kktd%1 z0nuWO+uIr!t&&Paea{ZJ9=aVh)QpYIhGCMdx1C#_tkE5Y1S0&kUKI;wZww{z4;cRZD47|n@-7|HFs zVpWYgjn;Zx(}m8@7mf8(ts7soEBRnN$au%f3RGyHu<7>@6svQORQt2XHZupuzF4kE z)B0{cfFx=E+O&$z+YziN)9_p`b|&V{yNSeq z-1H9_Chws4QMHg=swQ3$Vo+7_>E`%e_)OlNoegRZi$gd*^d2yC(u)eq**o3r%O@|M zVPFy2EM>)8#K{|)TPJ)iI&4p-DDl6G9h%$t-7VQ__~5Y%YV3gJ=i{Q%9XBNJ58QB0 z`gtT@a}Mqhws==91|$crRJHMr71H49;tZlWzbF%8qwy|1zsL1xwx$F{nxbyl^hQ(4 zy6L{iMSr=7Y5ahvp$L0gAzM2lL8VZ?{xd@^TW4pQ7jJJPdT`b-z9jk}I+6)h{-o|D z*2_m(!{0gGW{U|Cb~-9@4~a=hrGy!ZLG7cJ(bLJFtYwQ5 z^&{3#%0@;rFzXLNbCgM&N5mj)ovL8WNI^iNwdz1>5~NDBv5%TSDfQS`B~4Q@v(PHt zT`IFtAVrPe&{?g?gw)zRv6GLs>TL3^CyB7tLn#z&P)1E?pe;k%c&3@g+N6GDw9@76 zdCxcjhP@DNWu?=pBfVN5mfuE{6-p{uh@V0UDqWaxO3)4SM=5CN1ifZR$a}G1y zkq{}ur(xL1G$LPPI%BGYuk;8S5C(?(d!xe#vANa>VGf&%*nZ=*% zl}3CkXvLX()zQ{Z_F|daKQB$M2~Txdr^l-U5l)CbrME9AtPrjXpw1dyWpZnSwrbkBgm<3wJqB@DerY?Z_{APoUa(K?M(eE_w~Ve!|cYh zNYqli@=9Vnw+2JDPP+eoFvAT{b0~1Y=O?nsHINln9Yow2sfe>(oqyc6o+sEqE( z+r;V(L`^h**EJ@=K?+9sCM$SCq>>F7*9|76>c+qq5UNH^dUV7a?;CmeM=_0p`rT@_ z_PgC8@N`M@$}#??pv5HjdJDk^H6bC*eRJ3RakYX?O)4L*J2j>}oIo1bJ5|cP{t{Op zH2dyWvGWD2YH?!>A9FJcmfEPYUT>v#Bn309z7DBWiMP=6{TBDtK224OUCp;4{V&PG zsUP_skljVQ!5{g7@RsOrQaH@d!ULJ`13hSLougFQgJkX|S)r4Sj_fozYf+UI9zB*C zN-|oQUVa|vzW&qgspgbkq~x=E^2sl*FX-;)-|MLS3l=QOWE8l*+Gwb>80LM`e0!#F zWs1U$%jD-mp5ux^Q7n@TbJ2W})FCWj^UZXzs6q$>J)*^29R&-7-0S5b-Be+d<1MFPskSiAoKHB_HM445a1=P%|dD#0k=l?MxF=e?)%T zNze@$4Usj;OHHJg6xMRsuIi>0-(%Q3)vI5qD79>pjnsZerG64MW9s%yWrDPT;Z!kO z*0qgr5w3R4aJD$>V+~FxXW(}DqL%lw=e1r@*CcVjEUYBk8FPH5m-u-z9`yyj(5YqLKe_2o zNMM*Wurq&^v~$9_-YBXT;IpL{3r}L1q6i7&%?>K&%xT(n$m@t}Td@U|$H`6Pp8UZlBp=YP8tnP=V5@*C)SasX$seJ5t&!ASbFLDUNBS;= z=_6hIEI4jGs{6(wa@i=x-=^R`Pybs%Dm2|$E@4LYnSDkP)YEF~JsH#+F9&xHC`Ht% z*&_RCU{Z^GK@y0F`-aU3X@8Rv+UMRA*I_V=^{u9*R*a}mZ}&d(`Gi!l`S$I##d`mI zOOn&GheG#CadHhK7mF&#G4TIG^q*xZE>B{kL_l1M&`cdp@Z2L>tzt?HF)&kxbYI_(D3kA=n{t; zXx*@2$WESf#@_{uaRNh|OwQ;5d<^>Rot8Tv8Kq&^Q0e5_F8_1Yi=cY5K$nPHvVw*( zOy=Km%K0%U4In#o$I8639a#4<1NC@1&S^nw~YA*I*d zJrKz3WORJGr#I#iS&m?khG+z7;-*& zj0}|WMA)J+i~f)dUVDf#Hh)_1@;L8N(CE%Uh{L3*SB>r7L#(iL-QLp+-fyL5ulD3x zWDrPN!`&2fyBl6~$2rSw)A9tN#r2hdI+FJVv0>`QmgD|H2s+LW5;mt3xG7HCgY~q6wv2ER@GDBY8%nR)d{omXqP;1#iKnw z!pY9!bb7UA1mQS5Cs{tc)Ty%D=yX$Fv8rw)Gh~zh^4(OU`dQ#9_qt?9+gTLk$wja! zCO7zVHgsZNW+MCCXv%!+`$A=}Vmwuh17a%qKxcGY*wdl7zSrYU_XKQ(ReG&p0SM(s zjZH0uwh>ADi@`FYex;g9W^k4Iv7X~!n)->660%jbTf1YpQ=Jq~}$X-9D z7Z$3J&Vu54T|Q&kd+kGCB|Ss}Pj-ke`9KYcNu5D7SG^fY>ha4m?oo`HO$t-7g*{&q z5DUh6BJ-8@czLf;qzljWI`;>#j4a=mU%1TIF9Zn1PvJBI*AVY}58BO;O&Y!`?Y+$O zyAgI})P(XF=;o`UHuZ_Jo6-+FS+FXZ;Nm?w*}fCS^u3Pf=(cs5n3iWJx-(~g6YEt; z@oTgs*%*Po7lZ@#wmIt>D;`drY%*AZG_~z=w$n-R^4#U>ik}bc+A-e}ry>tyO#ZIA_Ux21?o7yk8fbZ8xC{$A_?{^1|9$G_Rcp}O z`?;A?S2a4IiuJ#)m*~HiwyX}l0kRr%b90+b|GJ{;84Xawea0^M=GX133L9`~EB#ZP z1FCnJo%!|_pg5PLKXV#7h5+s*(fzM2z+KVz{|i3;Ki1@hyxLM1B?AN8X{v51!)NaM zHI)&L?7(=J;Hf$%bgcs>%HXPBiz4o-f8bmEr>H2D%<(iU6BCmWZzBUK(2m0!8=FFRcQz?=Sx2c$js z;$gx+(^gD4(n`ALZ+Hh(_D0i+!p5zEDO`8|>2K3#$y18a_4fAuvMo(diVZ&7q^?u| zZgbwbYP@)>ny(zk2-C?^`m(L8ps+gWP~P&1{W`F;3Sb%_kUk4ll%nPDXs#ORVyrVJ z(`RNam=m}S;wD23UA?5`HbvXm+B%FD>zHv~^#k&M_;;&oLpkqh#hn9Z&To;>ataBJ z+`XFi`^SH$eF4n*rQgOwYiiRQ>xD5je`3iWB^ZLKs67}AAb$W8r)!C_R*%^E2?FQV_zW47w zp#X*MjV1sWX$9YWsE@j8K6>-2#T59;+do!TR-D4ZdWH7ff`XM@ers!MJv}|#o>{$1 zy1KgYXVTSGRf$O_mik?s;OzP$>bsH&-Xgo9O7RNN)# zfoJ(Ww3CV&zAMF-8I-zti64-O74;P$Wx zO=jaWu2T=QDAvlBU1>3Va>5hQ?9Z)kCbQICn)lT2Fc={pPD*K1c=u+W^%p}eW@hH9 z2}fcQ60h+0eqI!`wA>l}g`YqF?bAR_LBR%$0d7n?!;ppgSAWHG0$OBfk8mzQsQO;h1Jiz;QAg7!E=AvU7#7I|aQOZ~pa5 z;fhuH`3>*e6B8)_R;|wcc;9iWnflMlN)aF;pVegQxlCyW3X1&CpFh7SdjIYn1u(9t zsCe;S53nx#E4MtS#UdkP8W9n(We7;Kb?xVJ_Sq$WO6kz{;qkE{z)Rp$fEXMp1u)Pr zH){z*-JBhCP}Hvge@6GPEBQ52wAVhpcJ10$MU83_H}$5^K4f+2qYogdo`C@aRn^G& z`1t7`dyOc+A0Cd5j;(9Wukq7VZ&#{b@e<#?tM}?v>d*FeN=C-5EZJl5`1#Jxj+&6f zEzq292jvqG$n0}`d~O1V7H+XSK`IDeqnRnSMA$CTD;mhiPE9Q?D=XV??R2iE_P3nL zS4r4+p57%LQb90mOKPZl6l0!V;(ELiB&miM3GMm7|-sobp z56P&p8_w-A;6TH`th_howjxc7G&6OB_(es*MOxY3L}x3v-2L`}P9%VznrkP!3jl{b zTL;5Gek?J7Lttht{;ePc;G$qg)rDs*ICN>s3hp|9E-(XnV^Sw1B;>g^pE#MNr0y@o55fV!7zOtU*85O1_8$(u9rGs5CDIZuiipxpE+YTe4OZ@!#+xYmaD~*@4 zt)9>a#B#+Kz;xWR&uB%Vi&+R@@-wbhpwG?ustF{84G2hpS+9K$y%8A`16Eg$e)HxH z1sz=}kX)<}<;b62uep-9Gqwq4-qwm}-cvnXLn(W$kM(2H7v7Nek%A~Yoa;-yHz#4H z$U1;z@J7|{?ygkuc>_}Ve7-&0>tx5G1JERZ!%~4KuC2*#fF(eOzJ&tJT9GYUZUM;3 z2X@}u*;$lp2n2}oM*#26?E%B>`2D-dI5+cbveqF5U?tm9tWPp#qzqUqGCH~sL!4)i zDkQ#hXLoNB^lVA5K$Q|)qN@n_FkbhxvNB!fH)`}4tjV>|~CsNKpyCjJm( z4&)a6>>M0w@wm~F1=11x!CJz$QgpH%NE?Vl8wF}9{3|UfU%!6cCgZ$XZVQX7wHsyu zG+aNP;{N?yKt)RcQ9-dywZLKpVosAgPi}#p6}$j8+_3spv&b7z?A28}!2C3j-kT#w zn^V{Yygp#aPLnk{ov{o6u@c8ISB9UWMhfC|0en8ddVMRaMZax!HnuA=PYEF55W@LB zp#>0e5~2ym?X@;?B?YT#anM`Ct4x4{?PtAAgEZT2rWXqb`+uH-UH&A?K z05UJbtF=s%(1NSg)S0b;jaH9GpIT=nD`mV}i>t&vlMPbw21!|18wp#=?EtCdx^ag%X!VtV@Pp zFv+0}`uP}>Ssf~Wk3B4tWGGzyPIg=pTQuu4@p3paoW*P5Pb;K(Qy#0G7Qk%yiUA2U zmht$nu4x^#AF-+qcy||dpdf7*lwONHyITA}V;LHZEfgCIsZ&}zf^=kk z!pI14a41WAZww8}U6uhn3%WSO_;1!?tkMf$wDoSd8)erUm)$4ny`C?7kgu(i$H zMDbnpX(Ymb z$A~0NO7$|(y5Lq@Napnx(|mWUS_t54qyd%HJe~c+-PNl8J$VXX z!?qyAj}-gIUui{6YL)nhD9M6KFIy%Jz0U17J?y*3@EnlRPGyoq0%F_KrKPttSOYa0 z%OG77c%qX5l(=B*}xUxYOKRx$>kSOA)rPM3K9rCP5+Y#DU2 z%zue&od3phWh35|paO2Qj&^QPzaiGGzqf|!G59=ryr%NqTcxTLwWeVmGjKkuyfq5q553Ekn_r0WQ=9~8RF|e6!1UB`k;|ME=KT_{o8->!!sJ6VPu~w zV+WfNXRot0srHng*lv{5?%Qn%)%VFThp!F!TxP8kOa=|>9JJB`*8crf$D4;j1I>YF z>gX@ItpY*i%QE6Gu;pbXf18`%#6Sx*Z!iueE`=w~o7LV(AH|9vpS@GE4=zk>Jkk7! z4Ed)^Ix+8=Bo&sNJIdTV$Wug~i0VR5o@JfuX?|1s;lrZi;Mf#Dn?_cLqV-1Uvl6|1 zDdNLwo(ibE;!&Hji$ILdk>qS@^_o#_)x;>iQA6smm#C>UfE9e_wl=yDvlc3F%vsI5 zo*bv9Ox-^WJ8(T@7&0ZSI8E02msNttY_#lxj!v^>{%c!Yfm!{Cb9MZfn|+FVnI^R z<&mP`!`E`BH_<~$GM#IOOmFVer`Q@9&*l?i)HhS<3N|Gbc|FsB!yyh;6gLyFEyM8C zS35(@Y2@v-8zNpf{i~pLadwQ^Tk35wRR=WS<63Bre0Vw#^bZ$#0K8xvi0^%*3{t7_ zeKRcq8Bx>%Vjk}9&8B6lUG{{7rNHVl`Raa$fo6Y=Q*5+HW}B5}pSQ;Wt8EI;32t{! z%%i@#Y^h76jmEG;giX)Z>}ZB5HT>?GtYLHGDx#?{&duBCxYy+fKK z&{j&E(RL`Mjd!R!Fa3izw5=`uDVO`=)s_8S$27BqOF^wBbyVM12hx3>^8gYDnf+Kl zj=pA%)$x<)i4mx%adgo4=E`*T%c>)>WjQgMq+slx$`f1CJ3=J0bbWDg)Vgkk?b9df zYA76PDA3i~hEVVBol)*xm6lAq(AUA`m+@_J-2mof5GlpqFV<^XS1grm@3dRf_~T3c z&h~PrxMFJZi@I|1Y4q0yW_SgONi4UrX(BY6*PcdbB_r=0ug&WuSJZajk*FiQPqqb` zA?uf2StpW>^(++n+S&K<5;J`uP{!|QnCdeiZ9^3-r6Nu}qthX>f$am@`%gc3sCY=w zFaP{_w;#wK)8hL8SCd|9SbpSP2?9~t4Sba>3>QpOQ4<4w_XAF*BK`+S!-}KT3Nif(u|_)j%P^}M?IJ0? zw0I^MkRky`%hhmi)h6fDx=v@OnJi|tm}RJ0LB-87sLvT^ZS5~8p$vTrX_xd-v>z}? z!QG?DlYMTxo(q&m_M8m_d7NyBey9t&XM4W8ljC7m^1 zGr~9;@0Y@ln{&L3K03)G5sqjDgHBDSNwd}KDKe#K*i~TZh@`v4t9I{59YIW5n%Moa~ z^cDbaXl=;54vDwo7uvop8tL4Nw5Cd?BMPDG><N0u+>I#pNm*#FjS9H^n2UmovBTWN7fH(K61!=+NGHAs4nm7Mc^7cxjE z6WtSx`EJ}N5H_hR_tOh^N7FJwY3q)b+v=#Nhdufj1eDUn97?Ox3eDh##ae@!(5c)sgd;{CbKIaizcl=Mcv7K07FYP4WyX6WQ29oA(2LUh zk#L@D5O0s8$bZ@V0^K|Pg`73qA<@rQB6tVjZoSDjjGrGgc&OD^Wzt*wt#nkR~5Wk)$ z^-WMif2OqIRJ{wP*>C^$IKLkix_ONcE?r~ZJIho~mrlgVkN41cIIbFllqs<;>Y1;b z(pB?vp!6#FQj)aYQXs0rjKl9x zE1X-e?HvnR+1PMbcY*7Rw>}Hv7*fBn3#V|-Bvy?Ljy22i0*+&|sVCDi(Hg5-)!&nw zKUGH2*X*)e*rQ9$KG9kr@ZLgFc<{{^#dr(By_wI<-D~bDg?gGrTxo@0HEp+gm*RVq zraHt^5a={frMSP`zkh{GCfx&0Bi}bu%!75T ztn$;voN$}Db1Z`A4y6|)tOxa-go0z+^5JriZQZ$LpToFI0>AB#kN;c8NBjrjFh$v9 z7U$Xizcy!?1cZIxX8AisBb@K&JTu<1Ryjg|Apt-4pSP&iI*$Rz^IjHAL>~86Q4H?A z$ntM(c7e=-4h~cebUCmCf?PcU`l_+cy$N1C&KY3hYcS(B7_TIB+gY zbBpf;nyJ}jqN=7L*LF8e3k<$1zBn$SB`ns-g-W3ByKSKGAv+PDN7$dI?Z{v3O7nvR zq-cyaeA%{(<8X55sdBEPiga8Tz(imt^;S7g{FV^8m$usj3%M|i+}jYFhhY{(&lR6d zyWW`oUaPIfjBz9Sg*CQMSIkf|YHKJkt#vl+!Tqe&pT5sa-KVe4qh0-rZ)n<@`sTF> z8k>P_GVANsbgxrsPMXcjxzZ#(mL;crTKB*1f88E8av!IrZ;y4Fst7WdQV;8YXWkik z_zq$smYI-yS^UK=#&N^w=;;_!Tg0^PO=8`U&SiFig^+svCjn5AFk_k1fuE+R*WCA? zTjAocF>d8S@)J~$s2{Y0H)G!i{kX=vA3`_v2Jy!^^t!d;@t0KB>u0Ty(6xnGW%VS( zs-!v9pI(VH$o6VOu!;msMMsAmVcr%ykSV<`MZv-{BCK9TPX1+bv_wzGz#xoWEm@ya zJa>{JTRN2dIdYDn=I)Hk?>SfUZw<8J-Q_a=TQPa#6AzbCYPC586<9u5J*W^he707Z zGVge>D!KM|p!>zJsL2aK-H@qRvgMc$;R&MZY`U*%nLahnn!G4Uqqb4^2-)M;G>$!b z_g{Jeew#fM%@uXsCTbpQ<5Q1O_&b8l)o;mGU+;^-jk_lV%g0g?Bjks_pLkXXh5Z_3 zl*tOXTXd`WuaKf!o;tW{$BJK1W`BI<8kqmJ{ZVvaX2eIVP=adV)s9f0&){|0>cVgC z4U@)VF?E&6uVR9zFX^qxz$0eOg1L7c)$=P{89$twcjk39%gsPU>A+wMm_Au`6-$I0 z<$7DNXCog@(^&A+Yw@KAfue(ylAV}0M-^Lk4$L1b(-@@ueJw}+)P8#ELly~3XY5e? zDscf)GF5)9piH12`k-3yvsa+)UXtoZ01U)Q(OgOW&*KGpY zYilK=>0YVYE}h*W-88x7$E5t~q1x-oNs>IX*;rQdE{oaEMi|?PM_~&eK=;IZe`pcC z(XmYL_vEy}d%gX93{n)ce4^%j`K$IN#8@oo(OTq3q~+h0LUR^td7}>t>lo&^LP+RB zWTTIX(!slrZ}OkJT_b*&v)@cThk8Y=MblL`1iANyLw&-Rn4{g0#kGNs9&Y_PeIn=m z87lQ0=f5F&&K#za9cXNH{`}lm?XP-!!fL<85RO+`nht=Te*+mT0NIc#D=V9ZD<~-V zE3t{p>E4u+lLL{tBx)!q^mleAa0mjYkz#%BGP4%i%acyIM^yXG_e6af$K}e23K{7m zJ~>Q1nSfw+MOdv@c3W3KWlh55#sW%j!rtfD(vlc)X|^tf&R=f69pq>D%|czbS*ydH zbh4b`QsKU&`jYK_AaHJdC}i~s(ayq>t3*Xk9{Tp}+gsp+9pj`=pR#sF^|a=JBT%Yb zh^$%gsdYnB6AK&LUpH^wY$A<$8C53ey;i!@Hx)HlKl7qJ-0Yh>Y)Y9tNHLPR*WgF? z9kI;&Ticys@cMb7ONKs+cW|YSj*i=8WZ_7pG?jo^G*C$}X$?%fBgVo4qiQpX_*ASa zCMNc(+J@xM!h#?GN;h8|O}Tn||5g)d`2ZZpce0#Dc~z!UUiU|By8fA6-_grLt8&;1FDs~~!?^FzAb0(8303I+0z(@)Jto_7q&q_V@$+el8 znI}N0&g&B(DOx@5q)!nMKY++xX*(T9# z_3}1X{DJ*f*&te=?9iFSWB4ecdv0?xhLXqN^`Bo6tCO{*2zkI{Z*;QJk&}zQtarv% zc&_R4nl@1=Dk@$v7XazWA3S&updop{dVm;#WsP~HGbR@B#HyjB$yM(<&f zK4d_V;P-ahB|@`oX?gj@xb$g+a*_NnRVxGNq&l~S!(0kyP*LW#}I z%}?<7>mUl^n}EwMB2-p;tY`-A`~Zr7hK7dsXhm)Vu(KjiMw;8&8W7S0yv$2?ccJ_D z@9(T+hg`8-A_y2DDlk4EYNu&Vc}h16&7aZ zYXDgFDK1X=#f$6i?(QH^Y;5c*7AX#tItp5$>9Y&~14jSZ(_?E}+i(EieX}uMiCfv4 zZ7V12>Ju(MeY$OBYkLDkPftHLKYvYJT-}5P2a7?6>gdAgoOQHWHkWRx-nHx20+sw(3h8g+1lEAu8rK}=jV5Ib9)W| z^Ya@U(E!5o4G?TaRn#>2~7miQLwws0#x zA;Ilv!-$Q8gH>GIPzGLG>s64SQC+?N;Ev>j2ML8HA}lN|U;tL&a!XE3{E(QKsH##+ z1pFC{qWa28o%%{=XXl4}Rc0vBK@nn~3U|_&;Z_Oa#<;%iprbM~^JOwIY5s@tD zdn|p{gBf=dkYS`1ac1Y>(0kNI6gNUbO3J{_jsqa~TK9EY(2L=izttQ}Oxf+#-Cw>u zn9TF^@s>0>(mr2B=;d7KZ>rbt`1{?4ZU3nU9N#A zV2I?qiK@iGN_7H0f#n<9^`5SbuU%a&pdF6uQ=*T|&EI5ZW)9qW{`~pS>hqu=blCe) z(5OOUt67$d0Hwh3_V%XXya5k5V}?9netMUGWnF#!sE8eekN`7aPc%+EGdPu%l~oq= zhf5|TCW`XVy_UbEr>7T12k4T9M(Vg|^*g5mzMwr~JdJ~=xw*ObRaF{i6Yx9i#QVxh^)ov=JG_Sv zA8KoBZ-mSay(1+fV-ypkJ2*Ii{gLqVyNivD9h;F6v$A5l(})AVr==Mz?)+W+dUmp} zVcyi*A7f{0YiwhKLq$cklMOv6Sc6{fd>0*QsqO3u9e9*&VnorTygEBOFGD{}0*b0j zJEzGnoHecyKs&Nfo>pCkgRJ%YcS=o7%~-iyQ0>6*a4b+2V|#o20(EQ~8=I?CR9Z*L zQjAZlsswUWZ*+hAW@2G+iI|vJTSuoEHt9r2<-@OErXXz+1?s=RtFhj=Az@;|01vOM zs~eG;N)3lFyet5AvZ3M5pFe-Z37=V;u_~!2v1ELWA0#6q0}0DdNG)O0v4FMM*eqYy zw}FQqQb$yT(K@8dzfVZr^Il12ChFF$Tb*COs$3XU(UtF}-zo#S0m$Gj0a9FQ_A ziEH|&JGV5_e*DniS%}m4)|j84uX>Pm?@c~GWvN{5J$w6-!}aoV-aoUm7f^ry{#Aw@ zUK9n38K7FAzJ43!c(iNx>h)`hT-7)4-xH$vzZ-RxUST)Nn4ReF>cYMfE|tdo1omZH z*Ry=At4l78zoN48Syfflp@TvV0TfcJIFKBPNTsUswKl8Leu7b&dV1dtrTm0514x-* z*FSf3{2n#(Z-dy1Ax?G!OmHSRCXMUXU`f(FN=(rn5Msb-s7K6iVh|6P6iQ`-*h(?> z6PIcw;<~!Hyi36S*4x{$vpfpM9haP({5h>CCHf`Y0MJY3E8!wenre(ud$e#wj0Pr% zGKRT;&Rnjpt`<;?kBtogBVYEtDcXO@N@4~1wxRLDW zxo9wce36zG1tRz?IGBNp3x9Q@rUS||?~R+$T-JVk`0~r+&TDnOd$E+E+8iM@H9`#^ zKbHBOcu@;GqD)LoE?>SJP+ZKJr;ri?WwVySLAB90S71jqTHb)>gvrh)$X$!36~*@V z_ZJoxHeMXab$s$f(#3@z++7(25!V&o$N6y!2n1(jWO#{bSKCs+<6gRX^JZ;*z1iyb zs#@>8l9E)r4yds3gmb8D6w;=y+y}&PAkT8MnQLonVbRg;u(dk)ayyI=iq6l60?Qlo_3Newe`oiB9FE6jLD}`zA^;Cu zXs#oYNq8=qaU%&yLDde^v=B_*)77OEv>gioSNSqXH5<%k3uH1*vt6VRre6X`+5Z0T zn3+F+Fd(vg78;uU{=M?32^@wBU;Fvi2^4mGmDpPu4}uta1_^nbm7hF$Vi5Y^+pDA` za=U>+l0$t8k61tZpEoI3kJGEBnLRx{9GKTW&Mhp2WoFW+iuuz*`1wlMiSg6%--k#n1v$%Q z92`jriKqXh0{!38ta2Kthd}rY@asG_OoY9k_x6(_$YpqWc%JHWh3a#4 zK<})~d(Zw^NJtB462x?XMf(gHnDAdtIIdx=QpCvx8HaX2 zZLKhv5hiYKLhu~-MoSEaXLG>%{gc1#tLM%|{ehR42pmCQsgW22oW`)P+6^8&5KJ{Y zD_awFeSH9kEJuLQK)CWNQOgi?MPG8Mt4w*uT(cIC2 z2}04?(<2Qh0LZbKHN;N_TM$JhR8F0+ zzyn4|`@Qi6Q!}$Tutcx1Ch-eLx)>2bH2fMHYcMAfHDkPRcpzJw$d5rx0B!GBCBm@Ksks194 z50!sXPke4q^C=)ol~h(HOifE;PYlY-%`Kx&$R`>f8#9Im8*+U>&up~d+%!HrJy{rh zOMom6(2Gz#Ms5dTHskr?l3)LI?gQ{pmjBtQu=AhGNb%rDgC|IwwS`6AGIw}#F|DW< zy|D0AByN|N$8Bh6Kq?Xtxk+**G4b-gn9YIZej_|T?r-94&dT5oFfh;4S0wX>#~_4d zLV5uDmPK|SB8rI#DzG+)3TbO!jfjjSA|ym1U%Gg42GYNO@j7@tSQe-=3bCo8?Ggd)AJV9L8?}0VmX;q-126Zrs2E zA#Q~{6>_FrE|zbdoqeva>a<+2LqmI7&ZZ*nhnJW^a^;2lFXqc@psoZM?80=T7|3uL zln6lT1g&r7|2p=}nTzaaGrnbUR!4d>Z7SqqJ^#lDJk)dJMn^od)TxjpMc)YEn7ikh z6#^w6&7}Bru+m_^_7}2ae*Ux4Qqs~8cB!AsYromKJ03uXl2X|7=g;)H5Jf;l0;0hp z8BBvuKDQu=rradHm@qddfkiw^d>{t%4KV@|VE}9< z3}Ao;p|Y@LF&pZ95cA{1^%?Hmp@ue$jG`iby6>)z58#S6wzh&GR?GHJUn;#Ub!sM7 zp6D;;zEgZqd$=&2xc((iyg68{wsMc;#TC`qlvBWhbK3s!&bIj}wN3d*>#V}HZL#)) zxeyoa369R<#ehUnTd(bVdreLYJWZdRG4ChFcw=Te+V@iAOxMP!zG}&mq1)c?;NPYg z3%DAF8wN>YMh4yS@iF3ppq}zO#o?0#oW9rY3KoP8-5FwZG$Jm2uK7_)uH@uKx*_r8 zU8jCrbB$yC3ym?Sgsm;>cHJIDmchH?PbZoqofh8qT&$;2{ zqh*nm6xx<8xQQ7ycS<}+=5~B^l56Q?+wJf8u|vQ6_y^|P?9KP0V2>y_yusJyhXcGC zsSkn+DN)By27oOW=CE|2f?4joqw|BSzwl4Py~D>@IW@9YB(C!+(XjQ=8wevVZKb%LR;+eMO|&HRdmMFg>;U5chT?P$7YSB zrQg%XRc!o(-ALVgas9dsXA~rqfq{XJGc6Yp4-YW${a=wc`vA^^L@{uN!NI|G_-aa* zS?^YnVrs6m7PIfEsP3aa%$sdYj5@K7RBjr3!^v0u^RNn;P{;xstMnZIa5l+E&~^h2 zFWh^d>1U1@>cYkPv*6VX=9Zm!ysI+?!`zuyUVR{Vnslt=LNP`(;>H^mu@&pROCR0n zO(>Mz^Si(@L?X{B*J>Q^N5r>Fwq9m^GFw0AOfa)sIQ|@y#<0f~aBKD|i9c9TJgyU? zS=i@K_cLSEKdKB4XQLZ$q-hVYQmL(*L3!O3CB`4PR7}PgG<#mdxcN@6245zcx*3wN zO^EA$4{I5kHC1G{)jk5(&Ac3=MX8^$B8B>H1Y43c7tS^{2be`BIBI*Ik4Mk zxQ%zvwwF}zG6o5(q^Dh4*CQ|U{+%IPm-|f8Jxa!g-da;$UYxT(MvL&Z8X5;e|cZTO@8Z}(giT=08j^Ypy zAVqo$;6|eW(%20C6_SxQh}Qe6?eq*nwJZx$LN|T~(#U=-%0tN&yfd;&P_f+h^CcM& z3u`%T`s(<11-CvL|NY_!de6!CW+~tGY>q$7NvJNV)Yk%ZmZ+coEZ8mCXJH)5u?1-R z*)L^_A>q4>jeQUNClnsvyDn?`9Ly%|Z?qEdJnAHbS{x>yWzVw@A3l^w4&TkLUf52x zY4zVe*B}w&^V$N@9YcI;D=RFpzdledR?x-&?m;3uv4cK- zBc%vuRI=VNN?_%d|C+XqL7drnv@Ia+d20d67O^QQk(L?0WPrVFg3y*Bk#Zx-^BLM5 zS-I$my$l*J(X0!xc~yZI7|k>ZhHyZFpC{nIt< zzc`76a$9j*M<&$Xdoq92_r7Y{UMSj~BP7-ToGV%b8)ljuI89Wsz<#{hdhJIqN1$Bp zvtMfpKSbQ{+5!X0vtG!Exhc#X>x*?p^v=|{Gdk`Z(~ywfc*FKZX6jo}?a3sURO2Si zfamPxIB9?(2Lm9Q9uiK7@R3k@?psQKv7V5Jw>J|92d<~5XB~cpwJA;~$A#h9X{XgL zbk=1*wJJ6k3JOZRp?thWAnY%QV=i|w!XdT2Kq-bvDSxb|3^Co`H|3CsQY2p+{}A}Q zT41=R>8pedQ_$A4x5ln3EX*ZmgyG9{qMW>evkN>6s?LO?MLrok-=lmfM=<@Qe!X2c z9ej(T4;)XJeO2}2_sqDc#M8iiVp(BtG^d!x$x58MXh%O6GG&Z$K0a>SA9lws^w=Fe zQlgtyjNWv$!^`cCQW(buigjK)*3@Et)R0qp%qD7Wz_69c@%|=6)KC2qSUb)?rXtE$ww z3$d4V2Ro9qS`(6!heFz&1yw89LMbOF9wcvtGATVjf931zW3(^uQ=d8&wP#Ce>0N#J zoW9KU-ceKA)y8*5QA`nXhw`oYdp&$>$fuJzWz~+s3cvHyM{YN+v394p2@x&#)8tmf_SUXsr<*obt##c zm>>(MhlCw+5bZ`kQK*za8p*Ett_dK)5JS@QO1{n1;_;Ao*Pcig*VmxRo;en2*kud3 zGm15r*B(12oOQC@cJ-(bGbg4uc@;^n?lG(T&-ds;(sTjE$izg&372Oyc=+H!Ga%K= zoPJB|H~P^)#rhSWCH`D*Mht?56}iKhb^Xb~U-OmkRX9+hmNqqIggA-Yd)Kx*U6cxN z!vn4!;2fi_!=pk{DuC;#cKbXLdKNmH85b89`*n5!;JqUy29YrNi%2RK7M3B!f{`C+X_)VB&?R3^=cw&lIv_LZ~VJ_%Z7pQBhYK zMxwZ$@HSIqqrir+tuh6I2}1>JRhiUwI3^kHBG1j6KCXrHx= zk5f2*poVRMqTPAsk^kVlqWewP^VVZd@Ybze(8&*cZ$&L3&~fudqNq$o2>OS%Qr{I_mVK*$0`0Z7FDkA#NKZ~sDi zdisAQG*C!CPiTe~&U*qDZ%l7JhWHuDlyJs@K!xzu{?upz61mLSaT zWkW-TgXJ1ymPIIXT)f(BiP&XjWlQ}MMQoU>qoo=i z84fv@l&$T%=xiuaSzkpK(D@T8Hg3x!cmUA5Z)?jj<-2D9#qIah31PCmualEopfCa} zs`Wh*1PEvJ?~WL=g}ygS&I1s=pRXW?=lnAC;5h4`(Ds~4s6_SFV>ljHq%dUFG(N5a z4r8vH0;ID$NLv1%A{KP|Xra-9>;PlXc|N$92H7t33^ds=XBRRdZOxFsu9g&k{rYuA z$Z(NPN(ep;HZ=1ffn|cJ!*iJueR_UZ9Jp)9vuDO&?1bIds1Z@nWrfNclC}cYRrtvq zu8*(pW*fOtCNyne;Vmuq&KHhxt}6p?po##6acE@+LcV>Tgb$$4&gOvpkJY>1w6(L_ zgmk?LDmq9x6WX*4mjjXW$s&_K&rI`YAIQnIKueW~j11{bwLqA({^^J--T%~cY+_;= z3@!Vg`;4rmzl7pSoqghD5s6BnuNWL@9<*^CZ# zHKdRVqdDqHLFr=th6eau7@(e3S^pxXmhlQpB09Pl7zvswD*Y4)ZA2)TB2RLB>~p?K zP7EXl%@H8|Dgh&b+5zMZcLdaR0y(qNFJ*9)}qmlhT}k*f^o!PylQ6hNsZBqfE) z(0f$?I$4)s&J|2kplZR;<8&Pzom$uBmoFK?6ic=uE1apB_6vfdZ{IYMf_9-kd3H2?20^bxk3ZabxM;Hs7Ekfr=?kCKFH zW}@YUg1o%4swx^%cLc8s3V$Ew>Rbtt?&B%)P++#oAgzEtHPS$XOhQTC($*FikW|zn zE@9IdQc{<|v?30Ach)#gb6@WM|sHu45rTxOy6H9{nwakT&(lMXkXAGki zeVz*V#)6XHyv!jUxU)0+MFu7VRz%Tv`dm4x`o25ahD}6mM>+cQwSSS~-iZ`f(I8&f z_Rc4nKU>5ato9p=>R_k=4*ww(0yae&ze(`{^#=Lz?&VMql08jxAI+3)&DgRjyo7yn8lld-f<+fpq&5%MSQ`hQ26+jMXJ`@A>@i{+7qX_x{xapLywyg^Di=?VZu(c~plMYf!mR2@+(SxSg#(^usAN z?LB(Bu{J{Xl9%|^gnN6y7YCECA5t6N-WU&}czMM|ciYJJ@`bC!otX03eB0HXraUY_N&+~oq&$<&2R(yy)ui65}Ag!mQOmGft`<@y+2$g59Vsm zk{e8UexO_X_6mlClfq@7^&^Z-Pkr0z{C=aE?qZC^-bcd63}oGU{w+hRn=_Mq*9chD zrwC6^Dl0=)jhQQrob_2bGs<`k#%eM{jdh2;xYwAwPq+;7sTC)YgCoU|V2vg@ex%XC z{-ToH!as>2pVo*W=B3(7t?YimpG|UsoJEmAW6{Uuwv@$u!@eeGK9^YePi}oLa0aDU))94?-_z=$B3>qtQd+S!B3A+;b9Jbi{j}0X(T@yb>~MDL$JPhR$)?sJwM;?{rJoiqy9o48u#Ynv7R6;kB~gyq z!GlaSj&n|mRFV+m<2#{J_l7R2vC28tx=&j)J-K(qA2{LLx7mKqdRk40?&0W2db+NB z{c{iT#%BL=t&x5p=7YEIozkfu9lR~GpB2o>=j%d~``*(`;EoepDT;zL;n=yCPG*ga z-n_#_IN$zfOf9yh^>R1HZ$Wd3YnKTl_ROlQ@?S0{*3y=d7iI?4zBCAYag;<<)<>h- zcFpH#bv$(3%E_{UNbj%X$)EMlBoy@V4idZBJJ&9o%N`z6TouzTNzI@2u=-6J=G7zB z8TYkTFF$}$jAp*k8?)8!N}1ibNGeCa&&kNGDVgOrlTG3cH3ux8-~P$0eTRFEU?7gL ziUeIj_5D#wQ0IWp>`0pQvi#sq=3@D%f`;hTju}%G8tRShXi@STUR9rx9q^^OALdXdha@bzD zM+PhIpu6IS<(w$wZ>&}EzVumb$yGPr2>yfDi>(@0%*`Srp9NDg|}-HMuYvu>{JhI(+*eCi#F!MBW{Q0 z9qAaH$U7`*GAOJj!MsU06;(vt8g%j`xcXN| zZ@#AT*HgVxx3@iZOVi~wofj!`THeP_?1&uUE{rDH`kR4mg_B8dLTqa?#{}`$F66fr zca5wdKP@%GFL^Ta%zCWh?AI~zpL!Z9d=>gxf@ zuVU_9cweykNb!ExBmEDCm*1@1xw~K)L4blTf_MNuDv%zPr75eRh2~+KRN{1EO;*Y{ zBNj_w5MfkabZb^EFJS6r*l0N|zfrY_z1bI3!)XTnv4{g+`lfJnTq|a#d$0Y%l>1em zP}k666<;#xpx-TXWd!-I{`nm|>Z)Okxu`h#OYuv)Hcl-2-?rPmA7;JZlI3bwD!U}- z-_aAsE*?|EWA<~J>CLdWSkd68l5pv}(NyJ=l<`xxIChn{U4Qsf4de-uX^6Z~?LuvR zy86@bAT>95l)m<1_0LDX^Tmm7^AC8{h*4+}*pbOUy%Q<%jtr{I8jEH?5yDQmAQn7# zX#zYseo0d=BduGfg(BW9GK5SX)Acu`K`i zVi<~5LxE?d__8YTq8U#l{gs*i+)TMMca$^Cidt)5*R z&$)Zsn`tnej8JllzLD$p;r;IIxZ5iJ8G^?Da}S>$LS&G8bK>6ZRmHU3>5$5Ts0SG0 z+kd9$hiv8xWgo8gm_!Y&SH5xyR9pC4`L$PoS7hB?ar1z;{Z5=oP7nHZ9uAY;_K!)HzSsO#a_ z#^2jx@v*^I?lY(=g_`sa7}QS0#;5Oq!L?U==y+{Lr0eu@6VlV zjuejf?UHUS1?e#d$_if4dvY;>)|;Y7Iwn0HznNuJKz6Uz5$rM9UC7 zyeyIl`0)=!UUpNv+pM*vANRzE(}G|*x|&S&YRdSV>`)N{gygFx=KueUVd-cZ=JrhOe#*6w1N;zqCwvyao9{g3rpI6X(td8%{B>he6lcFX zIk2lp%X{M9a)%Smkb8XrZDK?ib9IokK80IXC>dj z8??Kz55CRZdBo%AL>1EZ0Gz zND02$w_+aowi{w&^W>_Q>xa@YEH%OOmMYa~R^P9A<^y|Mb(!eKEl)W8!MU|!BV1;(bi%M}>qXI9R~`7k%s|sH z$u;lkA(vk9|78NvD|SwIaoCCB{}?KC@BRM_72e`0w!`sNE?v!Q3KruMzrf^@zK7;R zx%?;bsVz=%n{ugORa{MQo3f9~0_MB4-*{TL)jHqlb3dBW8l&UIuU42`dpwr+vW@O@ zOpTOd%3RD8^}RyR{-SkqPLsxmPgdL)`<;-8y=*#{2kgkxlMY~JZ9`cKGRgrmztLp^`^Zq-vT5_HDyqM4jN|za%2W+i-IeF& z_#@a@TNyl&r4v_svpJc3V&-$>=*-(Y1~P*_VFuGJUtXYn{xqE3zlL*fr`7HP3S0lj z2GOoD+FcJz;cizN{6(>U7qyDue$M}HX&+AERM{%15W`WGmF zT@sq4+{R6)?d4Gdm@=uukINtTSSom1DYy=u9V@#jDNYrD=$`KXCOF>gpsRITrGz;S z7+S=G5l5&+YmLr8NjseoE0BI@@7rv6#edbqw_5?vh5#f1?Dq@!#+2C-uio+C1_TDT zc^65g!?fWj1Aw%x6ad!_y=^C8mHph-_6C_|1L8Uue2RgQ*;Xi_A|F#>f+DZDxHyV3 z_WJqkjVqIfmHONw0+*xt<2$hVXNSVlv#Q8VH^In$XMCgTLBjy0z(3}KWd52 zj_1Yt%gkhu8B*vP0YTt=3?Y2sGL(1EXFI8d0{}4h3y9kQ?B5C~A>=l9PPPZ{z~rcu z>p5iGN{TTKL6;(cZY-<ic>*uqG z?3&fFGIXfp7jH8?23lJfFsj3U4IdD9MJJa}6G>!of`|NauAB;8ot^pLpAxsUw6G;^ z$tUw;!Mx1*r?+L>p5pi&Z_1F6k!1l6SiABu5kQPQZ=*qnj^bq`G`gnpJUIDf*B+pfI80WnIYk`1=0a9=e1PW#a&~jD7K^9xVIRS<*xT2z> zL=r#(amZkc`}!0cX7vnmoH+#ATyooItCOA{>cb};o4sk>@|yR?YTjch~HhSK3jZCuMmRDej-VipRODhVSde_Y@V<06SEDMi32{9)4mBfJUN z3+&x6U`~OE6aL`Q-qF@%rwd2@j2yrwue2RkMcBsybtgN620T?U4kW06Ekocbg??I| zNo$A|?EUu!&x#-RHLx?ti-QsFMT9D^b*2xPElZ*0=pTxU+PS9_jB|Rj?ypvhLiJ_F z0{16BE6`EH14l$hkAN6d!t@0^(g%Gm7*J3IqGd`EmwamcQ0W-j;+LFyhqoW0^3`Cb zGBF`R-Fha_>391g0O)qz!{C+17i|!(;Q^o_t%64R2mO*O!Nl-Y;43HsM$?1aQmBo{ z48Ow%AXP6XR!3mE)x1pGWmrS&KeV4P;JI^tp=T{Ho;?%`qwBsfsSOmM62&@f#cQg<_Kfs@805k;*DR6>Q9w6Hj`zsg+;e-sr zBp@T@LVA4^RX9M@!VPpSD7sX(RA^A=z^vL=pjN(qT^gPTNWD{aYwTCo2`^+mgF#xD z`BAj5(E|)UG(um3`!m@7k>z*+mC0bL01V?xvCy5?0bG(gFllR+=oi2U-Ol=SW4`L* zeR6|8a20-K@7;)+OKZ>EPlY9dzK(&*#EMiHj;XIJN+PfSRex7l6-_5D~orSw#*TaFUup!`mo% zNm&V|BH3=;(t-5?>GL&g@`l~-Hk91P0UTtJjjb4{ObKJ;k^Qy-WoQkUBH#)iKw8ZS zv5`?x=4s5RToLC#VSpdgQ?LCA9p{LMh!=y4>OMXV!0CTX!-zud;1AxvfD(>@5oq=s zH-HEFffq1$K2$&c9m&`C9)zfAd1onty(AXCrmvq0Iz10th{_+8MZ`J1>&q8J&q3gG zfY%`5722GO$ZNDq{3uH@uFLrNBe4vUtolH$iFYnA=&d z?08jqZdrg@aq8E`e!@qIb7pM-6Fi={PB1Y@jfSbqSBO~v(lE0gdqG`!v*jXJ(nD~= znQ-Q)9TFy>4$kk0C>^Y{_(th@Cq2tG*}it>5^$0pf=*eo8lHo#fcl4nP5-B(FmS&j zl7xEf_zF$|6|vDqlV1-Eu%MKMs>HuYxPUYM?B74}16?7QEXB7KpGhh`x8@ch=+XlE z&sj&|KaRru9vj9w#m9W8{0H^!>s-l~bQ>I-;UJOaHhG4{0qJw1Zxs^e$62W%mt7H!qYOsUt8luW26Re$Eq765OO`K?~SEZ|$X+UN!m zDlr~x|Lye#*myN$@MpsfG%#7r{}WhCumi=LzdLLIz~S{PoFySSxe)B46&M(jt6%{x zFo9V@F+Tt_3lPsE-$CFHc4O})04OmCMy$A~q)rY797h1G!BsO+4DgaCpL$_PFCinN z6b#GKi-%k$tC~X;(|$+AK>bk!$Nt5%?D3Zu*UYQM1Q)B_Z`11phodJjtTo)rfon!UPDLA0Tv?AkM>et~PBQ z9X6-O2QW3J9d4Jzp~DE=djMK6H#fflW9T(MoFicn{=|hppb`cO@5{*8fb0R?e8T85 zkS4r<$?3@N9v=;%ANT-fZIsp36S!)BqKWPdsVxoUMk3sdU@Sqc6jFs%AOrEcLkY&8 zTm`YS>Rr$+$OMIj5$^}fqoqm1*5`EFW@fIfctZJrRJ1I1H3t&i+=t|xh7H=H-n-js ze;$XyJ;AK=e*{n`FYfPmR=tQ3e*8EeJS!QeZVmvF$Vs^zg{cuO%6yt1y_5x*4wQHz z49>{@GueZ6uFE9As9afDsXFo9RuEWYVFY1mukp;6QzjzhF;Hbt-gbM3U`;@mD-3IZ zOcVzCVf?h@D_6Zdn8p#v8AwSz=3S_ zlZMt4fq`hV^YhEV4KoTu94-(->PC~{eSzsUgjyEh=|w{rDK5X#e*jr75}o>jH%z|k zD}o5LvibS*SHZmJAi&q&L72xo@j9;n0y%^R(+J!g=l#$SM&L-UnRH9B|G1+GGbD*R zIg&46;@uS}T!bZ7%E^y`Q&28AZIi9|H^6>;XcGZ?z{^pixD;}qn zW@d^bM8^o(FTzMHm{QzzbQ5PH<7@(^IQ8_ZZR19l6HTs?ej?x6+7bXd9fY3w?rI~4 zZSStDr#DJ`hXK)?!Zv{QnkQSRTx(au;)7R509Eo0;KX;UHvkDjUalP zuRoa(Gll#F#2{Ym{O2y?fA_(nMs!p_$aUcE2NS*rH$?rsBV^o!@(%?Qi~T?lm2b|K z^mUp6E_KNmh|y2+FQ6z%@*>gx`-@s`E;W6B`)6fBg6Qdm@~a`XDuX-sXFhzV8w`!F zzwv&HMmJt3#Ys~lU5}WF1DpEr#jdA~1-Zpx%AsrQ8)M5XXKM@cPes97%4eiYUuC?z zEUxk@o?=witI_?@a`8UAKbvtaU+izy?B-o;``Mw}nBCU4IH>$yWRYwv{xP&4#3Ur= zPV#@BM(2jS!kikQON+7UCokyg+$(G-l?tAiTYBS(N&4h0h?2~Zogw{Ym9h{96*lcH zBN-ddQDOAaER4q8NRh1a-YY{aZaa1JL{Y=eC$kRAi5hlL8rq?N4MFw9R?Ud|#+vk( z=Di!4vW<}+RBsmYA2iQR#Np+u{JZ{S+kF43LV91pn*aMtBK&9hTAs^m>h&UYws^np zs0V2HrvE5J8%$U{FubX6Rl1+Ijq@PU=Vj&;PqK&$ysYYQ@!Q)SP3v(EFn|q?e445^ zs-TxgHSfbV8guaWE$8cl?+1%ohg1AN{>yLc?B!a*rxhLyjor7pzbaaA_7+Qx*!61{)3ruflC{xzf@=#){%Zp> zSGMxl$Q(BJzU?pGKCyFqA^oFKc2w-(jfG!a0rpK&jffV7nEcRhu_oF>Ydr)B4z$C^ zaT51!R1!Z$ztB3;VqVBzxlTz&mHqQcLyZupPKDoX<;mnwHs-AOlH*C6@8$DR1EZW< zc{YP7gCwJkDY}B#s><5z2Kr2L|hgeHTpvT#;9*mGD&foxkjk> zNQQmsosWAd7Ud**ddvDYN@>Cbf;&Dg@qVW(8HWn;auaa&A;~T z&;0GG{GXnzWLpt>lwYs#%a^Y7_OFrp_mACO9cFIG<)RjT@Z>a!tAckk-KW?0i^D>{ zUTBfSo|`rGBdy03ic5UI+&;dyh*3IO_;caqfH1ZzhSxBT{Kx?NZ7P3I2Dmrk-S8n==buGe^G8slva0e@enJk@|$n<`RlKab~3)n*!X!_(R?5i za6X&k99D>Wx8&2oo_u@FRKV_sgk;88=JtvGao`7<9qy79Ki zQ+W1#AD^{!O6f=A6+5u|b+&{}&iiVHdu=giHhfhMysT+oMI@)*M1^Uhr6L9fk_MY^} zZ)x70Qd{@l(o5++qE!@Tn$nmn^1h>S*X*K^yioQ#-rNEIcjRKWeuj>7nljz)?IEwF ze^$*k;(I;x)*I|T%s6=Ll5a)gFUHig;@B#@C0E~_S&P9}zrat;opko5p29Zuqkrqj zc!TTaQ{vIqt&aySiMvnQ*)G0XT|e#XzKP8^DxW-5DTTYsaK^v)(z#AC6|=~5^K<>T zeT5VyJh`uys@yuB02AM&2Xx#D=}Z@7s4&N3B|SE ziVmOKCB+nkKfb2!I)!*fR~ST73KJxLBul(Sz+z^2m*EFLTIef(-;=$$i-Oso1)nk* z3K4j-drmqVh;a^f(AboV?Q3zuF6X;yz4p{ zdVh0UR_6wN?ecPmnFoE}6>e&-qzrTb`-m-C{`UOJiwGwonxHU^|_zL6& zDTL(w)@vMia^p3xuIo_3Z5h_Z0}tQrAFLk_CRhz9Q;Ub$-5%S#VP=vSQgmYT+!eki zHfj`P`{JXe*hew%AK}yAqDNoUzWnR?`i=81(dMBk&Z9^8l=;EE!aFmAOxu4Fqs(_K ztS6IX)Q6idj4U)->pi_OxXtRiHe61nz~7(7KFHtuFdhv@L z1dqS|TBIb|j3<*nsu!GlQ0~(2`^A&R^!8~<`=tjG(Ps;tU$hk}l)Vk9IrRFyF_%Ui zju}g>y1EVC=usw-**MXz%IRNUPe?pmlTH6SP{Y2tOIuFvlO2rHc;{ADYZ2YFhx(IG zJsKX31u2W<`NN%(BNq%;gc9?+5aF8ZR=!PKY z8^+Pq&z_H;9C(Nx&WYGfdaHj34(|GTC(~{X)5Y_rA?M+q+{v%6goVyW!5T?Z{Dw)| zs#Sj`V;(;Lv+Wa~|BTbHit4t6WUJ5fnjcRziBGbedtY=XDlOhzKjV1#M^?1PEotD% zR>B9C`=tsIs(qVEJ9{!@j~$}8rZu!mg4OxN26hHVn zFy(HqZnM63=%ZFAZ*-WqQkO5IrGry#H8`3pm~V^~{QF zHB-JrI{Af(k<^}KVN2mC!QLt>#o?oVPh<PYGp zjgGPKyZ_BCKdzKscPL7`C4B*%cEDyZw8-X}!S3}*QX#YQOW!|!uivUCPTcHvDK3!JCR$_^BalDnpvu)LDHkfL7pu?=9^2yTd#l6Di=y}&c*3e&X9hCQ@O07>t zRSG}fa~`(zjzZ;zr)6GCvpm?^ zM_N(q(>e-4tJv5M*|gE~+g2|NS|l-D#*#iOdY;@!CirWnRVj?Dw+IP909H7A{&}d90PatF_>MvoUAd zy_$77UgJ=>!n#Zo-t|E??0-gS?bGJi#!sN~r;lODsv>_efGEO>D62Z?Qeo&fc&F09 zpPj`6s!UBg-5X8a!=Hxu6DM8SkeG^2W#TfN{Yz=etCRW?xgY1gjik5IzMf+8`u*X) zm!t>}kMvyl0T1%t_*RNM5}h`OoM8%w|IyibMm3SHaXg?{uyA)5P*Gqhil9ghNN)-f zI-yrtf5hB$0#`6(ZzL^z7a*_wJr^ zpOX(WlR0zd%p`C5{h#;${9>j&rLDZ&`(oE_M<@<9cR-1H{|Q{tiym;I-+yqIu;clf zZL}{7{iA}4njV`dXLYdX`1}=4Lq|>8!}xcWV=ujF>U&6^ZqIZ_n`T)k2+pfSF`#E?p~klKjn_+Pbaa~HWM&dVnm#^e z2*ibbJmR~&XAf3i*2!^w(=_X3F}+R4JY!~YycMA?s@MGJJ}-a9=8a-F%1zrlp4{oA zGb3EwG*~^yHK9!wZlt!22XQ*vET0YMo8$!>BU^pE9@)k0kFLCRouPAcWHcY)@Y`2> zS%Z0*y8Iy-KVk4*FkCcZg=Z%A#{=EWYz0dHJkmb2R355B)1lW_xaK#5u5Y9OASu*P zBVkniyO&DA%7ZEx#0MYszF(6LwmUW4USFr8q|Ddz{fiuW7Q0x7CQ;?2Ea2DYK15!ao z%p-@BZEd4YY~;X-ik($owJkXtG`Fzem1?hRHn>=uu_}@ z(Z;mhPxAPlcs6e)EIV3^t`>WXWIA7~j$Mz>yd&jCij(Nl5M@bS+`6#-RJorK?pQzbQhQjuIBp( zASS4De-wrlxL@lL0s8QZRxtAz=DTl)`4+FJ%y0@ji-mU6_V!ohFRDKoU3IxoEIp`DJLkt{Z` z(3#Z3Gw7Ja%{a`o3p}3 zj#Ktrs79HWUqUa5gw)}fNqI5K*lTIH?BR3D9@{PZjb1ZVkgrSplU6=%l@}ud5hH0$ zgb!Zs9#&Vw7bl&}`{0|*u_1`c33nf#&f|A5HTCGX_N%obYcB!uC1zbfXn)b~fq+A| z5bhwd{X8|GCk3079<)H1U67V$xtPb_X#5garOocJT8&>)f^~&UI1PT9`F!)V(Bb5} zZ?7q1)3@w>>(8e3zO|ZtL3tU{>J-E*Gwc*T8EBLf49x%eki%FR@SuVMlxb5kS-$2n z-YOi`sVd34)LK&1}ih*#N+*gvGLP_m=9bqcbzVA1)1oqz!SNJA5i# z{32+k1vSuD2h%!M-CMMzUApsi_IMJKMS^7{@$jOxo2+%`Qgz1?>O{$q7)-<&#?#|CIMvo4 ze`M`RPgnlR%r6M-mkn5pbJ|mK2zxA9bd(w24%TzIaad~j+X6G3APilF&*_UoS;Xr1 z;CdH7@*GJU*L2yK#odARe{3`Uc8(FhxCvR{_N$xk-oe1$*2Z~u`L%0fG74yW51C(| zoO$H39!ONi`-&o>hLk0yv|OcMJgDkT>CwFS#4TrU8Fo1r-^Wf7-qpGFDmwPs)u^v0 zDRE|}?9Y^X``z1IE2NT?6xP(Id85ioA>vik7}-RGTgaG}axa=%{vNKhLwe&vH+AD9 z#abxraJfwJAPp5T>5>f;=LrHlUx|$>AP^r`g#*e70BHe`rEsuYBJ6`|ySfadhn$Ry z3cY?Yq7|U)^*0iunaIfRkh~Hd=(7gJ3ok&^G*XLfPid&DhW^sWlj1~49ot0Hbhx);Bg97GrdQda6RrzMmTKhE;D!W=NGG19 z3gVgXP9?Ww!9+p3jP?csWrFHRxV$ANVRRf4a(L%6sS#_*7_S<&Y+e}#haTd-g2&OCZ z(Q`8ht=iZ(ZYxJvwDeT=B}xNB;$O~o{6DdMZwIgx8RN2f{#*>WwC0sA!zT0h8R-0Uf zp_Wh^8~EXqS$9^g$2NqNN<&knm=sq#$s-t2N5n$aILCnEhNdGvY#M++<)jy^4#B5~ z%W9YR@LP_h6d<1g;Hs?8vOYOo6iE;WtvSvSv35cGKVN(AX>?zmtYntTmaMB$sT+0W zX5dOyJ$I7i;gfOrg`Gr7rk}yNhIjf-KApzSQ7A7Qk43=o2oFW=5{B?E*2D_9;~9*l z!i54i+mTvzK0iEQcMSxJg)Lqtm8=8i?<-GciL8lye8J#ky9e(vlS(Fd%h5v0;FOr6 z3#bxIZhAkb3$PiU`IY|6v+~>9Y(!uV_qXaBO(^+c=<`9=fodfyDwn9HV)g86q=E_C zRcIrcHA`@k<2+q6QED}g444!IF*FPd1wc#ga^42>dh{X1MdB`0@AX#_5)R; zi{=5CT>~tN=i}`l)qfz?e~-2Y7;He9DZBk=UjVT2)35d&RP6!~wsE2gBe)Ht{^=E1IBVc55}J5D<}L|`h`!I`SIfs z{@j$l_t%O13G0ve?@varname(x)) >, fontname="Courier"]; iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; - with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_linked_internal_transform(varinfo, varname)", fontname="Courier"]; return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; getindex -> iflinked_getindex; diff --git a/docs/src/assets/images/transformations-getindex-without-dist.dot.png b/docs/src/assets/images/transformations-getindex-without-dist.dot.png index e18e9686fe9e653086e21720b0bfcf8408068247..a869326d3cf8275692d38ae5525b404627954fa5 100644 GIT binary patch literal 41849 zcmc$`2UL^avo4Iifr=Cb1e7KnrFT?7l-@!w0@8c$V5LeC=~X%eNTh@onjpOiA%u=} zNTd^5D0k!UoO}Ogt$WWp=UaDu_Z1g0Z`reF&pb1G=6MZORhA>UPJNw-h=}CXOBr<{ zqKmmiMCXuKFN1IH=(1b^zb>09$jK0$p8fe&pZ}GJ=mF6ynP-|_sedLty~r1)+BSVX zgi@3qeq#D~=Q;eGW;FYI^47}?>YvG(yagXF^VzvpYw0qN#)P|J>4{S6-?pm`0e}=Oz7;`5Pr_~&nK_@ndJX`5|30p zn>IM;gKgiFgm9^vz9T@ss&%-nQXeY!L*0#4tt5osCsg5DQpa!jWxrX5P)1<{o zh%rkw-rO1k)aUdmGygxRCst%_pZReAod=W*`)K+bwsz*nre2lf^OJ4vnkZZR&xhF2 z+DP_T*Nz!_n9u4J?6I-wLXWOx!ix%gyvaS@qig{VL-`EX#`h$1L%c0{9gir}bd9MF zO;kfTMXG2n^|8ev4k={4y$c75irn26EUAYo^STI!>^aI;Bu?;@xfzHC+{QZf=2KrZ zi5RM~F{byths?t<3l{9&-QLOo#e-c+5%Wj7E9}=6C z+HMa??KIL+NNjPW`cBK>8_hlki28^gjh4PNqgyWdeOR=k#YZSK?^M{4PZ;0%VO;1v zHjE!JygMz(sJc3pp5>`MU9dz?`%Nn%a7cq>HAYNJgN`o8nLxcNhk`W$bbcjIVmZPZ7SIkc~W=7d7+MiKMG?unWe0#NdcNXgkn59f1yTi{DR@vIxK}4I-O>vvwg1ZG8=HQFb;a+V8uIM<8!CL|t*{n;T?2J0C`1l^-;iBM_iCajDiKpRwnhLQ6G7<8(yj9 zuSq6_l>0meCk#Z`n^C+)V7%U127cT z(=$}GuYfeY_zuKERsHM!9OzEJ``t@4655gy4tyDKC5jT48M{S{kf`~y1 zzt4uMfB9x2Z)&0F_t|Nmoln+bjX9My6k>Q2JWY*~z-5)8k-!}^^6Qi;?&m#;iFGpj z4&i)iV4&)Mk-zJ9IP|8MLcifsQLKR3P}NFdXdj1Q;LAp#8h!geF>g}oSS(p{+MOb3 zb@Z>@nASe^z>}{zCL}n9K!t3I)}rd+E4EP`-X4yua0~3<6GhRj59tk(29jFRK_(jW zWZ|)Cn>orjHu{!m0Ezb}cOY4XJ(oUEm`K$3OzE|FlDos$!=WU88Y=v|TC-&{*a23} zo5&0XJ*m8D>Zw94@4O^-z41Xzlxm}#C2Q$HQSy$_qf0a~bID71bcfhN{XT{4EyJ9P zZJWT^c!SQOS}a#X$xhn@J$|665(=4;i)^y7k#H)usGn4pSN4iOG3I%3s-aLdF){fPUf}xV^gd@u z#mn-t{)InW<;pb7LZD?Hw_5Z|uLQl6fx?f^6Ce>PJm6C8i~n5Op6~NWLbCQN9nb1> z@U^PxehDe~4UYd0>BRr#?BstwF14frghFyy0p|6KlwYHQT2XbS+k=j;_k!Jpg znt(b^<1(pKjPkV##dc;3;tuPJ_dJlPu}#}8tkaOFTLrLsvT3w->C*a|cC7n3BG&M( zBTKNdsy;o{E7W4R5O%lqeEstZ$mBs&_LY$*=3T}ZcN;jnr1{<)sKG42ceKyed}^M>_C^1{{vsY04d3^xX-EI*o>S@Sf0BwR;XN% z;r9WG3%b&7JK1pUpdUFe2}6?w7g*v%p{jikg9EFv2>OD(z)*?pg^79v#xbAn$HY`8 z1IcmRx&1Z9KZ2ZXlKuj)i+@u8uUj9Uzv--*WObs;E=I#yySf!3j0{V%IiyhEDg_`@ z810AdqRX4`D%Rs-ykT%4CgjRkk;6z*<#C)r)Ayq0iq)*BQvEBIDk#UoA6P$2{e#rw z5h{fpJFXNvcjI4WJ+9YLu&D7a6Ow^?lu9byT72dO1(szr0a@UE%zoW%WZ|y5p^~Go z4t?)3hYiNNd1vOq+i40uJvMH$>{od8`fx zvK;4~`ubMZiVGR08J`&4@@tUSsLEL+bE!eh7wOh+*2o?T4&;8I5wuzkc6Gx=JG!qI zP$V}j4S5)r1{O{8J;6%teY~^hfDvPC#Mt-u|5|nA8VZ5DvOHEIFAFL@_Az_;*{Ai- zP*127w({=Ma$!a!E8FI`7Xjk>X~8|i+0$(+UTYsrEo^XMjXV8OfxptE^NLz-Otbk6 zUH7jSfLv0c@p!k?&RkbHMm-4|w@Ds*Eb#)z^Vm^WZ5X(4G1%?AElMLT@NiTaK(4Cu z9}k>6bVIGc->vB=d3!~-t=fLGd=xPt5f{E2@4R^9_<)=86Y=3QD%lbd_fJ#R$669X zCZ!(t6(D-EM#oQPNGI`s`sKw5=quS)-H$j5I!sw+2uPRXg|d9EaDmn>!iD z8uo9HcyH^|B0DPbRok;z%RH8+>&Ku=`Wgr~-?|t+-Z&q-DlLD)m&=Gor53GRqY01j zrW)U>0P#QY@iBnXRD>KxY+ ziY?y~awv->xf22@+^t_rx&JAcR1TGJ$5G}FP5`vP(!1_ep}gJ(4k?9z)|1y zo6CBeT4K8~;?O@33M8orT%wCOICvZ5GWkQiLEkY_zo-VO?l!ohJ~-gz&v$Uk|BLTS zJyKiYQjdB`{e+yaD}{P|jtHY;mpF^HR~kMql21noFB%ulxQBe7h16_0eU zQ8|e4p(v29QIhmkPuEZ5jGAnJUP`A=*c!93&7#q}+<~xfp!`$c#G(1ul0Kj9n_up- zN*thJ{pTa-6v-YOm-K?pQC(r!7eyMQ-l+;#zVcP9G3p`Xq z{zceL*uDG|!M@=#aYI2x<{odgsL#{v+0Tq4Qmb?~3zbE_&Q&-psv>TU6}^&{{6$X{ z!zqCO6GNPo$HwO!<|E7`hv9K+=2zk3E3a5brOd~b+dED^nMs@g=lv@WuNYa~s;OXH z)UU}%E?RxW8|&;>Zb}y!GgbB}XU}GAJE@o?7T1i_RBhR+RO+W{;Gei$E zMlIv?WN_%vZG=hS`@!b1!WkCP0*3K}N`b zA9s^hrC)Wh!JJy^d*=O4j-4MIUn?4Aw$E$$R<`22URUdB@-o#Rdh$U>Fql^avlBVR`MxmpdL|2SjifsN3Zm>#R%+STEeOiE-QI$({UXY1m{Vi4Ls$1 z77l&+5P_5Dl2*s9vA*>c8B(ntV>S!`6K8rRaX^2*x*+3`<4mAze4wy zS6N91NJMk*AN@MRI0z54(ylK;um0qDssCTQwEJt9F`N(|>pwa95Z`X6Nl$HUJYB8M zxxEv8W61&4-SCmAB%YlmWVO3pGwoKx#$w-NvI?g)^;|ui?Zy%#xxb<8!I(d!@ZbaW zu?hXBg@YhZ^+v@L<=|orwzZIKXTBFtv1s%4UbYIBxh3f#a-N+H ziVN2JV#?w_k86^SuJ~8NY(xr132J#`*g>w-;v^lSWa% z2}F)OXN~?yrQaQz``QuN1+mY)(jGY=Wup^Yw$pwm{q;o2@g|%4r>79aOP4m=!{)i` z_T!qTAzKL<35JMd<9s=|>-@(D@f$BE)q8)H;t3U^kGdZt{Cg!=d5?!S55M#EE9sjb zXM29+)EB}r?27DGC&;KUWV@6u~cHFNiu^)0!se7iiZLUGk5B#D=|upbH9wA z3$B<|m5ku{xh-8gg3&5!=J?cYlZSh1cXY|emD2nyH;d=>?7Igt{U`sFB`RgOj1FY4 z;KVb%GbXbcBF8wq*Dj2a;bM=H3*G1Hj8~3QDph0cvramUeutrlX$yJFhgW)Wemi+g zNqLG%yh4(xefJrc+M>|-w!1syq30U}Ek(978r!w>J>{z6jn6TY8$OJb{n8B6T#b@osVYg* ztGe1ZzTN6vKW1Qw$uGZUHT^Jw{0&z+OKPth_6<~D%b4eYDuy8Q-h)XW0{2mcHQMnA z6wtBaDpA@bQtoz?d7FGUmlL>@5jAZJF0?2fTNQZ<>g9MFOApoI%o1weEuInuuGi7& z-D#&x=}=8%X}iJ0R6zy0xxS)US>? zNnSw-`8)NEtnOV} z)o#}Dyzxrto=HkW-)a9^mPe$oFj;CX2;nDBOVSUq zLv;#CFb(^DgKs}bmvA=fuSnyb zNqiTS(&VdPF{q%WVm&64rqn?dtul+wO#dSThwi+jpK9==`tfN&BheIWWo5=dpsEWqMs( z+V;k>GC>_N2;FWi=H->-h5nZb#+;;X<@DW# zfZW+jmoXQ(E_wWpM?rs=sVmpTa&6IA){wdwwbkp3;IAoruC8^XOY|;h%5ZL7|Z*9Xl7S%%h`TlSuTBG9uXY3sEC?DlkNQM=Icvl6;rK}na=zb zyDpHk*PUG9l`^Z5+9fXrtEWl8B1?Hzro@afv*o^Iy3*v-b{#b~c`PNN+O_#C)=%EM zYb3bYbb)Dbyh4tHVR2uy++M-w5p*6~&SN{c&(?-P2QAqz*yZqBAZ)Q9qEj@ZDZOF;{~7 z=}jbTeejiuXr^P3c!3J#n!0J82HO=ao7uS!_>4)1gUTAn>h{W$SkaPvKVs=1702G{ z9PSU4kRk(R(b#W>x-oU^1=_(|qGfL)PLOP}g}O;&^*$w%#=gV4Tf|5`A$W09kJoXT z4Sex9joY^bi;cGAOiOrwRHFW(uEE$xCYVpLCM{{K$~|T3#585sNPhGa#8kkm&>Pit zy+^#@2^j{rl=jG-0mYTTrxARU7Cp<1!yh&ZEyIu6SSYav&ZY1Ysg=GbEt3OXlD)^ZN zy)ZTCiaB66@4bh&_p!;^4(p{KTvuAtqCvV@T>Y9EP1hpe@!Fo5)6e*L9Ia8^XlT^% z5IR**QY*FFkO!AwSQ@q8w#nnBZOLJjT3*4mESL$rxhE(qS<1T9V<%$r-lW6H4W%8L z$9tDp>mz=L*VmS0`pY_}*pj_I;+-Qlv~5b>K+r&f`cs$5o`wvreMi4V49(upuOVwY z-by>?V&mz`A(Co~a5KkxbMSb&2I31+VV4|<2YMJ!uoLj$Esyh?nBH4E5E4dZ*L3@Q zn{!5G2Ops)T>ZAX+j886LPnkU!?UhcA7>nDC#cr!`4?G=4Xa028J>`G2`o3##4dW) zr_&D`A4I6SAyt^ugL-jykZ>C4;I|kCjkz5HcLGzm&BTxvz1t=VYb}L~yxWM`ZZuyB z%KIUTl#X(&$J)>@4`S(02eaO!z#QqEhp%0*vzdY>v$H8FH0ply4fQQ~ zQWh_2v|$(AnmKQCwQf;QVtUZHb6f~lqUMUme_F9QJrcqmtohZ8`+lq%Abo% zaUUYkHAV{BOr3i9*`ivFO|IFjxbsA+Z5w@;h_pqvfBMzmJ67S^#AeMH*XQJks z506g=_Mbs2%-={!hznHyNkw;7#Z4s=vL&fUv=Ci{4pJSZk}Hwu;;QjZxu81QDa*r2 z9cV%OzWL!?w>J4?Lniwb(KmY}YKQ_Fx4o|!%PASt>d11HcjgyYV5BJgv(ri_?cyUI zm7BEN|9LTbbvCM0HV%nb^PRrfdm0ehN272sQP5Bi?G#VSsrWWN1h-ul@y|H#KnKgn zBUZ!MvUg8+vt)b^!fy3olJ|7AKNj>(DvU9LEG>iHePE zy5s{pTZyqfN-n?hq~rzVyt-z$ZbSetUuxG*VGg(2)Y5&iMjwpL&tnodX$txS9TIQC zm4XGO61+2KSgK7?)t1^Qme2Y3S4R&P}gYu)U8K}Ld>ix&s`+WanVo)Gfs`uUFURCcZ z?#b;hG$E*!HC7worXAXThPE=Om0uagNVtlzS+MD)dHy>WfWS0Kt22f-cvd#Ng3s#} z`18x~(B{}}QhRd{MT_~8=3_lCN^3p#f^iKNyY$(E^&0c$QO6K{zmPhizsPzhF9UBi ze-I$e-BsHfyvGPaKX|DSI4qI~yU(wAf%`Tu#oxr3L9+Y;5w~=n zy2o*`%8P(}mCH+db~=zvn3_*!eRh*?*0P;i6s|t-NTn)Z=iTD8l$KBea>L)rBz~+P z7xj55+#eb6#U0-l9#`=vYW46WVIlcmLh(n%&hdWZlJz<@c-7XDcBiRkUH`58;iv$) z>4KEDTlRtP$gcmBn6rQyeUmxs-@Zgn2Wj&DT*jE+1K9>rOaaQDXSp|FT}z?AZBFZU6r|S>k`TBmS%0i&1$bXgkbs=~ot?^&Nn8rq4jNJmHIqkilZdO`|Re`w^Mb?DC6HovdqQ7Td^-8_x42?^SL zyhUi}H8K{f*`J{?ObOl0Qc_Zrfv3m0n#`cZjSipj1$Tsz)6#OiauH1vxtq|PTU=b6 ziqk5BoHYfjsyx}WZva{f{vR4kPG6c>k>A1Al41Lb8@7S4wyi1)Bv&%0vN#C`+o%sqX5I?qmtI`J&=Zgjgx@3!8<{~?a^nQ@g|C5 zfP7Gc8kdGDa|?^b9x-qyQU70hkv3<`WB%7oZ~cNdASHT`-V`C8Pf&37!C!gG7lEyHT$v; znSbyq>EcObH`8dZQ?!0Me{-eTBs?lAs=1|Q+fGPWShyem$A0P2eq$DS2;^EsOHFKa^tYcsWtf?n!Aj@?)yvDvckTJe%ZJ5$ z{d%L$NVVV#m)QI!9v@d#RdxQt1@rPz@_Y9ZUG=@Zs{eQ(!8Z#F3vrL$bWFCeh&gq@ zF-ig5Fn){?NcV>dHO9kWhGB|zx9h5|T)a3xj=R{pBu|pV6sHHz0V{guRV%p($;nXg zk2G;_^)rP&vs=S~S&t!XwHux}JbPBsGNTHPbOk3Tr-tq={Q&06CBU`&yeuyh84OQ2 zR7KKBK+W1CS4K+BL{1~4qWV1#2D0UDq&A+-V4A1q%a<=poHuUX>?318AP|;zc+bOF zMl-Ra(!4yfK0ZF_Qy)7!JC{g4SJ{oa^pzd$69^o_!a5HhK7^`fzJD$)O>yT=R&;c9 zYG6ja_}(u~&>F2fg9mfv=g;5I&d$2+ZuJ3Zap+!S?Nj*h%9664t6RtZ-5)2B~t z{QUe&-R5bc?k*>3Vz0Z7*C)QJUtpEE87vKcGBY#p)<Bq^hc{k-~JR2T0fmm4;G&eVE*0`7hSbYY8YN~y@PC-sbXEcA%5Bxd#3~|I6fR~;L z;I;AXCcR{_bnwkIGI8_ic;79{l-|7S6zm12tzm!X^ZdDH_KS#ABVrR1lj}EbbfZw2 z1(vE8oEAFinzc1+F0B7KrI+y0l#9IIsat7_;ii@J({Xlo_FhDn>6M7gXnMSc08u^# z2N$X5%0o2^)s;iy2ycM18kC%Ar4^t88Cbi7n{?s@)z!kQV+f9ZyoB%W{K~uEPQrhu z0o8o<2=w{9)%;EBXpkiq5v<*y?G^z@RIQ-w3DtE*En$cfYG z27e^bb%o=!q`A5I!UPXN;JZa{vIyL}=G3st!7%9J1tORi0QI*wc(U4z0M?$eYp6%pw#bF;JLckhDdjHF_QfM44HvE&sLx@%Oe8@ueeTnEN^ zz%0eiE}tgm*>~`%5ljt8G*7HeVw-zr#Ml;dxFWumGm=Hcgju=gZ4Mf0hL^Xm&NJ~>QmRi6j z2gK1X3>GsRGv+0$ZI_ zo(Fg=dXCtg$mvoS9m5FB5b5eLUTzr$?nvC0Zaq5K7Vz0w7GJ+Rx=se300C+}vnX|| zOGLkIOcL{~jsB$h;<3!Hk|q6pN+~W9u_lhM4lnkmAWU1Ya~U@naA+0(YKx#*aH#<< z#4KPvC<6rVJGfN$?8D_$kNZGvK-s_=uZP3s-J%mkeJeNMy z8kM(G0Gg69rlzJt`6~2=&x>VL*oauWL8QQsJ2qM$Y%KuJag7$tpfD$g352%--ExcC zgT)j)wg?V2?~aFpM_DK6AC=w2mIQJSzlZjcZ|m59qFT})9V#tJ!FTS7;g{a==T`r7WeVYI~F#Mg3h5Pw1P(sgidzU7wG&`e7(2zWy|ufC>6LjlZbFA{DMH8JkL zyhvmo(!DUawa|?RndUOaI1n91!2#l_CV&To-KhF{0|9oXI2NU3ryyi`;wxYL%tsQv z7>~`yAuq))%}Fxqk{Y4F%<9n#Zp_LesR-u^qr?7(2-=e@(S(Jk(2GgV#w%Zp;Ifv zBq#+AleO51nsQNh>{;H*@!&VR|KJ6pu+NVCM5KndCL&L z{}vl|aAfE&kW)`r=(9jS4ZNt}MFe$NWMn*3Zf>si%3yA?RA9qme+JxnuC3Dsakd1t zI2dxML|%WGX$#lt!j&+ zhc7QL7YRkPXT`=Uns?@jx^s{lKll`G+NOjV(UsH8^p%>tqcet%rPO%U9?7P0QVY}Y zJzvNXW7tc0QS|JPAtL3w8f=g|pL`z}w(C-#&)%db!ev4JEM(2KM!=13>-z`G|>V<1FEcpyGQ0Yz@*LNs9oWL_#B|c;yz|&|t2Dz?+WS z=pt=+MK`Cp!|{^?n#uhkeuRYo%EstT+D=Q&bdFs|yzk9JjxB5f2s%=Mg-4c z+iuy2)ULneq9?>u`8-o;ZOc!ejmu9$li<5*<31D=zgJ6K;;_(zv?$x1qk8#n>tXH) zE1bWL@BbnyiA4RSDb0%)MUEa{Fn4wvUb=#+u-&!A4pczjUtlkAK)$0!998$AaI`wS z&)`$jvb{KMwQnuljnP;epYfJMt0_tI#WB-itTy?=LhcJVY}zobE;5|i$|NVPgzo?p>@5wvv{nU}%m(Q?avh0%*d1{7dFanSiFBN60e zd0>-p(7_m$NF(IHxmwo6nozTY)3q3Xl%Fh=+AzWIgQQxQ^z1G6$MrU-WazH{Q9xFm z@(Llj^Mc&A3GWt^q{V&&<~-Kb=0SO)YB<(q|`#7be+ibyP3SguT5fa zHCBx7eMFhNRL^Exe&FEBgSsR6E7g_P_bz#Zz3iB!tZLoE+QY6CLK^1N-5l5+hsrkLPIbS(%CvMB<@J*aPJ?82+A zmny&3TIcXq;lELa>GhC8*$SgHVPl%zN30SY3RJJ0$Ap(IAplQ?>NKUu%)557`v1(~ zL2=}Kx9={O6143-*des=NSI*>6cd_2CTdI4Z~kZj9KP3cSZ=CNLTT)&G_(`Be# zX)*R(_VNje+2um*zav?AX_R@(C@JTdSyawPSAYuRD2 zh{M&Fj=kY311Ft(&@A8l69a#Py#ETx>0nhFiIqji_tvNtVV02Rnc{eijrt|_Qa%2R zWS$Lj%+`(B1!l>;IeRQ{Wh_-4^Xk9giS6sA7 z1Zc*HQ3K8uH9XqB6V7uzDfJ|%0em-~fT7~JdVWg%F)^y>0J#`ne16z@^)c{xF-M-% zR3kA&?^66CmP+GeC53&)LPPgKl=GL}(WDp2rP*SM9*iURA3r)y)mP$cNj=q}K$%_> z^i>u~t_oHgW7zIY)uT=@&llp-vx)xbfD6wDsbuhkD#K|A29ZuI4eI+Fx{h=fyh^5G;tTyT) za@1iy{k)IHj2+m2FLJ=^O!YV^x4+2IJu&3oy-#Ei8J3+-ZaID7?VU0ZD)Fo1+83_e z94b)Dep9{~<11SxfZ?N;%ALN_ke@=$rJ29mi_Upwh(=bXIXGym{1~3~`aD)ER_p#Q zKh@LWNXO98?^$CS>o+F$IS?6Pu z%+#un^(&}}4-&Qk@nQp?kYQ3_AaaVr9M9V(hADJMul(m+mIcpQo8YJ_CJpOCYi6li zBNWXTjCM;9iAFdj*AUXv5nZ4rEgeM6wpvC+l=~LcHdUV+`K}5G*pF#u1RS!x8=~mV z3ah$>q4)K)ImwU0#ve9>8E@tWxkad#4D(L{D^#^}h@jkU zs&y96lSFo`2L%0@+O2yQLQEB;e2WkHvC{j0F*KB_ShD~i?1*L7d|kp;vzc?IK$DPM zwYv=jH>Ez*g6Wu6qQA$J>&X7Frk*KzYj%$ItjC0^Wr2I~sBTV5T_!3@M#5#P7z2l@ zH?{LFNio)43odE3)srqr6_Oh9*213i7Gf$Y3IG_}G5N7k3k&)BRSwjGc8O>xLl^L;ici8hw0=>SG=a{#q>i z>PHwp%#WZj{~tORDsJO<0Y~4L$1L)I{(T+ILw`Y@j~R&Cy|`_ONZ+ttz(|N z3a}lsu|wQTlDa)J`fcne zr;Zh6Ak#4RbHWS^m5oPlCmszlckR#4sP#}JJ$vzHUz^L{Hr#8iDgeNQ*Ca4eY-Q{E%xMY+E|!1%bnek1_B!|Iq~cn+e!d~#~9p6AFniQ&Ue z=vvh!7b>(rUg@~yb9m?k`lwVvN?%ks)z2}0>}xR=lKQ1B@L9OmyD|-W$*Xgb40>24 z4maeM(wmQ0x3);s@GtDq$EAA${;Q-S+rFdDFw7vuE6`mnh(sUg7-|OJq92}% zl7b=EY85Crv^ZB?&LGEAg^2YbDB55?+>$H+FY@3(E2dXK=o*am-QF@Pqs`W;vCl?N z%;3a>=in1sJ$$WydHTi@HkZX{X{!C|pdn)(?!$>aM65@}C_%V_?f2r$p>@}7_xilD zqk6loYqka4{eC6h8SaqfT)RWlF_v(4Ph9qH!?ono<2z|W@mn=&{=Sf}-n_4@pH&r3 zpO$+rIjhAH=$oYYh#zxaXJs$@Q2oyOy2Gbrk+1dz-j@brE6l6ogrLTkg;p(apRii7 zlN?_yGIQT9Jl7Spn&cQ|NGyB+3*lt{_Mt=a@zP!}@4GDCCMS!H?ST3hA4Oajebbl3 z`^C2<`b8e~;Gs3#EaQT%4`@CwMT;WuWNznO2N)cz9R`VG}npbDQ+bOQz?K8koV>{Q?o_ zsnl{lV8}|Bxqb0xd;45&f>Cp|^IUXdVhBiO!)PhLgc>#te`?uCut!DvxY5!%-4RO% zuL;u|(eY8jGm=6P1k~gH-a$Us_~LWO4ElF$9R3^6);z>#dd=!r)r^cHK}p91lpIxA z$#oh7Bn%7;e%=fZ2@!oDAV2}i;Ax`!eJUyp3}i7cnf3|Or;#9jf76B=-4k-S4+`C1 z($o8rG=@Hi%uF$>nDw;w?mO8_iUtcs5ED@iIVD-)lXT?S2y&BYYQSb?0?}4o;BOj@}@U3i4x5e)BK@1Hez((HWaA)NnDDn=XP@8Quj=wv; zFsc@6M#~>SK87q?w4?SyLS(*!Dsr(=JvEJ>UDLt#k~3ko%51SGDN)3g1vEN--i!(f zDUb%G)bIA=f_9^2mq3+L+TUMdt!9ai)1dk`pGD8xxVX5Hhj&QT?t<5R_Yz|n818^L zf1%-cSKseoAz>fP#p&e8`{L!xvunuppy!!Q^POL3em9wI$jH1&-|>+EyO;3KpNXyb zssD^2ZV){-?;`I>7PtY{$jRNELPVZ{f%g;arM&Z^5b2i?J3BjKz8q>Ano>}Y_dCX8 z_Ca}Q6Fj)Nt?e=yi((U~hCL2A^7>3KMF(8{u9zJIL+vZ{@UZFnRQ*04V_fRH=Ww(? z7bWbuc7I}G;xTxs&u@3svU(}q_xGPaoE`;0s&i+v;3eZtBG$-PZUqZ*j>xhv_1TLT zmq9N64iqP6dxfSyKDfHEz1{Nh8rk3RXoPN>wS9h9Vy``UH$5a|`OdT3PV*h4M~66g zP&?R&XQEbq!HuQPCFTl-XbJEDzih^Wc0Dc|Vkt@x=pH)~ld8OX+sQAS3B=&QPn3}tg` z>m|==f1$MofLq5i1vP;&{S zk1gMxeYk!5_FGWBAigiG`ey??AY4;RYZmO6g#(U+L4PBNQ7-Z~s0()V_R1-*=jP_jL``750s;c=@mt*njR4|Xpqt?3}40?hk zux5Qlc~a8JpG(eYE0fWC&Rv9Y60R7#3|$~6wq zuJ!WuZRzce@U#H4D9*|{FDom{%F6oG#N;8k)(JbrmMZGrwc!iA{jGxkpQd*gW>!~8 zR8&+ZoZD%=b_V1~LLWIgI_l`^hSte~j#8VMI(WGjR%qN9u#ADzvnZ#IEN<(SP=fZZ zu>T=H=yAOxd!+m=C+8w4JN_Il(N|MdeVdoZ(z3A}S0SOdzahbaor23Y3kU~t*4AvG(WPJCH!wg-M@P3j-6(bW=KZTeLPCJEKo3ACxTuK3z|e4@%F(z* zrSJ9w293hvVi|B-S9iBFaOReNKFhv)bb#^TplwWf?_NluMt(Dp^ulH0w^da_b@lb$ z_5c)Wt;fQ$>>nH$dWxhe2M-Rar%L*dMSo{!V|&6>3Y=X{PcIBeM-6jD zt9M_7bY}n|9JfWl{}jVg7UJ+60CYh?!BbsbDqyjC%t=5DH7%{6t*tGsK-7v`$*>8y za@qK75@68*Mr1cYaOFN|`PqP^7%VATviasdrslZ3Na7#PA)M+@KUu~@8LuOFC* z)pjP@=1WG#G}aLUfnYxW7#*dGjEn^C`Zhb85yW}oQ%^THe(~wL&*kNOfM8;-)7IH} z4S?lhF&B0qd0friT>uJ&0)!aBcK#~T=IK~)85te@ z0WJV3laq%>5CVZ0$oSg%CWvQJSy?$+K9`nT8tct%R2Kq(bZB@u3kK`kKx!4qnwc?z zZ%1a90DFaabylyqq-0!isub6xtPiyQrv0T30t|rlvB6Xw}@-)^`8S`kz0-$OP@JEiRVmV+V4s=EP!omZ=@ zrmmimme#cHX6M13mzygJ3C6Z-{dBKoPtglAGj<(o;8u-_D=U$^rVRLdlACe#$j$?T z9fX-C5G<^}PfQ$B?JJin$wrO)v5D*E-VH!=+0cmeef9tMK?8m%VFi`H5KcLV*yv zJ$bJrBQ%M2;QLOp`J=9QiFQ^;N7XJ(;}w@x$ zI}9;cSpMlce~rxVzkjJfl5gJ>`~m`~C@Il$b8|JSZD}D>0rEJ&|0?kez+Vg7_i~Y^NNQtD#>d0kF#zYH zkT-Z48~dSx!gVgoQ9s&8v8mvKNy08pPfyc5k&beHpOloQlw0hyrf2-@*)$Yo&Dz%W zm=Gu6IuwF}^@E|Duu95`unedHz}j?SRc)$H({S z?v`WHuDAxFO#1O-f3Y-g2`W5(IsAAXW+`W99teXtWPE;sDH9 zm?Tf{_ztdSPjDVFI(-&)T#b#5DHs^Az!z$KGQL75_s(0l){zpjh?0_$`LW8P5yLm} zBoKN)&7!EG5d^}x7Kk&TR%r1?L%&W&76uMBh>QyCamV1?EHzW#yuqiTp)pwh)ug}> z1L;f~Y-a>KCJiku<%0(*%h$j7Kq^SV#B>dIhsR};=BbfUTK{ua+uzcDetseaY8f>( zTFb1#uKL!T>8PVr$F(IwWKR^2ymaea|My%#l3O&kqB%k0E4qa7UV~bXjml zyA{CVp&*2{X0xAHa6jJYc*ku`4q_&(FiGE>oV(x`gURS7Xuw&fD&<})-GmBl12XHD zA3q4-GM$~B-V~Rg2Fk!a5(UWIgVV%eJO7QcwXLlQ{sQPwOjKrOCJ44rAPj=^kC<1i z<|pQ*OCV^w($>}n2l$&=dBE=lV9aeGtvNqGR9Bl2LyhW09PaPm#&=_6ks2nisG(W; ziJ(8Yb*NbU0t3bA-om{pGJ;o^R8b*>6J%>=$L6+c;SL!Pd{>}Su7s8ri9KJ^h#^=^ z)UER@NSfy7&!0iz;Q?UnDhxM}2kU?S668}%`VFLddU~*o1I#UfxKi%!0^o0pJx^Rw zBF4rvNHR`LjHIB!q?GXJ@zm!lSfdCwJz&OZRllPKD{L@asH;(7aSJ{fnUH|xFNPK@ zK?UcR?5VkV$ol$AxN}I|u-MqxMv4v5FXEDiMn+!G88jOh7zjGwnD{H_mz6_Od=3tR z{{H^;_4Uud$VWhsCXsi09Tw&fVFI4W;T{-rYkNDQbpk_$2kw|@3z=SB4F?-OhT`ns z(V;qj>P`vy6rnIU0oQJ^W4xt%EavFQHR-k{7#$r=0lo=tboqL)oIFE}SegP-r+<)7 zY6k4x2h;;b-S5~gU%4Xs^eI(uiZIK$G16T4_wbQ=MN&|l!M6XJo%P+{ca}@!!amwv z5m!?qg2g2zBWuHXD@|x+Wfc@0-1G|lQe;ew|F2&LOu9879wUbnC;}hH5WUVXheQmz zb7ORPm7Y!o#`w9f5BJB1CsL3FLZSZ(u~1rCLxB3>!Hm@qPZS;PhnOfG-OT>REZ8lQ z<^%nzprys&7z9Vs`02k3{?93gJUI+>jonH7#2_vcH8G)s+gx5=ZpC@qnUBP^EG0^Q z{@X|zoUd8B4i*ASo?~jxM1jBqf_WIxc~f4P=2}3&=yup5J4+xUO!Jq1piOkQ;5p`oF;A zi+Xwr!J!7FsG+6heOPxvpa1rikU|z4G+l5%T=;PL@?~#0>0t9_=H{^6u&8{&h$*Ho zb@uj3LPB=9w-$kUmFP40K$5U_R>rSmW7@FYu4Ry{P|v%DHcVKKL#nR%Ui~I+#3o zV{j0k3o0jH|2ySS)q=%>@!S%nvY%4lifcO~~78NBVdGb$U^U#C4+ zr<*t~F3@EUFCmCSzHTiMz(KxH+5+h74_T~ynKmrDSp?`VU^~(YUeNYJf}Nf?1_?AM zEWMk2FHu7k2RGxqR=)>l$?V;YLBl`s`%@wMGp%WM&%Zeww?$*__Fb8+jg77N9AbVV_#Vd2Rvd4L615P=mdC3J%*sg; zxvc~0>jkN(sUhG6K!`5ynv|-vzVkXSbDrQpVAahGgVciG^=rxvos8ME*B4V++D;oA zyi-DIndH*Rkt`hY4u+VotV@U{c@%n!!^EJ;2eWE#(#`PMvw;T>;gQ)X|2?Rr zq9Ptx2`k8Az_q^TvAF;`FcGjaP*&GGpv1iVyy_+1<6{4%5yK>DX}qYC|Ckz-KgeqS zA3+~9v5*&lsbQR7o4hbB4Gl_pQ8i}R@smpjWg(V96H`3JgXv)$Nav~PjB_YTx!%!0Vg`ETT_*@QW1;tv#qAL zON6O$=j7GV>vg9EmWi$z*@p$o6#sW!JhOyvK)7rO9Eu>9j7@_eu3La+fU}cobeN>*%Ys_ za(wglqPpV{!tiOYj=H(P%I8yj?{!XT8pft0z9gRFCI?o`%tf39zG;^13=C}!Y83R- zmgLb}oo#sads>~9esr&In0vXc*KM+;#1MBj4fn>&=bk<%SD^;DBzy@DfE1tmJ}cx+ zzn}oSbjSU=gM$MCmxBJ$9~|jhn#|Clb$&+{HEOppwCEeq`TNbU{i|*CSsj6}@DYRO zaoQ)Nr5hg=*LQ`}G`ARg-`Aj$3h`>}Oci$DUX1Ufot5ai9lgNv%eXmjWb=@l$7Y?$ z%-rGm$AVV>DW~aSQ~eGvuPT%tc8IGFP(qa82B7F3sdM1~H^Az&ssj)NHDrO{2?_Zn zI4DzB=gRl_tgde2rbu}_re$6p^Ve51&FCHJ^+uj)0%Z#2jbenNx_aRDwmn3&<&hGk zSk)+dj+g8_a|P5<_K&W8H^d7XCqokv*=hHh*>b|VzTS&9ert3{?%J(oUpgut3xES=;6#z#2LNFu0(m9O$dwC-~gQ@xbX*BE>+{%}&1 zRGu-Q+|c8~BFFHom-Z9%hRWI9ITI>_0o2H1Uh2s+eotBHj;I^@M*AzcO+DGlZDPh; zfM;GOA_^!jW(D&#ny7I=Qf{IBU+5T@E{SI;O{WEt!@2hzNppN(CM@|;xJ~-kcmLkC z!^Rbeh*r^!^35Lz#`kvOMs9# zNUR}|K~iQYoDql@l(0`vi8(87h1rzw9dGv5iB7l#+3)G;SD-j%()`r=a?7{zWW^fEn91GOMRQab;7LQsO!myai{T$q zWeW=ptUpW}&f$MKmhFe!X?Y=>mwkhNdAa^)SD zB?%4wqnef^9NK$RzwFD&E=mZ62IEzJ9xMPr;L5jmA>EfwgU4}-v@aZu{hfYwe=pU2 z$}pOYrM>#S+;7?fQQxV0g$Cj)PfV{+%u~*zUA}-Zt+h}jIe)uogHRFo_sq5&*gLW0(5@2P)KkX0xW zVcFe!IEaOg*Tn&py$%4b^K2M5669eJp49R>pJa~YGaF8fBb%G``@3qxD*bbF)7vg+ zFRhmRhl<~ziFuHbSU(8+I)qs`OaIwH0CkKx>r~VnaCU^#m^FnKfoT<~QQiFW2g#cm zZ8nE3j+-+B;*G`x4hjIag5yV09U}mJr2$M(%54QymCbrmK;Upb13*@M{_dr>vMjXU z61L;;Ww_1$;$O)e?vnZWmGbSp&4$iWNc2)f(&y>`A}_-;F#vr(*RDant@(Xv2jag7 z1P3S|y@~al(JCh#Zv$)=1z8|aHC>OT(Yz&as!KT)-iC1Lb^zw;y~$!5*eMpsBK&!@F(X6A{hMMceR1X3?Iy9l2rAK`!Cef!GF7RrVI*(@FJ; zE0GDTnQfezbMpH9Fzs1m+rQo9fZD1v|G^YCBbdvVUB5WB;Bt@?jj_r9*KpH=zGQaEez{w=adCpC5DQ~7c}3zP zkOn~!5gkY-U!YwnR-i`B&W?{{m*~h0~;m+Q^-B>h0#!% zFRz{P{}mJ--=`kf6uFi>6FRFu_q30IZ3G>K&&bpB?qZQUw)`z3V`RIz+vB0!r>Q{z zGbV4mU}0$qQ6C3tSs$MZt*xyF zCx<&X*^Mtl6`Q7(2e1;g<2&zGa!rcC7ViNZuXw^V~PqJd~HmK{AhpgIa?UBLNV zhYTL!X`nUhI>36Mdc*@%=wH<)`5~MMAb=eVqB!e_R@{4mHHr3+K|D=*gE%*PiYD@ojlR8kMLfa%#(1Nc3L_|EJpMr%27r}Ut zkOR!813v(Uu*AlurnBI&kTh=6b?H77WQfcEUnN_*RSNk`<7qcIl?fz4Vh{GRna}z`Vg%I+OQ1C+ajC}nk9#C|#btjMhE8JMY zSyxq8QvzrNy+*|R5>RO{w;|75{zzGw0J%XtJTxG+pst39YH)UX()snPk8IX?jsx(B zrojJA>Bjn=ya3EZRh9O<>Q8D%E~q;UjADuVfi3VN2+<7yjz+yJCjd@JJr3OAL#Qo- zT$!AloDg&Z5Gz0@fR}6K=o3JPC6ajf5MVKJ{{c&ZmIr{&@O!3=GmzZVU?R{@1%RW-b&Qf0KsReBk!xGo60_(b zH8+5?K0ZF{gX;RzxgV0n{<~zSeES73g`l8O#BiWSjE;?c3ScoT7T^>J9;l;3iiL%B zUSZECAZcQa47_0gLH)qQpeA`~WAjQfqX}9~khO**9gF~$1d;7A}P5o}D6{@QJ> zYk<(*WnyXt%o*B#*#4&k09fQ4O2Y7lEG3=%$vHCh13QpepC<~D3GuT-`q)n zI`{ttgZ%|+3djWlp&#X*7%ku(BB09vb|5l7J{W`m2p@qM6!>a1h%{wo4<2xFRk(Qo z@v{$YT5>np05oPtj=lthJFzNIR06QsHi9!Tpz%_iJm9|y#AhcV8-A{ z!13@vkxlM(Y?H6wNDfj1M2`VzEg)rM?v}8sCW%lY<@wtSor?;Pq=PI8mYGf+TQ`N6O8KW0`ed16m1y7XYUsR8c`ym_SvO zl7RRpmPJ1tSakz{Lti>M{V!~Gd91RpcsZDW9+v)!=yi(A7lfcrkxLf11^$c53JtJj zg5)JpiGjxn2?^H;2z(=i4lkpwIX(EkU=OzQj?aN%C|}bTI{unQM@c}fgH*1d^k6X= z0U$=b7+GA7H@Kkh)rNP-5yNusLkZJcPjQetIxB zfxz4}f10K^h#SKaXWM}k1X&1RDOa>k%ZK%5 zlMcX3C?Q8D{&V0R&mhp>5`$%|kVh}s^T93aZV?It1osH~g-PIU%H3S*^&-m6hmlj0 zH<-Z64kwR$!f{VY$)SI9jZJ-IK0Ua+cFZYK%fan2(EDWrz zaUt9Txr6YyLqlrI3c(Uvzz4)13nFZqX^O|t?=^9W_7@-ulK%kEG#MNf1lxfeANX&v zqG6!BNMOgsfSn&n!^fK_1Dmd0sjuO@$7wUuYW1ZIlG&&O_Q7kpu>yjEyniX+=@-3M zSEzB`!>sQ^u-2Anh=>{ zuhNM85%=qzr4< zyXVz8;WiK3E6dHwieo=xZ1RV2Nb?+$X?OWa^E6HB6^}GlHF~JtC^avX@0;&;jl5G0 zx{B@k&J9mnt1dln(z|g)#(lZPu;%sw$$7V*5bju4@Qc7!Om) zWKO*W>!S4|w%gk45!_x9Yv$Qrz+A$~`4e}H&eimt)N8OHugAYWnW6ax5Nd(C)caECC8ntm<`*Oo|V9gnyoTy|kNHl=0h|Ev^ zI>iIBjz+6WJbsE0BO6QUDsM|`5AXW=JD-1SdGs_IaVqjxRJl@TNFVwc+C;O!#6S8W zqTS^M;@Aonc%tFq;68Q8B%XP^=1XZ)qG&}Z>RoTU>4w@c%wI<+nYOJGINJEe=}h-_Y~ zp02aDuyhJ)g*AF?$BGyxj7K`#%0dN2d} zCvi)bf<7DOhL7lr-|qy3ObZ&XpN>U*E7z$$75F6Vfo4aCBYCE0+evoGU_RPk`Ws<| zUT@H*rBhwpu;%;fi!6cyWJZB5pJlAj0~;D{+C6!2x^r^x-7z}`*}YQ7HPe+TuGz2G zM0yO}50q%c0&CJL3+}sOllqkGv9!M?!z?IFG_=2M_N10Io0M-nVEgXR1fR2^DiIs6 zyUT^HOL79-(ieY5((WI8Op=HoSo0UN8YITeei2r9)o7`zYpmqUQb9<(jPAS4sfEr{ zbCnd=V2jrJst*0nO=T*}+!=kp3P0Nf@`mw;8YVOpSYBXW^|-TFquoNeu9*3^^7%c< zvJcs%XDzi6a!rw*IK@u8Z7UIl&39CjR?v9W%=PFy%t-TW^4m*DI?RZ;)Cl+tL1fna zp4V=)oC!rqOPe*ZH1w?gF2+|1l%=q7%Ng@>|1ILRupyTPO1wj+W33zxt3k4$O8F7? z#!8BkdSz#$o!MpzbD6w9mTQ{#qZcqmzmeT|v${erO{-p{{N3DVeKC2lEAL45-O}9~ zq=!|PH@muI^L_nZI8|L$>nV!D#ClX3##Gi-;#prDLGfZ^lFP}HhLBS?kJPw%J*TaC zqVW6B#^^W|Q`g7FGJ_vB)B2TTIv>dMF^bxRFJt#GTJhbtP{bx!9%Z;>E<8QW^5kkk z%M1O33VuD&>wZA!=uA4+;m5}n|LQ9BIsJN}x-Lm+kKT`LM_uRzjZs&3kS(jpSSSO> ziHqS9)sv==VXnUJ0mT(FeGkLT#8)hd+oQ!TrN$!F_McW{z~erW=Cd0lr?~7R-|5Px z5UF{9<#n|2Iyq}2a=ou)$jAP(80E=RyPWn_)0KQ9C(1p;`MY0?+i2Dbi9h~Pc_pm6`F7KgPuUvPbfPPG zmwJWMsW`uHLTWxgs^Cec$x2bPownaotOD^m;WjeljfBRd%b=0h6<=Hi^X}K7T5LYI*H9<3%`MfOD)59qm5zJK+0RE z*YO*L{l1fs*{gmiS?eNZYpX^r={aAmT?%8Fa_n%cCcPM*7!oHm@yhq%jU00e36DQ7 zerK{Y@r+Sf92?*6y`^8ZVyY5a_4K0w zwGCeS_b|cmy8*lHZoH4K#?EGCkQdZZGcd1L;)WzJ3FnPBUp9ZJY zudm#Pa%t#njpz%;8;8OTC*RlF&WN&qYF*9H=cjn!XtK3ffpzsg4R`10 zBaDM2MnC2I$-^n~_|X;kqU5nm&x_m@2VD&-NPPsBOY>3)Ko`M$ITugfr`5G={p)D} zW3o9RP6L-@aV(~;jY_8AA9IU3*EDu|RSlst`Dr2+;-*Qh(M9P1ht5Q?)foS=7^x|( z8pC)_bA8^MrIwO0@vFqs{*>%ZuG_ZL5sZ1O304CEq0bQmml{)eoN#@b0x!SgRJpTO zZPtvB?s&;@Y)H!Snv~B4zq6ytO$){wIv>0!NbAf< z-r}vu=+os__sK_gN?z{zq|KsTy<=$TUBy_y&yk4l(-*6i;|wcL3*u{XltE@+3Kdj)ytf!Ie4hK+q9?J zW(=*9i4BWFwibwT)qI+8pVR|%#X*Y!jY^Z?S}TYye=19!?&@&8b{4Z z$)682Sv(o57v;U#XbJxBr-s0h-nG+8R&uA%eY79hp>3Vh;;h?4UlioYv551+Ekf~D znh#!cS52ROd&Q-q?>kC|hF?uuF+5t7<$2lbE+NZ(`HsUimS3ko%~`>Qf{u#ElFHC+dH7G&YoDq`mpgw?@1+x2Lq})={^droU?pl-9=2dP%0ck zrOp)HbLDlfezoFH_^EwkzAePtm(5&k42(v9n-OEJLD9LOQM53O5cj_phq%AHfuck7&LW8f9OlS6c(U0x{=M`^?)ZE;K z@8U&)dR{NSF3|ElDC1l?k^D2Z>b$f*duh#J*njff^->~XvilEJTe8M-u(0_zqCdoQ zXD!8wOH~xqJ23f=)vp;iHQ6lhuU{GG>?Y4@q?`3Grnr`HGbD5q$MLOX#Z_O&Wn9Th zN^Q3hO&N-T!L-$cL~8%J<%6mpq9K<07A-DlIcF~D4c&1wLoQpaU)MFi2cP8Y9@Vhb zRFR>R6;(1nV|Fl1@cf{AWNmt=f^sN#K+zPJRuWV*_uT0qo8SbuWaE*DG99n5;NQ(+ z_EX=O3-bJJL~40}-7&IDgGNzZWABgS@G+_B-)h?OKoGivc%gE#>a4KA?mDN2E7F<8 zZ}iO7O9DSQ=}X_nou12QTy<243rq2`Mp+2_`eV{8=in46O9}Mh1Fp2(f?nmnVU)hh z6f2wr=3P4hTPfOt+cEp@u`7c`^;QQYqTfIhwRPQfBBE;lpz40=9nyRj&Xj=hGpeoS z@{1D{kz*(^%NrkSZFM{|*lWMwmCcl}Q-t`w@PygB7yJ+q=$@oXetcs97dMy#Kvz(u}APKqU- znUs4~*+M&xJ>|6f>0+~rWhorYgqPXBnG$Ghot<~)D4Ov#2dY{mQqm+NYq~|oL!1ou zL-#g*a){=Po_f7IlO!@rAZE=Ll$n^Gy@&qMvm<$YXo}`zsn^5spYqda9cE-r$dOd9 zhhJBou{kuAXvG!#-K_2YcQzyJ%vZ&=2ggnT;ZE8tln!XKQynqzTuQV=75nk z$oF6v_D`Q9)rU-c?8kgMOryS! zZ#!kk6-ha$KE9M4ouFA;Odifgy^<>IcG+dP*7aj1?rPGVi^Gda`%fqM+#CKpu%Pdg zs*>gvG6@^c8=n!ReM>{3_N-ZjbY7 zujtE)@>dyi&P-Y!*UNDzolkc#S$8aq#p$V|M5t0;l@nt;Sa!!sR*O3`;lNuXzHb5j?aW}g3u?AX*0#o#IZ+2@Nm+vom^L%SZ+ z{8WUc$#Np2YtO#L+q2GH{zHx1^u(5Aq3hYja8iPxXSQxk5PW;kpl z6f(|{3zVCz$ru6{fHPQT{!t;3BoHqaBR4~5>uit!dcb-@M zhUxRlUCT{HXQ2b#@mYI9nUM671_j*N(YJ&`SM}SbtV-JaQ>L1h@d_iycs|?Byxq6{ zb@J}p-}^sDp>Xv!D#w}8DA;*})Hx=E;e$0k&m>$p-eOB`x9dvllDNF!OAX^()^TU0 zTfUWlXv42ZZpz~5mqc&V-tF{beaJ@vW82Q#L0kD=*8eyCr;Iyl4Zdqbjo!tIt1ru9 zn+o+QvW1U$8_w=tD;$>b8S`%lcx;~BJm4)o;C|H#kP_?Y6i2pEV7& z9{r*^{i}Ps!>oN)qbKwb;^8PHCmDdqxZtM|sHW zHnYJ}v*CVM#S3Y{8XR1nV6rE;->Yz0B5s8Y_jR|9p1gQBXR?qX_-M#`qA%iebd1nG zg%f}B`s%*TeJG6nR(V42$OF>6_-h?zCYej^W-C2YfHqUHvwu?NUj8KqjEuZpbN!>_ z_IDwP8`Goi@5bu0nc`24FFpvgWZm!|#%mG{&+*%W5veOO&MdBYib*$r0~hEZ96l)L zZ^XG}m}|t@6&zh7c_Jw>GBLk9(1WWjCok|teoRG!SnxH|bGDf~cY;#jrN@psg!~D~ z;w0+=j>GEITL*@^j$s!TFO%&SITIwZq8ByrVB$Ot6p!^yacDP+kH;N)7|~pk@B(ln zW+tcn_LG31oJZTu1GwSor|2cK6zrJhp@$uM1E=E4JOJGBy*A=p{DS;p=0H!oBaV`J z&p~nvH{K8?kj(bS`5fe3-R{NA0N#dw0Cma#^SIytgV3U;Q-dZD#t0<3bszY{&TBpK z937cUQ&l$9K)K0G`65Uq--od`N&wwX?5Kfy{slDn?k{{ud3n5VGcr2*6lpqzj%Xn9 zvHAJ=OIoNGl|Y?Nr@d#_Cm0B{|J#C;pu)#_gEKH>u8pPf*as*dXk-Fnh8nonZIcIT3T9h=Ntt9z%7+io*_W>AXqju5`iwf`DAY@0_GqA zmwe7QMj=85q`L+F${i+~ffl2^?8T4kAf;?P{=)*TU)K??nf=xOVEKB=uY)?r5#=Ye zb4fpW(gC{KRJFY6nHis~M{&?mNdVKWq~+u!6cwWo=(zv!$GSRFXeL4=BhbpYG2D21 z16Az0V+ImhN{|o0WzNYp2LuQ1fe-H7n|+|_jzm>{G- z0_4%H2yy4iLzfO5eV)S-()XG(yS%c3_fPaUFg$DoT7BgHU>L~8SH>a=R@UneRW%U%rgaazL^MgdM}>ER1Qggs1=1qJtkJH;m;KnN}9JP}P(_yr364(PM^ zH8&Rs{2vglfRTrgF$14)y*CJ)!D=(Y(D|1_!;_Bb$KvAah{PO*(`~?l8v)sTlhwd` znnd+n3Ghfl!Xzm8E)X&3Po}7;77c-6zBZij^~;wg*z7RSS9ZVWC4d$=rpbgm1iakvl!-L)7-R03lPjZwic(~<(D2GE>J0je16%H?E`A-;eAz7^ry z-39`(d>H1=O*p?9{-KEOF2qMW+?)p1RFf`Iykr=H2ZKm>AV85P&p`kMU?lb zcP1g~{VXu%4nH1qgR#lfI<9Ii{!IE-!W?Ozw{;Vq{PpP0I) zDA7OjhLIrO9vTC~VDX`0_ysf;C2*LABSVg3W3xe$3Hu=jY9!!>|5*-@bT@T90n$)ZKo=f!dzs~ z1=PU9@rFyNyOq%XVO@|5M5{SW7?qWm9|Ex^i~yp>CKu2E@eGNOdll%PPRN*11G3`@ zmryxN8ZeJOCmKFC0S)Q0Po4}v5kAR;-dHoBh)IM!>tL#DHjuS2``QeMwNsoMC^G>y z2{cqMEe%ZDRm}z)kd>DH1~Y7-g{;73>#xikppjC6x^=cj>+bE{waY{)*dh1$K2I7Z z1%HI@TEEI(6?yq^qWfBzuz<*TJY@W>oLnd}n+3Sd1AZ0MIxp-Y3R#GK5g06uOxq?P zh)GP$gV7=&VvvO)2ns;z)$QbRS&aNzQ$R}{MZV;hF>+f%Ffw_jfTJ9m0FbGX$Ogb1 zFHjUYOuEe7qofpFSXf|h--i{?Aw<6B{Vf^T>8X(wN9&|q7=X-p|Ndk756IZ@#o?k9 zn5vu#s{X9(?2lDdBU95CMBFi-L{95l!IF2L3U5U;r!lX99Kq><$(S~B}c~!7!OM? zlfY`|11wt|=SQI1Pe|{1*Z4@lWbi>>u%^@@=x$pdiU)TD>faE=c%W-f8hGgcvOt?z zNwC1wSV}xhl*kS>a<~Y<0{_66^mp%aU{pbYMu`}B^#nGffL`Gm@2)VISapciIU}9i6 zFUdh`>%ISD??x-Fi~f(jTSr4SeBc^#Oy;irzwcex?GF&o&dp4#Z2iB@?BjLh?##$* z8_{7fzdjtL`tocp>&bl3cEf04Y%)F-2p}Zv#zD{=2xEX*Ve;r_I4XBPKpPb!E$u_* zxJPSsTj8WQfiPwv6B!~1VmS&lOq6#@2|NHw1{P(KPZ=60`oWEr0W}|XM_3Qq7x#v8 z;2~1LExjblM~>Yt1+41ld!rSWdGKTfTII6PuL7@EOQw0z?04mSgL04&%PG{D6j8_r zDyWbTXaXq;E3`6Rjm!p9f)T~!uJe@TFMdeI>DUH?YA+jRY#j2d{CY12KFE7C0yYBm z3|g+Jq5mBjeb3FU4l-4?a_L%gQ zV+Al}1QzJNMzKhQz-ACKnE8BUbS{YFzJ6lz3cNNTOUQT$mAd*g3%)Rh@cQ*%1suR! zXF=Kl-&}vPJBAn`vXP0_4T!BUFf;EuRf&XkyoN35#YT2AM~o}+F?7}?@Y-d2?oVHW z!R|2L+XnGxK+)%cr*Sq{xj{$>CZF0)0cZ1bbs+M5gY(0D%E(OU2;tM5d&=3*p-~jE zLlSPQ{C{3=5yXWzT+e_OR|9wbaVF>0CMbRsVc>D~5wg0!#wC&MyA%BSwGzx@hp{i^ z&$@^Nq1h9LJPZT-KA5k`0bN!v-~fN^>AC;o2pJ0QH0ko8#$hQN>82Va6@1AT(UAd8 zmw}5*4Wr-jEy$%-L&Eo}Zh zGq|#MAX(6@43uWzK3>2JHOziwhcQ?G#s^RQSonCoMdk#gE)0o1bg-Kc0#6S5ywu#? z>mboOE~DpBht>vpIOM|}cf6P)I;a<(H=RMU=Q?)&2w5eeY}x5aN%9|Ppj%&6Qv_bF zkpq)M?s9UfB7=#c`4glN%nl3RAA-XLU1LAA|H?DQuD;}ffxX(uq-tVa`=}HuDcx9b zAn@Lf&e@Ik+Ye;AKEa%9Cpfb5U|HbKKk7F&fa!swao8DDXXN1-iHJf+dBb4;=u8kT z=Hb7feWL`1e-T<;4nZje6PelTVM5PnwcQ7%E{2T=RP6*axC2mPnSmL31`#z1)#me8LfN!F83!XL1w>GH9tPIXLGmH!y8R98RlY{Ih{mci@y0d^mD-K?bA>L* zj29u~KQq(T(aBHoI@JW7#&_tR)#`~#W45WCNZ@n$7y!o@=A0OeB0_?s*16vM_Q%tvg=FB}`qLr|&y&D$npf5ABDUIE5g1I=t_B_@j|-=hDAm zi!zIcGvk)>bk|&Q3%b8SeJJ+^`a+olTPOot(x4$;<~uanKhunbH>g0A$^sMKzwbF< z(SEZ)r&E;<3z%9$Sfp|7fgh;i6qPGvHpPE@H!C(k4S%g#38hPA|al z!wctca{;ddZz5T&0@w7+*m$4+8Y-t8)-T5nMonkI+9UshQt0SO7Sx6bAG=OBQ3~c! zkcpgMsPF&x-+rEufmQo(LI!L+4!1X)E~6Ck;Cc#kRXWUGfpGm^G^oTj z&;Q-#i2VE;QSRZ0t3rOxfBTFp@KrvE*|e|1jF6Bb@!6ODKcXIr?%BJScBWh9u@+y3 zO!EKg;$cjs(B$U)bnp2$yV2F$S(f&iV#VIe@F@u<<_O#&4IC}fSjyJ98~E%C z-!8$oLp2WtjiJ--kWKwZ3T}jpc~Y)-Gpm)0o+UC^Tt(%`zzT^xR<2TgyQY`^>TEP! z{4_i(;oY+OHu0Rw{S$awAF->-s8rC*j-A66+~ZI_uv0n{oXWNw^DT%$}Cd=v@Ut>zf%@ zH`UEgA1(bU{8WMRPVkwUL@kp3dtZ>*D|)0pz7^0eu$8NKa#~#X>`6D}{p49Cxn}&f@waIa zuQQvwpX}87(Mp}MP|CE#{dFCytvq?M@%D|q(bvhhXY0(9X&tNj>5Rz}nY#279ZD6> zPu!Ks3%lf^Q1We-()rizt52_C-aX1)-u zbE!Rp%$m7hDzJrN;_bnqIN8G5-#;_TS*G%$ z>iKgueGM=JxrVJTh7RVdj;Z{%EY&PE@(FJz;*;#!%sT!p%rLHf)KOUBlrJ=?hv{mr zL-12R2irb-lvgUkSDaX0GjAeAFwcff(M+c(Il;b0tg`ViY4gd8waJ4Lld~7zz1FVo zQw>7)0g2IHKUteJ3N_4CVwF_(zoQ*n>T#pdpwM)U^H z`|DeY+yP&R1T?lBN&EkD-x*pbw-c6n|pBlHK?-~PWJj0j1;Neox-lGn<| za8G_BD_u+={?iakdMeuWAzZQhpl6VRM|y`keZX#H;;=97=;N7VBMCdvpzSFU)9?=V z^5V-bfgUoCS0sVugQ-$K%1os`tu9ZdN^W$9kb`yl_wbRtj>S_t+QbEYGBzut&<)ZDA*WIH zT8Xp5kGffh<(2I%mfEx%*|^86TU#5;I|x&5s`Wc=&3lCO{ITOs(0Ot&d$QwU?fk5u zdg&GIht{JC&Ei4bX==6YHs9c`(*+{bC$LGGot7PD`kUMAho|H3f+l1vcC(XQ!nW#ZJs5&3Y+b*imz#F!no z@5piNITSdJHax^3vvO6ppMrFR;)j68!>TKn?>s*(IUU&@*Y3%q){gg%Ec`nev6sj} zx}t;}p*4-@IUdrGgRuR8b}W+Tr@J#4cF8S#xlg_gvLxsx+lFg5bn*B;3LpMd$;RE# zu(dtwW+7}@V8ymZ{QhtzS2mIO;op!;wKv^F-0V6QkKgJuoDOx*FR)hpHF~_+^txTU z!bnYbD_v*!O?sHgDd*#`g3PV7re}Zq`?h5HM`Bbs+S9dqr;eYQ$mR45Sd&`$6zI=s zl-;s-VI){LzvD*C;j&WFba!w3c+haG9n`2dM#coHbIYbg;h1pl3t;@xsem>dr37CsiQgULjmDaD{p17^Dj z=mQr22sE8@O;J))-!ksjH~kP(zMbjNyd8}x)}Mlxwohy7KhDSX?S`CX?>v-juPV1(y@6HM z?4WGh5Xxp+h_1X{Fyvt;QSmovu)0LGxz@GpzRhcsBL4huq-h%I^G zTd@O0mL56P50~_Z#aAV3`+jmtE~AHw&&yvH4NKKZF4T_;Jgaw1(sK43m^0t&R57t)P-qNZz3bNR zT5yuxdhrI5DO^D^g=fw)H%9s8Ryh84J1DdmflifpUrK^pf{Heq(YzNG9y1a&E zCN6~9+TkWUx@nd(OcoaWB>$OEsj~IP( zpM>6+`lZjqCCb$Ez~B!Vly#$mNtfcYA(Oh~WG*@SzaC!+trhaKg@xtNa=m`MnbtLpw;T4@lG_f=Ht?Uh@;z5^Z=9_>QGGDUPF}j1 zGrE{kfDMemGozJT@zYgcM3Nx<1l{JseDF zjBRGFm1V=*A>@6Y%4|O}P}Mj1`i0h$f{WYgWi1)H{C0nG2X?qcdbGLJy}u0aT2}KN z4cvR8jS_kMxl{&~%5H%@a5i%pGj2O!uDD{=w##{UF^fKSk#=D|42xZAO`Y{kzR1d` zb!I1)r%&Ej=Xxei^B?<}xa?~62Ama-vWqniB(ZiBs~C9lsja9Pc{D_6nWIMw?~lFC zJS5KBdr1F5Nz83qsqExg!NHF3_}KPIb_;u({zCsMrK`%DP(Y;E-mpo2A~AQX&>lQn z>d)pP*3T!RryoLcO=@FQh*3Z%>v|4}ATguO2ha6FsMs*L`!0-)!XF5xYN*!Lwi{Q<@c#JI|INU=lAQ3ZVWsE3 zho5LPcf?euJ`JRb&X;%%s|$ABJ(7MUM_bg`L@D|PDl{P_GqPeNv)#YLm<4;0Z@_8p zoJ|}y-YzY<>D=9a$LKd(2for9nuym7o2rGhntU4`o)ei;ZKiI&zk${D8dfIR9u1XX z%Ql=)3+E(@U0{kocK@U9*wTK#@@T%vsdhrjOMu)(K+5`n_oKAG>BkD~EGEp0n8}Wz z_aAjwiEf_887H0I8-LHYeq*B2C(L!Neoe)IW^ZUv_F&N)hReik@Iin{;k zs5FOZuVxexd!~+2<^6h6#NKl?kqZ2o!AZTW%l(IL zp(>Fj8bxQu%9Tq+>gsEAr?Rcu*+y#Qs_sJ4lUHMlHs@*c?incx&J{i5-}T~3w0Ogm zxU+Qi6HT>XvaG(>R>bc|p9qy$G6zbS;Lv)XAs8G(%O_(-$`=$ep9a;(f>1vlxrcw} zvekUd@4CYDh)bs3=r|<$@7KJUnH)%1QS0LG=Z_!YmRMJ?J|rDY1+V zGPfdpd+4B+BdqP?`M-)guc#)sERN^u6&w_~(v%XoDj-~yro0wEBB)N26+ z6i`AD44{a(D54^X6zLKOB=jO(RH7g~Boc&(NDG}L1Va+$3p#7onzvc=F#Bn*^Kj1j zzK8F8`}f~_9~Gd3to7pIMZ`3YcgNxhX%UExoIH^WnIrgNsYe6fYT!mQeaLPgy0>S9 zCr0>aQ;MnjY*^`FXMX-=Z7ZzpFLjWE?$sz$VZ+&?yN=wxBvg(JR$V?|=YT z#qr9hMwMK2X_DUhTyJqF**=|L+qF>zm$*(e&>`GRqa3gG(T2#p839g(E@BoKHdS|T zxY4#6(dwZ0bnk!%*vp54jf&NqH4v4(Lnc`+IZyX&3U@8(?_bWz`1A&5(H^J}HL z$co6=>GAbkZ26YuF3b2@!O*V8sciKwBBk|q&5hNbgdA2Z^z(XEc1}>Nk9Jnk_+~0P zm~Z;*g5&&798_LTd6IBNA3jv{dN5Cl{gvfQkEn22;(M_%Oy%U3C0|TfQCX3WIT&Vu zt=;nI@mhJgo~853o!-3CIp{)rOFnjZd1p&qP@X5|h?|*Ua_V~YSAm;cQoZwTiaJUX z;kY=w7D=90RSlpDI}T6X*gCJrh{_alQqOXDe7l|2F2)hzAI`+(*hBdM_i3{B_Q6gDc_%mIRHi;DVG3!1_3Z)g>az-G;R*U!SoHHN~DJ6 zO>V=eAGs_~)wfg^T&Tp;eT)5BM;ZO?Fr22wcGRj8IDd_Fs7><8 zz8Pj*w1)qR8ZOFozOC!ph#l^Gm32t1f3DC%*D+`S$NN_OM!h+}F*8$T$g7FwINmse zwn!<6>vd!EO)>n@i%c0!D{5tm12$uCG@>tMu^6M;JD4npE||`{3;ZhNuOH}~m8$Q3 zI^XR9fpSi?PG0iRuPCXR*BQHQ6&&}gOOe_lq%-Ns7^g6I*+PMWz28RMeu;f$QF2Mb;O;#nN^nVmMgY4$gFO3Q8(BN67o*=GqUe`o&cqF*wBUMtWMe()fm!<2hyuFH^tj8*^Am-UbEi|p2MbXrR=RLvs zeNIe=BDSQ*j0I=X(5NhO@X1Nv5il~;?XD>~e}SO&!1`lPejxh;UR<`{H&?7q&wCRH zBP;>Ey}$&++qmKef*<%7r{XM)jWRAOd&gaC4~f;+YfTgjyI%b8G014gPT;2B`#h;x z5;^eIxiPzz;J3FleOy$>=qJpdYjN3NV0SOLdjYARvpPXCBCgp}Re9sB|1BXt`iAV+wb6Ew^I0-TS-X&m8 zU=~m@^sQB|h>Pz27{jBUhI-@9tZa8AwzByBK~&`fi);!kaS7>8bCwACBsW--NXlRn z#v={v_Fpx^m|s#ETEhhl$$^!$X3J(SC``w3Z8zt{uENq|#nFWW!wD&gr*05!EGLO_ z3op>D!ISE?Dnt`-CLt=<;?Tq*<;apA-_DZ9W0fNdL6*saYBALCy-ZgC zEXZ5zn(?{mlQ1s)fWSQ|V2>jHxG&-bfDQ|!CTM*3xeS7fUzo1(EAgH|L)F0o&~8AV z`&4d74y8VPW@b%qfWPVJqL!|twh_8xuF8fpqB%Vb-J*wwna>k?_B=CNMP<+n@I^Cw zhqV|DXiOqKJH50=qIP;|kN#htFyrb8kFmk^>kK>&4=_9kQ1<8xWU=5)@8gvlTz7)} z8vgDP+;^IDs=<8g&J_l(=)ADv*q2Dm~E)%niPCk5S3gy$6nX%a=Ukm2u20+w5IAx2-TdbA=D7UlvEQ6FT z6qC?QwQTb1iSWu>P<`$OwIqc(#vO~&?|!%cqu9PaFZ z_=6I#S6gl`bhH`d2a9hsX|_A7LSl*@<37N7bE$iLP0!W$feS|k;By|*ClOX>37q30 z0R25hUKJeQ=D^z^I0j0dx^+xTtDA8V;M-7Lps|92f`W)H`{@8u2Qv#OclYhd(ErJs z0g_Dxs@(-(F%ZR7q?w8AOYnd1g+>EN9eD%5P9mYG?F}$Zh{bKmr=`3i=T&HNr%F^yM$^5T-{{)(I+4=wg literal 40798 zcmd431yq&aw>FAMNJvPxfPhGMg9r*pNq3iYcPU6o2}qaHNP~2<>6Gs7?vDMf?eG7; z_l`5ZbI!P9+&j+qI)u%;H!J3P=3H|<&wL5_ASdw@l^7Kc4(_Rxq_`p+9D+I=+@ty@ zNZ`)pE+sSghGZx$Ar1$H{rl0F8v_UT98OAHRM{crmqK#>4;1>%0*J z_$NOZM(`9Kz4d$ksCC|(X-ciEsKB!1WuE$k4&xKam+-x)e#-PCQNKQY`|vm`N;t9h zkmB37*YHxblqiCWLuT!@j+=vgm;H17-Ww^7Pe(ar#s76>jWOcsz+wM$Mg5X{6dL@m zD;c-+-!E&V{%3C&9tgV6)z}y6Hr5sPQvK^vq^xt=Zd;8M(cRgs$14SICUIF#{Hb*` zbLFP~_h$?y3N=Dm(U;WK{TwZ= zk`=VEh4o|Vs|9nAOewR9Q$z^B?6DuP~ZMuZhUs}T5Z$4o29hJ z=elzdzS1*i4*2bh9cA4`J0bZwk?6sqhc0TSy``^4JB^vwoK<|jdaO|bZRfMG!NJ|J z?-$|07>jwCSDHN1>`6wCLrEdu!zCp9-mf0b$ZwZ9^V?e!qpPsLzBH+mS}axm5U+ni+RjN87vI+}Q4h<8*1&!T+c z$~YqCJxACaxla|fYztKxR539H1sT;DVNk^mI(-rp!OJ4r$~0y@8Ig3|)JWLMg>s6( zAr$L`V@F1Lp^tG@^;7IiQA@@BjPoo1IwAxY&j5O;)zGntNLl^Cz1Q(D*+=7Vz~#1f zKY_2f15PVWN%~2=se_|D@o4e8XQk<*q{afwHL>i5`j@}cIncXWSa~@R=xwnHH!&yp zv=-(NvkSE*C>XVCLK$&V%A^yU`2whNhVWph4~98@6hnA6_tTx3=hlx4eEXP>KI{Zm z;u7u~6-kV~TTs74J~>C3OnknYlFSfbYjY~N$;GIS{+OdHjvh2+c?Zp1e zgs`mGxAXlk;)DglHdU{%&$o@q&F_#8!)wwL+L*mxz$}RSFGErXr}5~8Ryz_tw_$?-?<@0_s>*i`Mk;g7t zA#Zhjkc9Iazo(?TX6)`pf{F33d|CarhpTaP2g$s6H#AIbMWo`WQ^hPrRv+k^X%>p= zYJ-zqScX@9H-E9|9UU(Qp5CkxY)1>GdS)(~X-&9igAn+CStKH$j#_Wy(AryamN#lS zDwnRdXnGl{zHsAGT=|&LEO3QTnc@tCBGN);B$|h4c;BM!Zg0x#*XR!uH5<<*Swn8J zE_)jmJS~Bn3)FyBk)w~dtPMiw=#ck~L2=Q^YFWMlgtm|tR&?}|M_#E`(f6l>J8mf& zriX0;p6~gqLj!Iybyt3>roAZf2@+)zJ1W0b%u3 z-!U?6oOx=CF4!nIH0W%YKk#EntzxHOlW?LDarv|El=m+skrjR$-z#rZ7Pmy3P2D$L zPYjo82p5QlV>^E~O)EJe%;(F}crk~`0T{6qqRM>+gQZ{8M4wBF7 zFv3^o&Tm8kO%chLEfK~>RRi8D{{Ho1DlCjj(wg7mA3a?0HHgqI8bbV&9@8BeyOy9b zB*Jzym8^fY_&kHwYY5rMCPxSI!AsEliTPN702CWiPVcIuw}|z(L2)5tW}v|a=0No4Pzm+|cG>(j{oj`>w0~j$f7dxI zJp_%nBRVK~$$k8uQKoi{MVEbj0-rg8(IX%ajgd^6h2x70lbFWT`HYHW1cs`ty@jM6xk~|!XkSCRgK9!2C9aB-7 z_Tk>sKK*IMhJVuTHn=DRO#N{^icoZ309A}cie8{2L0YV z$MYoq3CDI?{JDqA`6$ih4ds`wf4%Qrt}syE9rC9pv0DnAyyib>(Iifi?Ha^ zngf>f3O*Ob<3u6%L0 zDk+N}q$;;^U4J|BQ^Oj*aqmji_O)@dr@N~G)8`^lJwK&gz19e+;EZOW(peSTLg**&QzNcD}z~g zvvt~_c;=kV%nEk{$BRUVdfui(`3E6)B42N3W^X9DSh`u$W}5s2hqamq<>&CwSiPZ@ zn`V~vdSkkvYU|bM>P`F6sY613117dxdD}^%OOoM8S+n*#@o3DuR?F4q+(jIYn(K9% zFs*cl!P}n+Cf$79p+>nlT;41k%~urnX-QqwBII;5*Q~3H$KALaSRBdo zmeog*mp)4qWo&0&4j2&H!u90&ZCs&_!MDOBHl%7rOtv|CVlmciqthNyd_Mnxmkj1V zzg|&)vni2VKesV&2U$3#OA23+eFxk5!mtCc-jitbnD ziKCMzD+4^omr@K~Dnf7dLNOn-)()j>j0wM^>)A zuj2(BrClGmcVEW8;&>P|3yeT(QODzcmCU9lw#_PT9Q1dfjMj@uXTd|7 zpSFBj3E?zO-Sz@K-PQ{N3KzW-zB3|kqaaV6t(pnCORGKbGc*N1Z+o%aB~fF4#L&Os z^es!N>)NOzKFoVtKrW8G_NA)Uckw}T;n1DlrTEOXke_%hlNP z?d(|_Hok|u7%16kre3ujJbFvN;chh7-C}6P)^$50Qlu6{9tFRDj4@j1m!naZNM{c( z{=NHP{i5^Nbe7tquq5Wr^K8v;Sk-|FtBwFmD z=-(7wY}U%4HfGnYPKp)fyfv;OI0U3rG>bU>UqsxuM`imL<2&htV!q=X_otPAWjV$4 zq&FaX#$&ynIY6J%D`U3W_EZo$?qPZ`^qhmie=*M5bmJ$7WpZL+N>c)I&Vk|fgK~7a zHg7pc$ZUPOyzqS@Pqi_=y;n`Q86AZ( zbGXTs|NRP)z+O*is3v^->TQHUZzpw@Nw&S)e(u?naQrx;j~vhc$Hz5YD@?v=j(Y*^ z$o|d9cE<8&F6{RUPxd0nWRk0n`&o{dFXTPaP_#Huw$2FInmb~moWItnPz+mlVHVO` z&;M|;-ka*_EFw~GOBycJG&rvq*(f&nUDeKJxpM;F7?>iEbGmOzP>K9@boOvhxM19# zaLBhT!5TTHyT6B$Vg1SMrjV4xkbnQx;a%HfoK0QY#E`polPh;}HaCh2;qrQW_wExyqvb&#( z#)T_A(&Dq)j^=`g4Lfq*{@tW8^zirvLf~*&+@qQ+$L;r>%#zvGW8q%Vo1zByM&xFz z_wD9MHLM>;nuNo{oZ_s*Sg`EsB09^izDad+ottka){psr^EfBuYCzU&<_xAed}`36 z^*Hg)ci^Lqtl9PqH6naW1&OrDLx#~9B=I>*NC(* zeI>A3&iZ-GTFcGT8TFmIH@W){ip6G*FDAB_YoVT+e>w$H5glF2ib~Tc3GS_yI_<{? z$Fal9_^T=mb~{!!qYGh~MgT0|dNuf;A6QvBE(T#SYhz_COY5lNYGaW7zdT->G3H*G z_mW0<{9evCk8!&SH2vbCRdzG6mL$q@ybQ4}B~|Z^oMn++^!U$?*Y-EBo}0o;6BN>n zW+GzBdd0Ldi5-{UD&(FvdYv46u&B|fepYQiNNE^ZQ~G6`sxY|ujC>%7tsyg(dQsNE z-eLh~T5w;hshk=l4hWjZotxL9rP<7mZl zR{K&L2<0=zR`lk2Y6Uf9EMtzNQqSS5q&UF^rYF*E3KQf^Sq>dnRM8tY(V=f8S2PI?1Uc4DN@^STDTB5EVYwduTbYn1ptPRKN*M9L4^K~of) zaEmTZ<|;?H9W3S4*b^-3vGpxPT?azj34F=!`9(Y|j}mBUX>(9=%>5L?#UxU`L(|#O zoM$UDO`Gn`5hh>sVLPRohK=vNMALiCK2cGACO@!x7`F?o#cyV@Jh; z&?UhfW|@r%(a+!R8I*Dl15JnQ)VeCH*QOrG?@N56O+wzLAZ_rXrk}RWB+Ldmw^j#J zeb;82$q;XqusP zXe=cBWKg|g@=5;lEiD}xvy2*lSyyU8u{dp`kiS&Rw*5O=TK%L?2Agi>D9(fKVz`?X z_BlS;e8nE_p769p+e8GZ&ABgjzXnT=f0%Ca0XvYn@qKGsKz%oRD^dOT>tv<(_W_OC zoBH(3qamZ_e&#Dq{PK(1%-Y4HLLcQEzj;eB7yLsDy=Vcn&}cFpj222SOJ%Y>@Kc;9 zdSSh{?Oz@AIWbYnG>SnZkK<*MZc|8qI(s?-PPwYQ(MCG{%$zg99O_iy7$ksw)WO{S>+tt8Qhr855(#aT6g7+Ed zEEWjb4KKJ0)*Gt2b@b!P(cc}Ib+eYPHT#-U)XLzn+9%N_EIB0R+UA7xy;$^Y3CJuJ zELBer=Xh_gJ<#8cfZpKUz;cuJs;3SBohuKp)siY^v}I@027=XXk$LhWSrf><>y6l# zqXy^3fn&wue06_GF#TC1RcPG6Ck!)B$t)Xn>CeN$6CRw_YvCq(EKPTg+{5Fc?l%Ty z`fTX^q;zsI(ykL$D7ijl3|6!4bz~SwzlCN)X^I$kR7be0CVuKNp%ohi@oo5-zIAG1 zwLpJU~d!Z~Wwo4mJ zQM~pk8hKo=*Upw$s>gWQ+Me(1E5?Rdo_GZ$#e!ItG>=lJT#$1`__et9i1`YO90J=91(N-igY6uCL>*&+d)9_ z8B-7U*X!w*^Y-IC!;mgJV%@P~tyQXK!?=H+yi?UeCUJ+MCIPWBUeG& zQXi zs0P|&8{%4Ell{?1%aKaUbCtg;Mu(tIJ5nJ$IrA1L`hr7=R(>OyPvU3~Usl=w&dBZ# z<>|TKSo-x)7ela6tUt%;i+X}^v#~#yQ!3fMp4FpGwzhAij4!5|j~=4EUziC<3Y;`*32?-a(5)^idwefd?ZXv@jF4p0yh{ewfJC9>Yu7@Hx zM;yzq9U-*B(@ulqKIj5E{9kYq$cE&1F-woCW%j~}|5UWNk&%e{kblD)x@;!hRF4bk zi;~?tSad({TjcvNLlrMx8{7IWsM`*{0+Zu7w1{!a^J}{mft&=lL}Z$NM{iNX(KL4z zF84z|aFr2qwTh`mZ%n?bCsbNGdt2(!&VcUu^#SnmiUI#E!5C|5*Zg0Qq>$;K&ydEE z8zEhp#4*3uWZyZ}C$WkMaHsEZew>b-C3lNSknOTh(c<~vQluL?2DFFmQ>?5FNJq}) zQWrx;d8_uOiH&H7d-P7GU(qIt9`fSd@?zecf0o<7=#w{hR`OjD18I7hXn z%nrA0?jSi~I+ZW`YN;n#WP1nkZ_40WXNzs#j8XaPnL_)?jB-c!L#F4>k4>v4EyYVh zb<6S@zv~gfmzYCvQtu@1jp$h96Xc?&Sp{hJ=T?RP9+_ zT)q0Ihf6o&7rx(z;+dBMB7_43ds^e1wd|HeAqu|PktAeTf^8)1HTnMO_y;tDh=ggs z52h1`9=N~(0mE=(Rj0=HDxA?hpKSjmm$B-Djn!&s;`;O1dVFtc!68eY#e!MZgM$ri zwOJBd%*XQ0aQx?6ZJr91(!4}=+p$_i=(cs3`w+Terma!lPgu5?RxScumVtkYJhgP| zoAIns$sQb+hXMt<##5|MuNTvin2<1kp1)PB{wL!PGDzdO)jfXu_U$7q92_N6#An#G zDOVBPPE@SaG$<0M%Lxy z|5j4nX{RQ8IfA-*+$bpI!_h@uVlua1`Q97FlD_!l%F4K6CnU}Znk9_m!npL{L2&c5 z;>B(p*}V|xu(X_4?iWn&)qc~{?{&sap3Ixh6r^O|-F{g0?^=MhEJvqPr%vGqK3~OT zU2iJf;_r#CZG27)sN}u7koBM%?jCG9NZPjq$D=j^I-N~zSUeCsxX3^68m@ec@+z(I ze6Q`OE2`-n80Yi38F);f)2uifC(Dz!`8+oo?{r_V(*NDbE4MrgVdMDUQ~6Zs^fYGS zZr^wwF7#|XE}8Hb@7Z{+^zQf^?BhD+OJD7Vu+Qh%Hs&;3C^M;+-1r~*YC;Gk6s3BR zT$;NBqW&R|{vltVp;B42U1p7ubP+(c zx(3C}uf}Qv7hHvUq=)9@yi-Nv*nBf=zauQ5L6%jNAsr_yD9nr6C1$QE(_zV{<-F|V4wec z{ZHREF4}?zYjJ=0 zJ1;N6%LPvsa&mHY#$17!q#TqMe@x*gpZeC@ww>5r59UxAa=`+2*Zv*2WVZdNEDtso z*6a80QRQ+gy3gmEC7;uZn?x)Kqr;|J&o0#f5~yQG2J2NC z1{@$nyn+Nir4|sV@9rYQKz5yXo}^)ng8h02y*}(?@3Xb4ZN3Bq;9a|Y{P=QG&-ea6 zyx(D1hDAoEvxW7%etvM?bupFa5zse+B~5^RM5uT1tM5zJ9u2!2jRqmkSbnq*41Z*x*|60x3(y%aeb`B0U*_f=X zEUNhI+}xPhSd}gI_`bIe4h{?473JlU^72FV=`k^oe5G8}1{ z!=o1BkdkUI`MiD&2e9FO>Fwopbap0qSSctZbbNeFU)tZ-hh}Sg_>2h$OtbfI@s=d$ zQBF}&QBq2Z5?KHHcU&;;p6>2$G}LD}IQG1xSvffyh*V%?Yin!YzJC{Yap4BTa~A~@ zVaEva|Ni~EVi0aEvr{>Sex>H||I@78&pg`K}Bsdtyn%hOrH#VczAFh65wHd z2IjZ&*Im@Y*4B)pqoXBdWoek0m{^|IPTl|h&8euQgf_>rgCNwrw6xUloSMh(=<;$% zpy&MjoQ9E+@v!XECpb`G0_+TjfP`dNrjB~%_0np3eWCBCT8S>4a*1xEpS0q|*V)~? zQWGgDDPB?xxm5mG0^(GD=dUR#q!)*azO8nLh7EvmIX@pj({_L)-(B9{-935o;{t;GTp8(Ue+eOufn)Hl9QWy__pvGEuT?D_Lg zEiEk#F+E^#J0LbJJ=8{SUXb%v^L!frB~oUx?trf9aD8>PhG;VWOQqhpBzcyDj;BUS zPL38Vtx~RBR7y$;!K;ntbQ1o~jt;z&^ngD@#*g@w~V6vL*VNGFI(AxTIP;QIw|_ZcVq&z}#P)i$ZF z&3SsYr}C1osPMWV5bBbk_vYEz*?7}9`)fTh?p@y+Juk~$_GcPadukia_FQh=yJF_x zXc-y9W-2X}GT%LIgFsNw&=`)rL_stN_=_~COuSSNC+6xc2V9FoBCLA{#J=wCB@m(Z zW~(E;Z!hexj+Ps4!B|Y85BG@7>$}_AQk53tPcSfO1qD-pL5D#pq{aBJw{C8{;64>E zZvybv_??kjDyr-#fxQx+`69FtaOt3NW@SululKf&s8h|pBD2-DW}Abld(#y`Fl3nxkpuhL@u$LpMfL9tCmF4=H$uW7?FRL#ag&e#b|OJEE{&_h!rba*nvmz!e)VF4kS8~AL1a03T?z!E(ghrKDbD(l%!@PDewTcG*=f&{?wISmcsCjjKlfR|u2%ttiZ zLLl*dRWcXSDB!%~v|t3{JYw6akmp6I_3kG ziltq^!_F6nt+Fy)?r6`&{;Wp53lq#RT@hqfU}v=@^#x805Rm~u8AiQLJG!-o%u z@!I0zy_#T;0_GNf_b%;H-O!Kob)bRu3ZT^9*$Fm-j}wJN0JMf%e||b0%q0TD==wCXZ;)=R`xY*3*6k-e zyf%8n;#R2`6I1Vc@^I;?;rh~_GdF~$kS|?$KFJiHr!R^tGRolbb=7>(-tG1yi~9OT zhEDo8LDC-C(;HUXzOFg-M2WOmEs}cP@+~fMpA8P58IR!aRs`e&=?3Q$lOIcL+=>wy+1*+qXDGmqY^EYSPG1ZK8bV1g$Rb{3F#1q9jeXfh%T_m;*pUsoT zO7uin>}nrwc0{VOA<}hyRx9+GwL6&bTDd^e`#-TZz#L)z2DW_ zo~z{m&hmnlwKOACGe3bI+0Ao=Th%lHPv`rJRB|U7B_zQY5>J{a5Z!0rqB?ki>3*{) zv3)bBbe7-0{GN|J(rdt5frXW)M`ieJ6ua4GQc#-FpLT=0)5>i0yB~B-csg0^#+8~N z2YYHg^Lx4J_TcezK0dw@4G|HatP;u zqHjU?*RSx;fkM)c`hrRkFCnJ~w#po47>*nL1|X`!R=UyC6<9{>aG??GxGk)|4oWiP z-*o1vpkJLlU!Jc!mV>gsv5xif)j4<+n%F<5Fi^AFMUrrQtA^h2=CoyOTP%;uiCazR z?lIJ}5wo)KoB_H{lRPe8l{HKE^h&=ga!&2C*FtZUbGpi!B$R-~bU5QJqgGYe*qE|r zrA5c5M~?}4?Vf|=esYloyLRRK+a`OH`?uKG_M;1pp6GJHR6nrsFO>RDQ^O4{0)-lzJ;WnZOh0~l!P7~R?q@}Z4J9*Dxc73=SN_c#_)vVN42ei` znKZ$~2r_|@-{wlX9@{j)m(_+lfO{*R6^V*x5dtS{m*trkkD-;lIvbTI0`nl@u`vcg zy#e}gd$u!ta_uMD!tg8=QJ z`@G(4Z?R%zfA>nz@OT*^QfFea*oM-Cc1~E{Rolgnr|8@vSEaP+^6tQ3&&;v^Ld3jXxOc_&v7x z^LzFsZJ|_)V8z1E*ci8mgC?hss>!ce1e(T4u%-`n7jy`vU7DCStlT1=B!|Zh*)Fx0 zP_Wfmiv7C2`p_sPPkv`gkeHpX^@tf`=swP5yY;NLl3`&JM3eu!un1avyRiE;Ucx!B z^F>^Z4b8k+hxKaoUd8AxF=1f94Cb9oRbso^uUh&L2Z|$AshJGp!ydypN=t6nYpvRy2l=a`!3|}!LW#q8)o*rg}Avs9nvX#d40<0_!nLcq^W4{{oln0apzC0ow@`18N*2|3c zhp}Zgpe`006j??jxcB$Z2Sl6?B{3DMR$O-@4MZvDV-WwJ>8wh+NuJKEdD zV?JYsaCGL5$T5}LbzP)i-WG$4DPT_z(p+&pQun*f#g6C~skK7JrLleAH@-g74Kp_H z0>orpvx^dgP*%^&CpgG34g zoO?R`t{2~LJ~?VQ@{(?(@HzIA3wb1bng%~I?8poD)N=mEAS+?BC>C3Fhm z6z~}=V&Yzq!D%$Ou~jd4?5?|xr?VR@&(zzPbzBK(TpygD36GhLo?L*9^_Obnqy3c>KqX%^p z(S62FmPN7H*GVC=93`x_9CNEIEk>@F7+{{1kgr6vhNyer{9>3>o`u6P`2GiSRhurP zHeSk>F{7t~W7_FuKwch3ozB^4Od6@B^Nm&Q2z%=M1of7rV;2DJmxc-+wxE2F{p8*&!> z)fIbJSooc}_IAA=Eb1=GZN%VdX{AonR+@%fjW{U=uL1-@hw}R^6FZbWvCI$5+Qf zSh^D(2e3cC8x~S0wd^mGEy^j{58a5{dqd75v!m?zjUY!2$}-T7)_k>14yQg)K%r%m zgI}B0Co?TP@5NRZKmE(5bla!`cPr7r5}H)IDif4H+K7lgS}+-8YSna86X9jCuwhWru7J&#nOrpw9B!>g(LmSHA4^S_rvT##N^R**aBA|8L{qef{AvS;1 zRjk6SrXX`Zye`!~%3IZ;W6rE2Zm_i`rlxRzYlJNUUjE!-sM31hi}ez#Od=IaIihj? zm8=%7Ei|qW9m(gRiSX0%YEX&6(&3@&MiYy0U{6#Sql_78{biQXhSf%P)j|V;eo{h0 z!p3&Ny1w%2A8b0i2Q^(5Tltn3)Iynen*+5n=!C4Fz)52SP{Kx za#e%Y98{QrptZz%w`mg86@+!TH`1 zUgv(Zv~QqqY8tI!JkB&mge%LkD83<|DDQ`&QIN}aDvxG;P#0=>GH4*JRK2oSG4k?2 znBftJD+6AB`R}*(7q&fp`s#y>73!sMfgRo6aF6ZkwDtMz`DIyUQ{*N1Py_u9UVHq__PNp}1NxP* zfqxY%Xu02ne=N2p{aUhrmcG76)~D{?e=jBYUYW;|Y-hZDiQgpwt?{>l5!^5jVsyFP ztJ8?wGONEu4>gvc=;3_QF4u-&#w^wk(|JdwE4m_Pe0QkU@WTd@s%2um}!_oSXl}O5)s?pYb|J?*3 zBfKsh<`KowAqws#h175QkGpOXWflnKa|DQjm*qd^5oA5|`wHud?&XuFB+^YceiY0M zZ0cfm^-*GxowphYvDy#cosx6G<6kwmsV%T~bn<;O;$zyo*5k#o`6O*VBgEpHZKC&ll{TdwOoZJ;KoyWLac4u`J}IkY=dUvg5w26DBV-aeEP?a~3GzL9CgK z8!APwtQBlbo~(5mRAWV72pI(lLe}t_and)Q(@}qZJnL&P*d@<#HbJ1S@DVeP(ZzOA zEP~0eTQvZY)q#?ThzNGp2MLy{lL9zepA-_GQgi6E%{?^>>4r?#MEvRcnF;^ z=}=t975nP7`k~w1HGQ6?5USWaek;QiO36bIgixLV)W< z%67ehsl&$so(9feJWQ6zE9R9rAF8Qcp{~f&!uvIrcJ0>{+*32d)oL$Ep`RsYgO-6) z5W+I$?^N$7wLekt%yFa?gi-fSntKoTGGDH+<@d=t3z^**A@lZ&c* zG(s;`F6?25o84>nw4H(arPW$^pEG|HM`6{Q3*T*JV=Y`g_t@4SH$Pmj+?0)?;~IK` zZPai!%(^y|QjwVI9cCZfKC9T~jcCIViYU4(=SGaAZQL4vtrsNSw(h`Y+E1v>OyD|a zFY%T@LOe@xSP!t1UVus)h}ZM!0sI{i=SM8S#^UL`plA8)dej=3ern}JK}8a@Y*xKY zG7}t8E!1fyy>aTa`-awr^ZzNAbMpkrBg`FPh{%yiLoM5mD>RNVtVI{QnhcgD?osT` zMzixt@Y|5*q+HWq;QKa5qrGEXnB)-rCA^W3%kFoBJukA>ApB%qwYs-u=8LCUq5h4d zYa021Vf8YZc6#?iNAvY2{{(Bo#%Z@jn) znmi&5=C!cMw*uyK`Om3~S1*DV%`W`nw@pQ$=6tgor_CZyHPOl6Sn8booB{-W?fu1U zBD)Eyp7%K(;K;zlffUohI)4?y4K^rVA^NP~?sa33M8FVOqs^@AeXep)yGhQ7Z!+Rg zD{Xp^XFA8$%rgp2=->a0r<2wvIcTaIyw}1Y0Ukx1AUyb`F|L&Ji!oV6{&&_b-FafE z(WA@B(Xx9(KB1fPwipr~2iJy%VpC5INBVMa2+nvGEYs2_q5@4>3*P5lw2EIF3Ao$(7hoQx6+9 zJqOHm7Z1IJZES3UBO>0ZsKicXV>?rl^t)ZZbz8hbqOhs`6yaJS9nM)RCE3s3krlg( z)o=G!xouo%>NgF+V5l1P_vK!leC1bwuxuS2#TgR1Bz^Pdjon5+Q6Pi3c+pZWDz*ZH zm9;fC9$o;Dc?55^GTqi=)t1)QzL=%CBjxAk?_~yfyf7Nd4)*YPuDIxld}|x5BRfRt zto2!}QbkgOEo1aX@8?BcSC6dis%3xs%(#+1y$hd*Bo+cti=Hzwq62EQy<$i>6i7oW z{fQi4B7xygS*)q4ES~yOiHle2*5~}Y7GUh1TCsNPX1Z_0kO-6%&{%DMeu^DmtwtJc z^d}yjp87{AOk(!E7Z)F!9<$(zj)^G)6r|Vv`DFLNKt}?b5jZ@2rAzmEc7Jn@Q!h@H z!8}QHjU=CegeLpofivTro zd2Q{5OD!O1i(b8a`C3u&84z4bZ5M^V$Hp4KM6{IQG@vwHma%5g)@a z>Wb)?Dl>xFvBGS~5HvkiZA+2neW8)3{0bzz@R&kpSO@io!ZInm*pML1An`UIg5co1 zTNJ-fi$1qcB_c>$0XJ=Tv?OY5Vj@Z#(>gSSS)f|f2KHiBKu4EJ(@?R0L{|?E=I@@L zCu3>8W7qY$HUVigj0Oc^LQ+ny(=6Q^DfT_0mX;Q*7r=P8e?b8ik1OI+v$LT({Mgv% zuj5(tIzc~2eJsr%#bUpHMeR0bck}c#y4afrqEQzJI|ep3%&_?jAw9;Y%8)a*v55c@ z4FUoJ9Nf#F(E;}?F&i*tAXD| zQ+YZ2K2Zm?M8wkl<)Va?6w0S$Y65q6cK~7~k~%=ZekDVHhJEe;!wUe`x5Pw{nV=gR z8$ajZz*%2khl8utjsQz!wA>ajQ$TY)Eh;|NNLY%6?OXvUbpznQ04{+SP2dqm3zETb zXk=t$SEc1799%?1#L>x#Ur-P-*!7Uq)YNKfYYn=;Q3{esiI0hmO-`=11)y~g4t8Fj zZrUHr@udiR6N7E?B@+`O+}Zvtfv&DDp^yhBKBGFEyu7@dyZcYu#U?5>d2w;r+MT@y zJQJHqFRn}?d-n7_sLmO96AxyoKU-y8Z$SezGiRo!~699Ne zjeQ?*x;bPi_B{tW)Ys>)S%hTgxW>=J!vjQJeJ~x^pyT5S)*NcP55e=^nh;=9$w+!! z)R(xZGRfRU)AztCuAGA$92{kymo~Tr1drifJVOB*+S1Mrs+pM?uYiEjpC6)j4i0TV zj9~lt=Q$wmB*LoSijNuTH2aVtA|iJ8^@)j!K0-x9i%L(AY-kWPHZ#l2%X{MY9InPa z%oe?a!ei^FDP?VVczE~Wm*8M0*ur+hP0!4GkWz54!#2o0qx@} zz=^W5^2glDvNCi08eTrWdF;-F#6;&!Eo!O}t;)*Efj2RCcV4*o_}pm9#d^)fsS6Vm zs-9uzM@KD2kwO!U;>%Bj0a(#sU?uNtYlF{x7v2UmCQUExY{j27HH3hAeaX!H4qynV z)!$N6!-0mnwz-*EUalz6C`0S7&D_=U`Eez*x}u_fRainoLNa^gz#~~)JT)gL2Z@G; zX2!_S-kuGx?k-*4KwbOukrNJ1NlEE!cM{W~mXw*9*=?_^>vxwp0RcfxU0oX(-kR$> zaq;<(M`+jzEYvYQODikz1O)M_xwUS`RH&$^AC#59WMoij*Eykp=4-%BpNDHRuq3PC z0ouHRf>GhFu`yf>42*Am>x8WOpSYd27n?-@D-!f@e`^V~wzVC4FbK};1J3`Wun-;Q z1M7Y9_UpZP`$fmS!V=^eRC*X!ouR$g|w_uS9mxh6O)0h zZ6p{5WMP5);^JcFC>!u?c6N3jUDAt+n8b)Obn}4K0M=$&vdlk@xanzV5HagDh4sa= zWP-7(sHhBV7#JI82@Ue{^BXR;d?w_!O5heZNB@G1P6+4e=^3>^kBv>lkHiR2ZxgPk z4&?qrK5JRor(gjl_R4QNn@MaF6BB1{SwO3Sfq?=Wz{2`wX1S+DU?h9;fs!v|FMV}v zU&Zc3Tyv2l6Z>WW>B_gJzTWckGDpPhcSQvR@Ynjr#xL3g=%_LlzJO)Ic1Ck_bWBK2 z7GsXDrvIi;Fg`FS7at#A?ZLpv2y~smi$kE9RPxG8OG{4yIQRCVs;a7%R#f^B#I*Or!OmX?nVOnzjpky8g@yh0 z#o_kKPiCk>o3X1;ztf z8P>$a1bD*yb6FW%VNuZ|IM|Qe5BHgiot-(;v2b9$z`Mf_);2Z^g^MIOI5_GW8kOp5 z(!~B5RdwDE)(RRLAvZTS-13WrGQV>>sQhmytd#$EYWAtLg%~^8w#7a97X7{qPGVbU0 zPEJ-A>P8=?a^)$k^$o49pOTP}>@U_8Q&9aIhr>Ef>+!vvEVtomG@L$(ws-9FSFVGw|h$1PCl%TYfbPEUwA|cWp zf~0hVfTW~K2$F)7v~&v6DAElg0@B@e=HhvO-}t_N&N%0carPL`GZZ&#uXW$^o^f5* zoEt8UDk&{3wisjvw})Kr$&)9tj~`39y9g0k}snq;~lDWtT42LnO1;9MnRufB!ZBl$ru);cwuml{GYi4-Q<<-e+KJ zZEikz{1_|r&6~|3ZQGe3YVTJNi$TPcs>+f=?Dm>kmy}KJ>Dm&kyBM-kJ4)P?RuP{D54g^H>8>D^9!-Z|&rN3WX&<2U| z!Lo`JBJPAxEY;cJ&E zHw;)4kWsXzNe2Dza!4X8btk>LT97eD4)VQ52-;$&I|NhS$JT^IM7Qbbufk0m#mTxi zLDmOjr5Kr+CBQmP^_Mqh(fL5N!j?YHmTxx)1U zDmVV4x5aW)D?RNhl|C_R==>A zfp8EM;Zab8fk1)|ZpY_rJH>9M3H{EUJB`!R)Hg$VPnHNJUrAWvmmNJ zoWBKOnU;nIb^ZEvyekh@ZUSZMj*SEuDcu!_s=?vm0kBq9p?~}-xQRT_+(@upoH?={=tWOrCIv?g-{C+NV)AqAkNzLMEqHBwXB*Na(WOW z!pFa4YikSgm&@RZuVG^cz%!5BRlzkjF?rqCD8AbB<9TKRiUTrYAZ)2eO5#0p}seal!J0&0v9__8N*ar+~ zYieuLLBR%fmjvkHKv+^_(s2{X3!&tOB>~HXZKd+PxI>JLtSmaXb4XT^8j92B%oDOf zI!;b}UtiyqudzFbkm1klY;;^)5WxC|kn+E@vwL7*Km+B}2ooDTBnF28_?$jaTS~wU zLk^3kudk2f(}}#+5>SM|To4YEcD!QwJJl>zIt8nQNY$Z~qdJyS_E1I!$=@!v5(@`8 zr0Kk%e4qe*5qf}xZ&48ka!{dqm6ey5)X*U9hWd~#T9yGaIye*(pwZ~-??>|R+M}gB zI5^jEab+ONkKBdrB*VHe4VSe44n1raaxJ;cn=I-DQig`N;i2r{iJ|yY&f3`9lSU4H zXy^vicC-QlWRQLO!IQt2yGu?^esQp53ptVJwD^`OA6Mjk>JG;Er&oqLKrXZCXOfWJ z-ijt97)?`Cdhj&WufxN!!9_(yNBe?03G%kruV0%BwcKN7zKN_pZQVu8&kotyaeN&V zWDJGLL7Ui_FK7*3G?-sSNB`E|E{j|zK$STF%30)I;R8T*W&~!9Y{1Kx9IyfZyD?+~ z()9|<%ZcIf-Z3Z!78bJi_4R=o^y>c(Jk%zH@WIMem6Zrd5D*Z6Am4}`;^CpFsi{Fi zlAfC@cd-#tZHPZ|&xdpJ1*B8pUo>9g6#S~e`aII>I`FVq;wZS76+gFjYQw` z5@a7yQF!0m+M2t%^3WWJluo;BGCV!;0u~C>3-n$&wm%lGzejrNw1lP}&ca#vC(nmO z!eN;zq)v?Lc%F&zv7i!F^fhv!mR+d5K^$G=w*4FwxI(VrXE+_^A0wKJ<^%$K<;YeR zBxh>N$;-}twxG2-5_C;dh!J4=FOWx=-`G(^@SADrxyIVI>6z7dVpWVz1;0au&aYu!kage zu&0RN!)m-7yP%)|39cXtx#Fwt4^hZ)HiwIrmP?k;W}*>7>L$c!BuK(aDQV zFYa35|8e)D=kvQbN#+Nq^?!Z-h!LE-E#oqM@zeWuLPVFi@@BVPcbnp@W*QyW`&6Aj zY1F^@{D?aA204ubHCiDFvBCSe!Ti}?mnh*w;`u<_!_z}289Rd`41Tdck5~_WQOAbk zZfJ;04cv1TzDrB%|28HDPV+TzL16O+kZK}04mbzE*v9V8g#KKLnEBPGk+Cik(B(F) zq@E?+yvtH)e}KYa*MCEet7kjy8NmGabXqLxAcd|nUf=iV=;}b6?)SL?sf=2}ppTvO z{T9qoHQx7}3y*OG$TA^q2&$eJWRAEy+rH&=uxNnpT89b&?e;Z@Cu1p_3MrzL5JUVW z(!@cM|0G$M$vK4fW~yK<`gFlq_zg{UWo<7?qPBON+w+duj+1t7tTePsytRJt3_HnM z`9sNzQy^f}(be4?vn+I4{$R5hY|t%4h$SvQ7k%~>XR>D*sID&TgBLP6-O zMzCNDh-;Am-f+5V3pg4Eugyd-r*SKiezDgd7{%#+7|P7RA*DPflSbncs!H0!-(nE; z|K)OO;9R(n%YJ)#yQ|2J`EozizQnXBxoXlOdX{(bYnvc!svD4%5FpX9Pn(2Y?>hpl zMv9%0f)__*SVe#X3Ga;;bT-i?8 zK7Nl&NtQT6=b;E%p19Ah9JdG+CB!z!?E^uXZ#7YgFCWYL0P>Gy&#gzo=O^2w+~!du zrp_X}JhCThv%OhCmbaFNjH#MVjcv}f`6op9&;HxLxyW=A>&ttuKl(O}qn`%yHw}4R z`Php6`a^j`0M~nxfy+z68TWL;xKWk6p?)uy#cr~w>h_{eR35r_0vHP#cLYa*gje+J zz_|8!ttMX7OAul*htrbkDUOy;V4)`2N4ltOeGf z_(eZGU_Eq#f@20vM5$CQLJg9SVg#;sdk~kpc!f1&Oi^?mei#_su>aRiB5|E{EhVze zr6IDB=qaap5t9Yi(~WI?x?<}N9KzV?Q0pt$SFlkJUZwUccD&x8ztT4*lijqN&U}rS zfWv0-@AFe~bGDPfw|SQh#`_2q3{0@Dl`(TqEN>Ct7aCEl#oZ&g^jk6B_21K#gJqvh z`6fvvR{PiA+o{T*&_@x5w=_?)`qwDnigYQ~h34ci`T6ZI7PelaTtWR`?V@rbW)=oF`8`f(Ei49p4XFK*{;BzQZ7k$_oO9Z1v-}Qm zHxCb_qG@SqIcPXP7Q4g102##%usOIzbMs%N;o27HA9zv)wr~72)eWj%P0Ic%%Vj`w z6pxlm-O#-G$<$nFSTe@zO3<`e@GXgcW2{JY1cFot$XmIBiJiR-Is8Cp(};;tM+$FU zgOWZ(z+t|zJv-~sni@U`9SPT7a^y{Z}0XO0f)SRUUOhdvNoH^KeU=I_(p za%M~LlIGq_g!;RY<7R*3(zm{)#l;UGBR|`UYXI~Q4T`UapsWUw(5tKzZ*t&$gpc;u z*_}VU+El_KpT;n&7mFBRNRebliDTxas&n|X$WHdSb_#P?h2AS^5DC{$32!V!(Qx|x z=*}T^yp4Z2cKyQjKe=%I)jCMY-`spVu4{8X(bj0}>kH@x5{LXBtjk@g(*E7z>%)WD z>NXD7c*@6N;RaS&jj2%!8zY@|E|dc8uH8DqN7Z74Pt-WxmUE;Hx*j|{N1y7iGgvo4 zBOst79mjsv{C+WhYqN`VVaP;4Zk1zakSG3hp0(1A#o&eM<(zKJA+|MAL0f#PNq!PE zY^wq7^dZ?y6;SZfiHcHz3SZ-^IX<8i!E26J8QBE3{X(}n4RqIA7>d6mDZX_^GP)^X&qO~H`i=#8}TTod^rYRY;gc7;_ z_QiO)G{A&VnT(9rT3abHB7%3+jk78NOMvRRX`;%Z7-~9znPB^7A&>(+A|-V-7%5@O z9_i{*LNNk3&~IoM121~Lu;EN_YkM0Kb~JW&cI0#WMxolm1J)2o4kRRkO;myod1iU} z4K#xQ!heB8kzGSp&QTrc0*wZ55hRWIPN4XI%giB)0*i}O{MpTrq9ZXDXgL~ywvfik zYDXqeGWkJI!EJVSMWk4h?LmstbmB~SFn)f7aykIqsL0^kWu3~Gm*C+5tP7H&MJk_k zwwc*k|H?{!AWdd7M02lyhLHU0HD$c732%r)zdQpZT%bWWw@At||804X0 zxA<_NS42ijiyU$Y8sNp?S!KClkkJ1M`X?!(;SgPoVGb#Ljx=<@{Ffwx8Dk*C->wRI z_*Hl?1UZB*pkm8mHl#`g3Jz+AW?%y^F1U_RwUAZ-wAJLXUKa#{_b{^iSL^HRkqX1h zs}3nEqh&8H$OU8&vKxUol%asDp`o1Dr^12!rG-)w?2M>Nf|mAl7_eUGB)tvAMn^{n zRN&vh-jQd5%}7g46-PEBI}zE8rMrMh-~;>!r4dqV!<7({F&bD>z#yEpJf(K&*$NnN-M`6}lFo5wD=`s|d7?LN6Io zZKYEI&0!2eOGZ}K@6dW@w>rjy;C%qS!J`91ft38TOA-jI<(>+(BLV=yk-v81hJQ_s z5MXd$lZLba{)EG&9BB&l&?+Gxm>RC=0gYOr9yTB_8yhcy{)KuUsFh2I?TWmX{`3Iu z@-iCQ%-UK6G^zdxAkh>54^QEA+cH6{X!DNh>i^LKEID~X9Zf*s|L2bp$g+q5K|?ss z|0lL0cpY&9XzF@;5&!;~BQ+Cvcz}pTsvH;qJ4A5D3+fy0v(OBf5HKX`X?0-jZr{F* z45UTUBVeWwuY>P|cX6%3kr(+AX^lv(0Tzxh%2515>m&zYw#c$nhyP#D3UmnuJbXiJ z*#B7pNnOjI>OhJ47+EzeS!?TiaP*s{f+k&KzfEEc03tzayvG&11qW>F= znj{nipLlR|L<=7^I5>FWN(2oMdBKmd`#NUI%J^`t8NE;pk0aCq=xFH1ve?E#H za}$zd4nS9+KmvV3Aao@H6lyS9VhRWs0F;dZH%J_l8G-`oX_vf_yR24*+8RNr)$-N_ z@@cNg%E?IpK7_amIKv3q4~PiZTTnxT51;B|$R~1&pH56n=+tg?vW(Pu@WG!E94J#| z^m7kxxU@eo@w6-~dAFG9Y15@IKA{7I3YP)rS($F2p7uHS%IinCWPl+4ht(k9u=P0a z-?_QlVDbPNO=|Gq+cmX0=NBnMEcMvq0 zknjp{LHNwjx6R4p9GVyvb)m((s%IG)S2Q&-Xp z4vFx^V~)Tn!Y3sS2JD=hn_E0z%^2Dh`91bnfp`H906Kh8(A9jh-*DctT-bo1{m_qO z09|uNai9!vZvv1D3<3D{W+?v=Bplq`lW`^+pgj>H!{^+4bwL44ippb!(-R;88Vx7rSm6{hgne$Wi?R;0ke$7MulDT(J`lI0nd3gnvE(B@evY z)(-lhhLPiMPJBRsue^Y``+*KvDnwTU!vNNk`>U#}CDN1}pPEAM@f1`Hm6es^nJQ(1 z;WXC{{=QS>05q!R3FH>52yB#1@JvnZPo&QYUH!>cmq+sL0dsC7oK+*3&(fkOV1ph{ zKB?txLZ?3BAhNTw{{`lPZ^K3GB`E{QBWNKK^FCT+$R1qsc*1_|G7?Q7_}XMcE4+IN zl&c>FopKdGwq|bL2svsPRDZY-IwqAU@s-ybFYe(ZR7zkcfFn!iHF(9zWg=uGWJTTG z-Cvv8d`>poCN-H62~LXV)^*gxrT|?``NTwFVWAB02J{LEuVKGj%yPh0#%|UzEK=MI z$*9qTK_BdsqHmQxLHnB_h(u^rrS2pHPzcNH+)?ZA=eIEzrw@M$+0nJU?IVBXfjHct~qfE#?l00Ha9O%SyvYu&NlEwp}OE$ydd=c_xjJj!jA#0$GNEC;WuD=A0Q{_zn9=e z-gNPHWemi+|9dUbAp7*+i}E7RxOjWEKN9Tzf4exDY-9R^M}2}%u~m(a`}2` zoRZQk_!w`TGO%hY+ZwB4(h%>`8>JHTo-1`B?ERQZ%2~ek>_+fz$7ezt3=Y@3&05oR zR7i$;Ov z_iky`r7zEWah zmpK-WAR)o&Jx8-7S#7&oF(u3Jw%%8RT=0rp$|;&Qru$yzmH+NW(6Htc_Q={lM*nfR z8I-}R<#0!%>{Xn)Y3fUthPyhIFj&G81<^4byx2#7K{~`2Sb=-}uAJxApecX1UhoTE zBXv6BW^LOHmZxF*cz)5j)cD&ekF#6Yvbspe%B{?IZ!X?yzPL1&U0rkmR_SC~g|?C8 zQQii3v^Z_F#OB1Mf$MX@iA?=x<*L)(`dJ=zWh15SS5-cUooVX$e-yDbh}fqY*|mFh z?5}7z+oSH&(4Bw8TJL@TiuW3t!M*$bnRPc>#p*AoP_vwSC5#nvPLGjFrkpbH{eah&ML(PESEJUett~Z^>hB*SaZ-}MWFLv{&N1u(# zbtiFS;S6!Lf4ORK@II?Zv^e$9QIK_-ci6)+`61K!w`E1q&`mnRqUaAzCDVAWinwha zVt+q%`7BK&n-0ydepy{ts4%^nDtMztF9oAvCgPv*^Q+6gYozBRDpq4DK?y?hdCj5& zv8NJobHCQSWc~k$2yH$sqdcMB_CKU#Px6b?2{&9hV9V=O`cjkbn#8HV?dQ@y z516YQl5Z{_zbGg1*h3%8Oh{qyIo_I#f&7Ni5Rq`q5+X|I%f5boIoRW&dag~#;?A6FE z7}vZtY^T1`rdvNHnCSiG4V%E)zaz5LZO$^~vLX*<{Rc*Zk6iaXyPkdxU|S7ZOAz~S zgL=1W$7;JBHE@vK>iuF=E@Wd=^9h~O)@5a)>M3fB*uzo!Sjx80fOGFqh-(@BnjblO zB2{3{cbv5A%-8%<;~EMrUFw#sP8gW4H><5T6{jy}KZ}`_*vfKs_M^-|6|Cz9@tt{~ zt7LsRXvo;E%*}1tJ!wAJVZv=^j{KdZUXiYph=rL=i5I%g_U!ig8@HFLW`DA}z8%b{ z5Mfq`T#`}X^;WR856u11z0r35{sG-JJXMdPc4_;_#X)}Rb8K#W54$z3dlrSs4?HUw z9xp3Gt@(+MKrZKoo?K2{l_ztBaRgQrx_5+SMQj`mH_dr-kb(WOR}6#HpAU+qG-Oq% zUa^ngGMC&3)MI}i6b!~2B?jfQOn*z$nvAneT&^plxccPUx8;;o8EHcSRc!3+Iz7Mc z*KwTRzToMfd9Zgn>Q`E4^>g(*3H&>n8msJHiZpPM;Cy}NBR=rWO6S?&rz3mbh_n5a z;0abO>toiUI-c8se3a^ux%gv~&-<7bvagTo{(#*v*O1Lk4^2?9rv2D-rV zArBV2*To#3hv%<5iL-Ku_QbOAGE{^N*Y;R`5OgI_2axE7T?v`9nt5aC$NO-l!nII| z<(l8TQ5c>9JrPCnl*{&ddSpP(bNZ&rhNPS1f?Ip49*j;KcKX{59kU5< zNpCkkrft7G``WTXEXTfvL+0uO;)BDa$m@PI*N7xMi^6F*$rx&#O4iET$X|ywxZR*{ zO|H0mVy#(UJ(P!EVAguWl30{BjNF7lBVIN!bBn&-_C}Vier1vyeNK)CTdB0QHcoXd zUs9~*X-|`!i}5#;3G$imt9b>__Q!%8N05^R2P-#5q9{{}gX|o4reesMZP#t4uH*TS z9u@N*i^}Uh3f9$j|EiP9oGAIc^tw~yllXv2=*F?JrJ?EAqNr<( zIXTvCAMWTr;%sljh)^hqd^;jw^ooo|RXtq`WwlOZ#Fxn?K6Get4ZE!vzOG>C@+%tg zS11?-^Z<3oOIbDj>dSF@ui?2o?Yi9$)HEBuahv^RYUyi%?*CR5(bkJ89DQu9b=q2I zi%FHMo%TgsbVsrLR9+>PIsr!^qd8@w34;?moFtIFA1+_EY$Sr#z}fiX=@J`6Gq&hYDufLS~=g) ze0mMOoE0~Yc>S>B)Az!s+O`PN& z-fXcd6-w~{qd}V?FDWgQDDuY2ODF#0U2#^#T~&2FkaTvPDKnpk<_FDv^w$Y;s!q>u zZR>TdIJ;=$ta{s8GjPTkPkeQAJ<6=un29j?sKORF*Ng4CI=wK)`Bw{L@6FFKsst`) zzin|Q?#!0T@$|R+-QSzUMt&dRjz_<&!W6SU#zb!=;5-kR`K2}%4d$|*|8%iuv)f?k zq55hfn2kjEo0d!^c@rfpdjq;AJmeJ0cof0^+bG!^on^Gix~n1ecuFb}o;1JzBzn!c zKXpaSM@l}g;`l?;es9jdNXrBFxtmt}@&2LJ?y5wmem$|Gkv_V=8vmS$8OTu=FR##i z@-C`)mnv#aQchKOWsvf?0Q+;m3I5)*X(Q7OmXQQc;<((UUL^-2w+}fZryb0~nj`;`D9&|lCct%j8?BwSK0?q*#D?%Jw6H<=br(5yE_2?vT~12!DhWN^ zjK~;8Lf`#^Rhv3&xg(?2Ch-w<%Et11?ah|8oADx*cn>5K^zudnmL(il^=$`*7MgTu zeZLEn>N~f-@Rrmh(N&X_iO4By?-Y)_N}%W(*%zrO=hf;Qlx$QZ>0;&bug4qK^H%X> znnx0Zkw877BGPox`+CjVkzMh-X~PfesQ6otXrf7*4?B zeC#Vkq26bgEmIOO*#oe98axVR&ZF}G7l+!R8t-o8(hUU zR*+}$uaUFt?2byLPEc==%tkvN?`?NlFaG+H$xD!HJ+X4(PI-%j3CAX988ePBW{>-` zhS$!y$Tkaq*0_C$&AHyadbL^TC7I}{9re^lgWJvllnli!U-n~M&7!F%jS zFuMz^;U39k&1JWq0BGS!Z~FvYbFaNQdXDMq^zKCU&` zuVQkpvmVzNHLc7Ai&T>2NmuY!&ekmsdfyK4oELsv%-D8j`}F1T=ZHfz4LokrrtszL z;-rWG-#wR)H~y74rQwd1O)c5|879aM!q-VYx^W`3{5QZCf@@w{QrkdaSlz*y-xVqc zf6j!-zx$8fojg}*!3x2>iHxz2teM}3m1=dCX?yf~+9S62|FB?F#{qY4Las^xXyU4u z04=S^GhY$K@WzzACl61m@?9sSSKQsB0%u+OJ%opJKL*@mh>2e}|CcW!sTL&3Qp+)y z;$OVEBPzT+apu5X{gryh?o3MR59GjIMU!bVf?1Qm8ApOJH|<`rjM-OeS9ECi2pTnL z?P6~_Zsj|8*@f=d7Ee|eK+ktQMDug}S9?mo(RTcpmrl66)*S)ce+I9uidTZi)^tVWd;`}UU`rB8fb>h3VYA;m5i+6kB`7pk0uCMZlx;o40K3%*cY3}tnWZ}EH) z>GRmd%QUp<5LQ!@yI+n|VrjVFDy<4GTu3htd1n^OxBp|t~eTP0EjgFGNqkBY-;7AHi+Knaf4{PpB zpsBoj?J^9>3jnSMbQ#8tuP((4yK&JgCSw8AUfe~u#Or?*d#S{&VW&?X$fl)|LxROb z5LF=BaULEbfX@J5M}FmOCrgae{0{As^dxZ+@PRJF^7w$OwXz(-NWU~y=3>(>V$a<{ z^^J`UabRj;644cCFpJK+!=(e$h?)RO`_AT{El)&DYy<=KfcS?wj{(3%zyl#l2+&yH z*SJl4^U^UOV>d&xvKSEh0-oYF=uSc3jR2#+=jX2i+%5(B*sotzSGWM|Rn)0-CjvGI zh_6?rrT1Z|$~{0+4S*O1@!dnv4+BX-T7ZWif;7TJw`Zs@!4F`%X*duscj)PXXhK71 zOF~R^g`Rg$FA6Jv2*zoC1X? zm=P6iU?A=7+n2Dhv1!$zeUuUr^uK-U2ZR7{#D>uOh%gN>WlKp##Se71@OEhNky}xL z8z-ubje;gN==z}rxpP`t8n8HlDmkxJa=wFN{}*WUP8Jn?n$jP>87YS05WXGV-QQt` z8luyF`}QU>a`kW_rR|-fXvhVEN(C123R!61=nGN?z9#m{*49>$aT|{3-l!Q6;sLNp z&wG-?kw#YFVT3l0oCegMlt%&r#(Gwt5!!d1%%kZxL&#;dqg zVIU`jkvmtRFHr;L=+vL?X#){tF2f|~b5=WI3VoCA(9Jid?2U|4UCc@`f>B~f`zY{3 zz(ltIo!{Pg!7bd@BHueqIX__lMENu2-r=8?H%4*L2WiPospbas{g1$BKfAj6ud=}o z+(*R3ONI$J$2zL216CtuaCmTzA4?4cj0t`z0WxPK6VPJ@V+8B>FXLV*ee>o4zk%&l+S^9g?XhQ z7WfSYgb*`IIyzy{MDY#01{^4$9ih%g?7Z$V$aTPJgD!ic5e*mVYzEzzG+Ym1>Y%9t z`U8u*?f`+^sIdYg#Iwy_Q0xgn%L4Rz92_kdmL*B!K$9bA9QnaHj40p1clrQ(0zS$U zX43KdoB{PJjCA+_ukP>{x~`VE5Vw@w1x*g{ZR(eSOBiDphS|@E3KyQo7dnJkVT1~( zPYA#zalM3NbFh?Gh#YRvr8RYQTt{Z|rOE^A7$}Ah#D-08Ur`SyC+ABSmmtJJgTMs5 z0CLZ0Dr#zhK&v8ie&OgK-P3UV5OyDwA2@lB;S!;UYQ(9g)iC7-wAg<9h$SW_hVaJF zlZ)`)AlpNFJdulus6w9{Y%Ma*7gSUl?FSt;{gx928oqKIdof z8`*Kt5jkcU11*}rmWn!KV`DF_eg$!4_={W^zen1#F5YWSp*9K>(8cv0%rL_B zGKRsRm3xNAiaIGw;V5BU(Dq!U_ppyoT1fN}>q8sh<`99Yw`lM^xLmG~UTo-mYDS@` zW*o3pQS)|mT|$T!<}Ug5H^ zsZ6*?-PR!T;RD6%WBThfezeLDG~OFvVY=R6^L-T^ymB;-e&vC#REv`nx-Frha--8j zbYBX;S0$b9uP)ItUt_(-NPFYaE2*juo)0Q`p$B4D7Du_)E4yL6P z6m+&5XquuL(L=7>K>hanyS}a;uk=V>o~2~P0Wt!OJ+f>AGi|lQs)b}@c%OK1#4g>F@8)6MUSiLR9g^ z9ul9Jj~{g^zi@RBp-`An2A5F3x#1|f&jgS_Q2z{4iruwIZCD)@=w_TL>49G0xijy1 zci%!chX_g-_Sat03A1OPLK6{Y+_Pl(6p(&ts*#~y^#%gOrsU9{oZ)90)8E&JNx;UMhlOe!jfT5QC48d0 zCqPFB1FL#s9p_7WU^>CVNiy)Xi!hT?MOF1Z^v$hyUZdn`AGdADx0$Sdfy^QR@svPU za`hK`cmia4>FP+aBy`>kfy_Yzxk)ZsSa5MNa5PNx^q*gjF2ENk{{x+5HFSjxLc#&d zB|_Ch<^-b)8G;a9V`;Z_0YS5V?kPD>>{Fo1&D zd2|63ivK`l!SuzrVtf#Y8LxwxAPdmYJqw7iV7-6;DM%3H9zBX4Uu8S-f?xRx<8~i0 zz~YN|l!Klw4aA3x^z?zIF_%#M1+-hY?r5M;8@_O7mT-*(KHFsEO(5ILgXx`M7V~7L zlsG|4gLzpXu7UhK9W(u*QyJX;E(|1SI6KPIfw>+jxVLy8v#5Rf1Ul2mTU;)izaFe|-`!fT^k#;RQCMyrk*}njx251}m z5^d`>(Rbxv`rC#o3F2Y>#*W7G*vw0~ulbwS?p0_OErgSH_C=6C-sOYQHPTUb~g zRM$sl;UgQ5qNYPO{;o!`Q2-YgmmI5En6js5Epl-fJn&4yB>M@-!-&YpazNx3m+Sfb z`Dd7a;za?s5+z%NLMbu9fMb@RX64)x2!>-Nrf&qDU{qaK+%YeQ>Me+(YM`_$1F0f> zZd`6e^UN=hE_zYG2S?rUcm_5A$xazGQpJ>dLo%5LEv)fJ1BKEy+vjX%X7M2|3=4!SO=}32*Ou z=-%OmLAXW>=Q(W0ie5{{>7poA?whPA)B#L$KK_|N^GH!~m~YzCVW=9M*jSZA7U&C7 zLBe#zEcLMs`7_MjA@=yBk+7WwZTMpqwkhC{Jjv)$C?2XVZZs6UUXh2JCR|cw0rQrV zmCat_K%vC3p=mb*R2;cDXyVuse(2(vnsx4F(AB9rXo&m<%-EHNl_`KX56~b+iM(p1 zz{eu5M*07bf25iK{cSDZzpFZtqEM+<|G{8XxWQaOEEG?0T`n3*$pLCkMzG{u2x%xD z7%HRkEC`hvXo%Plyk5m2TZp_|FCqaq)#{;Dx>0qxs)&!+D64z~&g`!4*=Q$qZ``Nx zj`MVUE$2$ES=OM5yW}c8^yhlEj6Y@*72;3V|LvjVeL1gp?){*zRlnYrcGNXo&A=Yl zTw+3bzImfcU;l4Rx#(&7%J5B9YB#O({hydtZwegATc$k$8$^9?x%7`HWfH?ZmNAgLEC(a{l@1LdF{oKxh_d^*Echaw+F?3sTfS} zI~Kj=3-A9${805p1HaP~;!fEFB`lge$3yg`>h*7zGu=THKBkuZqVaFMD zY>8KlNtlb}lrEr7?q3g0B%Y+`>x=trp_}AQPC0yZa{lShw9Go?4&(mzF_HZg#C_%|e}JRCkF}it^21(T1FlkkV2P312US1w?b~TL%m4Ovj^1#b)Q??mmbA&V9n08cXYai? zD>|V2bePf3Y^oz|iC%Gh$m*9qgU=VC*1JoAGQ>>IOXPJ8_v8cmuf%^!5WA}!ZTyGm zP&QPpGNV?((y=T4z;8PIk&>f|w4vMn+Bx%D+Lg)D9r50Csrf9+CgaWOncdY+RxY)A z$IaxUx}dZDcgcxc(KeKc8RPVxIqD5UUsM#amBUWT6d5NBJ4&oDtIluCeeicLv=yJ3 zAZY1H{H=WS$2Gq_w{Xfuig|-|dbL9%r^6>qrAE4pP_?kEpl*n6l+H1-QUiX7`2I~cS z@+2zDg2yWVX9Z3_OP90TU}VJl@2ca6YjPr<#kza)RLY%Xx3q{s@C@jNU_|I= zb&}gIMtmqddibI6uBMp9%+un~+Ke0vjDtM2PrOTGwqK@B&r;X*nFJNbniv*r*CZx& z#|z#_cmMkOATH;@WbD%Js|)5)MtIQ ziCf`h2fsWWR{y;d8}1n~#BmSfOtyL_?31%7@cG}ua73~#Ewz>reL)Yejd9o}1?RS& zJpZTejuh8^cDoci%4zqrg7D`X{+sk`v#0du14TM7=0C_RBra#2ns5o5M(%&#$4Ram z5lhxGW6h_fZgW}- zn#M6Fk4LiE74!`81m-`bVTXSIy>eLn=-&KGA_|O}Dyw!z2YEW}Ueb^1@eST_9wi&m zCa_tl_xJAY{4^&}p7Ofi8b6`%d<%OwT>mtAzU)U^#ln>76!ovB19WHZ*?K{fpY~tn ze}nl9l{0qP@l$Q5R|>Xnt-nY<9ZuDMw+L(v|Q)8_t|0d}4FnoY{E$-*)>1Ggu=ky{n<3pIja`HM=We`Vcf?@e-U zKWkv@FZ;gx?t`1Qu`$baf-KO@UGIt~*{8^2#2f1H1HSK*ZtaO8=4 z!xbFf3`-+I@eD&$wqacRKL|>?(>=&ZyT}abt=SiCl{7omupYs8g4C;&qo*F_1)p4{ z(?ID|3d@x3lzL&-ZEF2+|J+dTc~>jMxH?x~K8W>^44?I{E~X$;9_vib{ps9DdhCJF zxb*=cpOw@nzM4mln`+jbaVF@?1)Q8(0;W-F^%QATI2~?>JVz#-seaQ6WZCFw3GK@- z(B*ZL2SV?X^@|?UBv!9k)*Ht>U!o}5^KiO{+)t_5< zTi)IzOyIcodci%d3Qq0v&#Qy=zR$WV+`k^zG3+i?v!c6D-fMFuhthZgxr^$;l<~#z}GVkC8}>_Bes)d$VqpQ%4&0 zs^bbqLygLh1=QR=XVtTc_99~f%a0T$<39-w3uk;kf6gehAM9M{t*b2N`K7R{#`f)B ztl@v7_qjO^SVTk*nDPZb=23s1%8R~H>U6s8z%x@V$ov8~bO9{ub2+G-7;kp=B_peZj<jmz&Q@<}qtt&li7RST)9k2za@t1S!-`)z+7m3pvC6(H4?qh1uZ8SC2Z*~{MP zd{3a(G5u_>qp&1kZ?G%Hdid<44&_u#dPShKQe!WV`(lWI-0q?y<$}DnY`uU_xP2l{ zZyQb= z0irgWPBQCgQ(RpQ!N!hS5AJ0?sRE3CC!RoVE=cT`9@1yPhhN)|K zGK(qH?75m)j_D}8Ikq&yS?bEndp|dJCc{c)J~Q$ue;v=H4Y=kF8a8N?x<(t z;n?!a*fCD`g2e#yj^Dc&p!8oojzqJ!Y2UCGxtPDnmuD@K^>KIH zva$aY(cBKB)3_o!i$VOU*4ZNY+=0_CG~3$jOL2T^ukLH4O$NSbWpT*~i93FNBcyjn z05^Aa=u!3YPf>DyQB9{c_owS3zaE%^24}x5?QyO7>=_0{v~ z*8Bfv9WM@0Nay_7ZIC3$+J84~*2B#EhnDA>BXjB#hk$)IP0I+fx%Kf+k8N$1I# zQoydB&TVgfV9`q_H-i(&wQt%!JGeWnsF27PlhUC)wuY5dv;01j6{V`!J&W_Z$uuuN zn=PDBX~Ha-b+yy6%N56bsOwR}T_UW&SSC^e>pPlHOsTk3%mvn^nYpGUViR8TFm9#T zvoMNdue|N(QjduCV`0*nd(Jd&ZGwKKyx;3f@wA@_apV9q<^z&aH3u$@% zAz7OoD;JTc_o+*+Vpi&reai_2ku+DUtQuP8s$JKQ~9|ao)pB zQDUYRJI1c6d4*f!|2)}5>W{#ElN1oTj`mufz z8Li^d)AK>Qe0rD6)XX%F-}Z@S=VfuH@7IV?By23=3QHeXCFv?S-$n~lJ>4)46P|aq zfB5!qbe?eJ2x~`Q&a{KkJF2-9%f;6YtsZZiZG={dqGxZtXWke4a`pgor_{vW{{8G~ zV7Ru8%eQYXHIwLKRFT5Te#4lUmG523KM=FLznj&t9cFB$DR`D3Ia7FNxm!ri!M!=o zW!l~czBETW#`4a7!JJn0*~`1rQSZ0--^cPM?CG(!uaaArn1VXV$(L_Yl_8_i-q3xTA@Q zg~?icuS*5|6rFSts`8~~tX0bdO8%`?*+5Bjid6&eAc5k;rTIS5`7+kgJ7kg3*~+ly^T@ZJEO9X ztd*yDWAAj)i)o{C7YL-L_)M`s6_Spu!^>LB!XoK9HVjZ(+m;Ov&HGn~$}%a5T+(>- zX!z|>i=EoL`p}xpf>>`ZMR~5{+`|d2p7)ZvLKRArwT^Y0Kc&5nw(%+3x7$}c;rm}2nBC^2MRw=ut@*vahy2K? zeh@oRqf*AoBjeU0Tvj$%n2(p$n-;FkHWXtB^uePe=??iG11t@ z!*D2LO5wmH=FxCg;TA)z;PcpJa`XLLMIXPVVWsdTZF{eM-}x_HN>5Dd25bk1clq=` zq4seTcg{TLrAJ6x$9X1p4&B^F;Z&50o_Zx_aExdS0A@{4O za`@|_XVR^b@8yhUx76?Cm6P+~L`8U~`6wt370?~pFPwa~PEqLnRYPyNJj&%*E)7ts z?-$7wi}}Q9%k!$6nRj>JG-;6kE3Xu_a9p|ZBk8`(I{O&ic{YE`SJ7R{X!};}d;GgK zi5@w{IOa2z$2((jLXM0*nZ_@s|0OBY?e^#kV=j5WaVNZ=%g*Bc>he8#wcp}*pGo;- zvsurSlGZLJR*jXr)ofADEO`d_Tpwpb>fdC57qYnl>_omXb%_IWaCPL7apCo$-js6l zrh&|*ZP|49Fx~7+Jw63(1Qq|f1}5%LXI1e0;#ob({IrHvn)sdB zKZ}aU_mK1#B6(~-{$)93cQEF3V3O(4(5CifdF)RNFOGhd3m(k9sb*MG;lXz9#2-+~ zXOA3MU-wZCnKtWmIy%k$^u}gwRHO4m1NW>T=kh75W~}Z#Y7M>0CFz+T@unS^|MEtI z!nW#3bCeop8yuv~`}Q2_YMxOfYGHdxTS z^yOADoS}z9RQKHYnI;_V~h|$qmmekW&6FTn@xi6KZ(uCH%RRToCRvMJw#! zvD^P^=~Hx9#?J}*bgWwBu9ADajjf-23ft=Pm@Qpx0rx6^=Xif!DDvvgxA4DzpSgW{ z^DBV=`<{ref97)W?RBgFaa}F-`medi{PwsX-CGG9gG+g@vwQKOwxsaX=qdBoCw*9y zzsoG^>a4?`vabi*Mo-`H7ucOt0G=in4jd$3n!^n|>Q#q93wVTk3ve)Ql~vE=H4a~Q ztlpk>q%`3Dw$dva7yS8_2Lm^R28Hym{&J^7`^w8N7hg_up7?jyYhWKyL@H|cMnYlpsBn-XAd zyZ6ies?>}ndU*mX8g{J}F)Z07W+T1SA@URd2m7CAu8J*su{~(Zuf6Ibw-+}@UE!Tz zykcQgYy9fE#qpO9`KxQZ`oc0b)W3Y+tQFJWb%xwNw)M%1GvP{2>n|72uURAB?y@TP zumkIr%#|GrFK?|l^>$T#mBi9J=EV<+ZKZyPuPwQ=#&ffL@bg1Yz5iVKb+7)_TKkra z*HbKbXYc?0C%SR5Np9dgvvmojd2^QdKkE&i-={0`W`p*ILiW3VF8q%ARwZw0b^N8U zgry^QdhZq9T-X8Saj{nu#r}JN}T;1HY^2w#PeDf2k_FE-?)xU}jWL8||8T~`2 zt?1j?Kfc~y-^c5eN*Dx$Pn+;o%ll^6B;e>JaANq>O$i2%O=l!Gf1G-+Q~e@v)2FDI z0K)<+%OyvhZ?FD;DbDLhX5X)?y0L5bgw_YP%LPBJ`I4PmvyEH0=^<0_zPiSs6Y=Yp zTa{luvbQ|z+cB5Bch~GnpSfzY&04wYomFQtmaep~StY1@{_CA>I;o5cvO-+ynpaCR z9N;c3y{)|X>dzDHzmC6&?~)8H^pLmtQEmI>SWvEqy;gYLv~TPDFYIW)e@UM!@cg}p zD&Yl;S}Wrh?6I-7zg)k~Qq(YS5!KjInMg@*3k6j=W^Ri$_sX^ zckwKmY+~@?M~tNEzaQFJr~L9hY2V!WoAuP`Yh`P~p37Z6wehXzxiHP?W`5Ov+*iZT z_q<g#vzU(P)Kd!@_7UN7SLD{+U?=K;s}uMPfoUuzezF`N$?6t)w+v1_`U@9U+3p|U4h zkJL8^ZK~aMhYvV@d#WS79XycB5C@#d0go2Wiw6x+H*kjq1x*6p)VdlpD$Kzt#xOMv zcmd0ya&-KtJ-BMDV>+ z>YI*3(3Ak=6?&|g)!)1G>1_O7f z0}nS^5Ag=WiY!po*U_O-Ub`OVqXeLbz+gSm0Si`6F!+g3byHO}tQ0sk1p-%Z{%4Ng VTKPmKAlwzi_jL7hS?83{1OOC&nhF2_ From 0eb79b16e83e82550abc2a8771312982b8a2aabc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:18:46 +0000 Subject: [PATCH 152/182] minor changes to comments --- src/simple_varinfo.jl | 1 + src/varinfo.jl | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index a0d9e5925..19589613b 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -676,6 +676,7 @@ end # With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. from_internal_transform(vi::SimpleVarInfo, ::VarName) = identity from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) = identity +# TODO: Should the following methods specialize on the case where we have a `StaticTransformation{<:Bijectors.NamedTransform}`? from_linked_internal_transform(vi::SimpleVarInfo, ::VarName) = identity function from_linked_internal_transform(vi::SimpleVarInfo, ::VarName, dist) return invlink_transform(dist) diff --git a/src/varinfo.jl b/src/varinfo.jl index 369c68ca9..3b930eebb 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1449,7 +1449,6 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end - # TODO: Replace when we have better dispatch for multiple vals. return recombine(dist, vals_linked, length(vns)) end From 071bebfb4e7fd669994ae5dc7aecf1cfcbc92f6b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:20:35 +0000 Subject: [PATCH 153/182] remove Combinatorics as a test dep, as it's not needed for this PR --- test/Project.toml | 2 -- test/runtests.jl | 2 -- 2 files changed, 4 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index e2bd0d7e5..80c227920 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,7 +2,6 @@ AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" -Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -27,7 +26,6 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" AbstractMCMC = "5" AbstractPPL = "0.7" Bijectors = "0.13" -Combinatorics = "1" Compat = "4.3.0" Distributions = "0.25" DistributionsAD = "0.6.3" diff --git a/test/runtests.jl b/test/runtests.jl index d15b17b42..43d68386c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,8 +20,6 @@ using Random using Serialization using Test -using Combinatorics: combinations - using DynamicPPL: getargs_dottilde, getargs_tilde, Selector const DIRECTORY_DynamicPPL = dirname(dirname(pathof(DynamicPPL))) From bbdc0603a521ad40a0cc4063f3f8f50df2dd0030 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:22:01 +0000 Subject: [PATCH 154/182] reverted unnecessary change --- test/test_util.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test_util.jl b/test/test_util.jl index 1d23567da..64832f51e 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -84,9 +84,7 @@ Return string representing a short description of `vi`. """ short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" -function short_varinfo_name(vi::TypedVarInfo) - return "TypedVarInfo" -end +short_varinfo_name(::TypedVarInfo) = "TypedVarInfo" short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" From e2f4d185f686d6343634ee18d28b75fb4ad5fedb Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:32:40 +0000 Subject: [PATCH 155/182] disable type-stability tests for models on older Julia versions --- test/model.jl | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/test/model.jl b/test/model.jl index f8303e260..6e1a052aa 100644 --- a/test/model.jl +++ b/test/model.jl @@ -348,30 +348,34 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true end end - @testset "Type stability of models" begin - models_to_test = [ - # FIXME: Fix issues with type-stability in `DEMO_MODELS`. - # DynamicPPL.TestUtils.DEMO_MODELS..., - DynamicPPL.TestUtils.demo_lkjchol(2), - ] - @testset "$(model.f)" for model in models_to_test - vns = DynamicPPL.TestUtils.varnames(model) - example_values = DynamicPPL.TestUtils.rand(model) - varinfos = filter( - is_typed_varinfo, - DynamicPPL.TestUtils.setup_varinfos(model, example_values, vns), - ) - @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos - @test (@inferred(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())); - true) - - varinfo_linked = DynamicPPL.link(varinfo, model) - @test ( - @inferred( - DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext()) - ); - true + if VERSION >= v"1.8" + @testset "Type stability of models" begin + models_to_test = [ + # FIXME: Fix issues with type-stability in `DEMO_MODELS`. + # DynamicPPL.TestUtils.DEMO_MODELS..., + DynamicPPL.TestUtils.demo_lkjchol(2), + ] + @testset "$(model.f)" for model in models_to_test + vns = DynamicPPL.TestUtils.varnames(model) + example_values = DynamicPPL.TestUtils.rand(model) + varinfos = filter( + is_typed_varinfo, + DynamicPPL.TestUtils.setup_varinfos(model, example_values, vns), ) + @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos + @test ( + @inferred(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())); + true + ) + + varinfo_linked = DynamicPPL.link(varinfo, model) + @test ( + @inferred( + DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext()) + ); + true + ) + end end end end From 3d823ac119921cb5079b742ab2ee611a963cc3e0 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:32:53 +0000 Subject: [PATCH 156/182] removed seemingly completely unused impl of `setval!` --- src/varinfo.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 3b930eebb..bab89f118 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -464,12 +464,6 @@ getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, v Set the value of `vi.vals[vview]` to `val`. """ setval!(vi::UntypedVarInfo, val, vview::VarView) = vi.metadata.vals[vview] = val -function setval!(vi::UntypedVarInfo, val, vview::Vector{UnitRange}) - if length(vview) > 0 - vi.metadata.vals[[i for arr in vview for i in arr]] = val - end - return val -end """ getmetadata(vi::VarInfo, vn::VarName) From 607bdb37a98b4c65a3159721d26efb9f35db4075 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 18 Jun 2024 22:22:41 +0100 Subject: [PATCH 157/182] Update test/model.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/model.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/model.jl b/test/model.jl index 7f59312c8..3360bdea0 100644 --- a/test/model.jl +++ b/test/model.jl @@ -356,8 +356,7 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true if VERSION >= v"1.8" @testset "Type stability of models" begin models_to_test = [ - DynamicPPL.TestUtils.DEMO_MODELS..., - DynamicPPL.TestUtils.demo_lkjchol(2), + DynamicPPL.TestUtils.DEMO_MODELS..., DynamicPPL.TestUtils.demo_lkjchol(2) ] @testset "$(model.f)" for model in models_to_test vns = DynamicPPL.TestUtils.varnames(model) From 55c8098823af6e5e01eab817626cee76a3e93916 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 21 Jun 2024 20:59:31 +0100 Subject: [PATCH 158/182] Apply suggestions from code review Co-authored-by: Markus Hauru --- docs/src/internals/transformations.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index d4721bddb..67ff3c68c 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -21,7 +21,7 @@ For certain inference methods, it's necessary / much more convenient to work wit We write "unconstrained" with quotes because there are many ways to transform a constrained variable to an unconstrained one, *and* DynamicPPL can work with a much broader class of bijective transformations of variables, not just ones that go to the entire real line. But for MCMC, unconstraining is the most common transformation so we'll stick with that terminology. -For a large family of constraints encoucntered in practice, it is indeed possible to transform a (partially) contrained model to a completely unconstrained one in such a way that sampling in the unconstrained space is equivalent to sampling in the constrained space. +For a large family of constraints encountered in practice, it is indeed possible to transform a (partially) constrained model to a completely unconstrained one in such a way that sampling in the unconstrained space is equivalent to sampling in the constrained space. In DynamicPPL.jl, this is often referred to as *linking* (a term originating in the statistics literature) and is done using transformations from [Bijectors.jl](https://github.com/TuringLang/Bijectors.jl). @@ -37,7 +37,7 @@ end Here `log_s` is an unconstrained variable, and `s` is a constrained variable that is a deterministic function of `log_s`. -But to ensure that we stay consistent with what the user expects, DynamicPPL.jl does not actually transform the model as above, but can instead makes use of transformed variables internally to achieve the same effect, when desired. +But to ensure that we stay consistent with what the user expects, DynamicPPL.jl does not actually transform the model as above, but instead makes use of transformed variables internally to achieve the same effect, when desired. In the end, we'll end up with something that looks like this: @@ -55,7 +55,7 @@ There are two aspects to transforming from the internal representation of a vari 1. Different implementations of [`AbstractVarInfo`](@ref) represent realizations of a model in different ways internally, so we need to transform from this internal representation to the desired representation in the model. For example, - + [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. + + [`VarInfo`](@ref) represents a realization of a model as a "flattened" / vector representation, regardless of the form of the variable in the model. + [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). 2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. @@ -91,7 +91,7 @@ DynamicPPL.to_internal_transform DynamicPPL.from_internal_transform ``` -These methods allows us to extract the internal-to-model transformation function depending on the `varinfo`, the variable, and the distribution of the variable: +These methods allow us to extract the internal-to-model transformation function depending on the `varinfo`, the variable, and the distribution of the variable: - `varinfo` + `vn` defines the internal representation of the variable. - `dist` defines the representation expected within the model scope. @@ -263,7 +263,7 @@ we see that we indeed satisfy the constraint `m < x`, as desired. The reason for this is that internally in a model evaluation, we construct the transformation from the internal to the model representation based on the *current* realizations in the model! That is, we take the `dist` in a `x ~ dist` expression _at model evaluation time_ and use that to construct the transformation, thus allowing it to change between model evaluations without invalidating the transformation. -But to be able to do this, we need to know whether the variable is linked / "unconstrained" or not, since the transformation is different in the two cases. Hence we need to be able to determine this at model evaluation time. Hence the the internals end up looking something like this: +But to be able to do this, we need to know whether the variable is linked / "unconstrained" or not, since the transformation is different in the two cases. Hence we need to be able to determine this at model evaluation time. Hence the internals end up looking something like this: ```julia if istrans(varinfo, varname) From ad959ec114aee117720eeba54243a1ffb9588489 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 21 Jul 2024 17:56:54 +0100 Subject: [PATCH 159/182] Type-stability tests are now correctly using `rand_prior_true` instead of `rand` --- test/model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/model.jl b/test/model.jl index 546578222..b354279cf 100644 --- a/test/model.jl +++ b/test/model.jl @@ -360,7 +360,7 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true ] @testset "$(model.f)" for model in models_to_test vns = DynamicPPL.TestUtils.varnames(model) - example_values = DynamicPPL.TestUtils.rand(model) + example_values = DynamicPPL.TestUtils.rand_prior_true(model) varinfos = filter( is_typed_varinfo, DynamicPPL.TestUtils.setup_varinfos(model, example_values, vns), From 9f8407059d8387c8f62095387a9911f6c238385b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 21 Jul 2024 17:57:24 +0100 Subject: [PATCH 160/182] `getindex_internal` now calls `getindex` instead of `view`, as the latter can result in type-instability since transformed variables typically result in non-view even if input is a view --- src/varinfo.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 91d2fabf9..238b07c95 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -521,7 +521,10 @@ getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) -getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +# TODO(torfjelde): Use `view` instead of `getindex`. Requires addressing type-stability issues though, +# since then we might be returning a `SubArray` rather than an `Array`, which is typically +# what a bijector would result in, even if the input is a view (`SubArray`). +getindex_internal(md::Metadata, vn::VarName) = getindex(md.vals, getrange(md, vn)) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) return mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) From 7d399349aa0e8dac9df360d0d528926005db413f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 21 Jul 2024 17:58:16 +0100 Subject: [PATCH 161/182] Removed seemingly unnecessary definition of `getindex_internal` --- src/varinfo.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 238b07c95..f239cedb9 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -462,8 +462,6 @@ end const VarView = Union{Int,UnitRange,Vector{Int}} -getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) - """ setval!(vi::UntypedVarInfo, val, vview::Union{Int, UnitRange, Vector{Int}}) From b554504b460616b02406c3aef9b1db730bb66e7f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 11:52:31 +0200 Subject: [PATCH 162/182] Fixed references to `newmetadata` which has been replaced by `replace_values` --- src/varinfo.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index f239cedb9..600517da5 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -113,7 +113,7 @@ const VarInfoOrThreadSafeVarInfo{Tmeta} = Union{ transformation(vi::VarInfo) = DynamicTransformation() function VarInfo(old_vi::VarInfo, spl, x::AbstractVector) - md = newmetadata(old_vi.metadata, Val(getspace(spl)), x) + md = replace_values(old_vi.metadata, Val(getspace(spl)), x) return VarInfo( md, Base.RefValue{eltype(x)}(getlogp(old_vi)), Ref(get_num_produce(old_vi)) ) @@ -155,7 +155,8 @@ function VarInfo(rng::Random.AbstractRNG, model::Model, context::AbstractContext end # TODO: Remove `space` argument when no longer needed. Ref: https://github.com/TuringLang/DynamicPPL.jl/issues/573 -function replace_values(metadata::Metadata, space, x) +replace_values(metadata::Metadata, space, x) = replace_values(metadata, x) +function replace_values(metadata::Metadata, x) return Metadata( metadata.idcs, metadata.vns, @@ -168,7 +169,7 @@ function replace_values(metadata::Metadata, space, x) ) end -@generated function newmetadata( +@generated function replace_values( metadata::NamedTuple{names}, ::Val{space}, x ) where {names,space} exprs = [] From ddb1dfe5245864b9add77c113838fd1fcef84fb4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 13:15:36 +0200 Subject: [PATCH 163/182] Made implementation of `recombine` more explicit --- src/utils.jl | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index b72bd4914..84f673e86 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -320,23 +320,21 @@ vectorize(d, r) = vectorize(r) vectorize(r) = tovec(r) """ - recombine(dist::Distribution, vals::AbstractVector, n::Int) + recombine(dist::Union{UnivariateDistribution,MultivariateDistribution}, vals::AbstractVector, n::Int) Recombine `vals`, representing a batch of samples from `dist`, so that it's a compatible with `dist`. + +!!! warning + This only supports `UnivariateDistribution` and `MultivariateDistribution`, which are the only two + distribution types which are allowed on the right-hand side of a `.~` statement in a model. """ -function recombine(d::Distribution, val::AbstractVector, n::Int) - return recombine(size(d), val, n) -end -function recombine(::Tuple{}, val::AbstractVector, n::Int) +function recombine(::UnivariateDistribution, val::AbstractVector, ::Int) + # This is just a no-op, since we're trying to convert a vector into a vector. return copy(val) end -function recombine(s::NTuple{1}, val::AbstractVector, n::Int) - return copy(reshape(val, s[1], n)) -end -function recombine(s::NTuple{2}, val::AbstractVector, n::Int) - tmp = reshape(val, s..., n) - orig = [tmp[:, :, i] for i in 1:n] - return orig +function recombine(d::MultivariateDistribution, val::AbstractVector, n::Int) + # Here `val` is of the length `length(d) * n` and so we need to reshape it. + return copy(reshape(val, length(d), n)) end # Uniform random numbers with range 4 for robust initializations From 3b08f1d01434d6eed35a9e1c55d5ce2dc3e546bd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 16:10:26 +0100 Subject: [PATCH 164/182] Added docstrings for `untyped_varinfo` and `typed_varinfo` --- src/varinfo.jl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 600517da5..959e9beb4 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -119,6 +119,11 @@ function VarInfo(old_vi::VarInfo, spl, x::AbstractVector) ) end +""" + untyped_varinfo([rng, ]model[, sampler, context]) + +Return an untyped `VarInfo` instance for the model `model`. +""" function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -128,10 +133,15 @@ function untyped_varinfo( varinfo = VarInfo() return last(evaluate!!(model, varinfo, SamplingContext(rng, sampler, context))) end -function untyped_varinfo(model::Model, args...) +function untyped_varinfo(model::Model, args::Union{AbstractSampler,AbstractContext}...) return untyped_varinfo(Random.default_rng(), model, args...) end +""" + typed_varinfo([rng, ]model[, sampler, context]) + +Return a typed `VarInfo` instance for the model `model`. +""" typed_varinfo(args...) = TypedVarInfo(untyped_varinfo(args...)) function VarInfo( From 96ccebeb544429d1ba50ea20669786fa8082bbb1 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 16:13:05 +0100 Subject: [PATCH 165/182] Added TODO comment about implementing `view` for `VarInfo` --- src/varinfo.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index 959e9beb4..1db2f82b3 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -533,6 +533,7 @@ getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, # TODO(torfjelde): Use `view` instead of `getindex`. Requires addressing type-stability issues though, # since then we might be returning a `SubArray` rather than an `Array`, which is typically # what a bijector would result in, even if the input is a view (`SubArray`). +# TODO(torfjelde): An alternative is to implement `view` directly instead. getindex_internal(md::Metadata, vn::VarName) = getindex(md.vals, getrange(md, vn)) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) From beaeeaa256d3618d155e700ec663c0914cf5cf98 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 16:13:26 +0100 Subject: [PATCH 166/182] Fixed potential infinite recursion as suggested by @mhauru --- src/extract_priors.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index ff77e1a4b..634740875 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -110,7 +110,7 @@ julia> length(extract_priors(rng, model)[@varname(x)]) 9 ``` """ -extract_priors(args...) = extract_priors(Random.default_rng(), args...) +extract_priors(args::Union{Model,AbstractVarInfo}...) = extract_priors(Random.default_rng(), args...) function extract_priors(rng::Random.AbstractRNG, model::Model) context = PriorExtractorContext(SamplingContext(rng)) evaluate!!(model, VarInfo(), context) From ab2c98b58b25fd018ccaf525194852f6a56158eb Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 16:16:29 +0100 Subject: [PATCH 167/182] added docstring to `from_vec_trnasform_for_size --- src/utils.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils.jl b/src/utils.jl index 84f673e86..048cd9b89 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -256,6 +256,11 @@ Return the transformation from the vector representation of `x` to original repr from_vec_transform(x::Union{Real,AbstractArray}) = from_vec_transform_for_size(size(x)) from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ FromVec(size(C.UL)) +""" + from_vec_transform_for_size(sz::Tuple) + +Return the transformation from the vector representation of a realization of size `sz` to original representation. +""" from_vec_transform_for_size(sz::Tuple) = FromVec(sz) from_vec_transform_for_size(::Tuple{()}) = FromVec(()) from_vec_transform_for_size(::Tuple{<:Any}) = identity From f1f7968b8329926f2ecc9518db0beaa2889e16a2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 16:25:07 +0100 Subject: [PATCH 168/182] Replaced references to `vectorize(dist, x)` with `tovec(x)` --- src/DynamicPPL.jl | 1 - src/context_implementations.jl | 10 +++++----- src/simple_varinfo.jl | 2 +- src/utils.jl | 4 ---- src/varinfo.jl | 8 ++++---- test/turing/varinfo.jl | 2 +- test/utils.jl | 4 ++-- 7 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index f27bab87d..eb027b45b 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -81,7 +81,6 @@ export AbstractVarInfo, @model, # Utilities init, - vectorize, OrderedDict, # Model Model, diff --git a/src/context_implementations.jl b/src/context_implementations.jl index f6f1a72be..f45b2bc5d 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -75,7 +75,7 @@ end function tilde_assume(context::PriorContext{<:NamedTuple}, right, vn, vi) if haskey(context.vars, getsym(vn)) - vi = setindex!!(vi, vectorize(right, get(context.vars, vn)), vn) + vi = setindex!!(vi, tovec(get(context.vars, vn)), vn) settrans!!(vi, false, vn) end return tilde_assume(PriorContext(), right, vn, vi) @@ -84,7 +84,7 @@ function tilde_assume( rng::Random.AbstractRNG, context::PriorContext{<:NamedTuple}, sampler, right, vn, vi ) if haskey(context.vars, getsym(vn)) - vi = setindex!!(vi, vectorize(right, get(context.vars, vn)), vn) + vi = setindex!!(vi, tovec(get(context.vars, vn)), vn) settrans!!(vi, false, vn) end return tilde_assume(rng, PriorContext(), sampler, right, vn, vi) @@ -92,7 +92,7 @@ end function tilde_assume(context::LikelihoodContext{<:NamedTuple}, right, vn, vi) if haskey(context.vars, getsym(vn)) - vi = setindex!!(vi, vectorize(right, get(context.vars, vn)), vn) + vi = setindex!!(vi, tovec(get(context.vars, vn)), vn) settrans!!(vi, false, vn) end return tilde_assume(LikelihoodContext(), right, vn, vi) @@ -106,7 +106,7 @@ function tilde_assume( vi, ) if haskey(context.vars, getsym(vn)) - vi = setindex!!(vi, vectorize(right, get(context.vars, vn)), vn) + vi = setindex!!(vi, tovec(get(context.vars, vn)), vn) settrans!!(vi, false, vn) end return tilde_assume(rng, LikelihoodContext(), sampler, right, vn, vi) @@ -603,7 +603,7 @@ function set_val!( @assert size(val) == size(vns) foreach(CartesianIndices(val)) do ind dist = dists isa AbstractArray ? dists[ind] : dists - setindex!!(vi, vectorize(dist, val[ind]), vns[ind]) + setindex!!(vi, tovec(val[ind]), vns[ind]) end return val end diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index c2f817195..db0d3b0f4 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -545,7 +545,7 @@ values_as(vi::SimpleVarInfo) = vi.values values_as(vi::SimpleVarInfo{<:T}, ::Type{T}) where {T} = vi.values function values_as(vi::SimpleVarInfo{<:Any,T}, ::Type{Vector}) where {T} isempty(vi) && return T[] - return mapreduce(vectorize, vcat, values(vi.values)) + return mapreduce(tovec, vcat, values(vi.values)) end function values_as(vi::SimpleVarInfo, ::Type{D}) where {D<:AbstractDict} return ConstructionBase.constructorof(D)(zip(keys(vi), values(vi.values))) diff --git a/src/utils.jl b/src/utils.jl index 048cd9b89..29c7ed104 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -320,10 +320,6 @@ tovec(x::Real) = [x] tovec(x::AbstractArray) = vec(x) tovec(C::Cholesky) = tovec(Matrix(C.UL)) -# TODO: Remove these. -vectorize(d, r) = vectorize(r) -vectorize(r) = tovec(r) - """ recombine(dist::Union{UnivariateDistribution,MultivariateDistribution}, vals::AbstractVector, n::Int) diff --git a/src/varinfo.jl b/src/varinfo.jl index 1db2f82b3..bfb9e5604 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -552,7 +552,7 @@ function setval!(md::Metadata, val::AbstractVector, vn::VarName) return md.vals[getrange(md, vn)] = val end function setval!(md::Metadata, val, vn::VarName) - return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) + return md.vals[getrange(md, vn)] = tovec(val) end """ @@ -1248,7 +1248,7 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar f = internal_to_linked_internal_transform(varinfo, vn, dist) y, logjac = with_logabsdet_jacobian(f, x) # Vectorize value. - yvec = vectorize(dist, y) + yvec = tovec(y) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) # Mark as no longer transformed. @@ -1348,7 +1348,7 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe f = from_linked_internal_transform(varinfo, vn, dist) x, logjac = with_logabsdet_jacobian(f, y) # Vectorize value. - xvec = vectorize(dist, x) + xvec = tovec(x) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) # Mark as no longer transformed. @@ -1624,7 +1624,7 @@ function BangBang.push!!( end function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) - val = vectorize(dist, r) + val = tovec(r) meta.idcs[vn] = length(meta.idcs) + 1 push!(meta.vns, vn) l = length(meta.vals) diff --git a/test/turing/varinfo.jl b/test/turing/varinfo.jl index 30408e598..f1d805505 100644 --- a/test/turing/varinfo.jl +++ b/test/turing/varinfo.jl @@ -14,7 +14,7 @@ elseif is_flagged(vi, vn, "del") unset_flag!(vi, vn, "del") r = rand(dist) - vi[vn] = vectorize(dist, r) + vi[vn] = DynamicPPL.tovec(r) setorder!(vi, vn, get_num_produce(vi)) r else diff --git a/test/utils.jl b/test/utils.jl index 1fcf09ef1..3f435dca4 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -43,9 +43,9 @@ @test getargs_tilde(:(@~ Normal.(μ, σ))) === nothing end - @testset "vectorize" begin + @testset "tovec" begin dist = LKJCholesky(2, 1) x = rand(dist) - @test vectorize(dist, x) == vec(x.UL) + @test DynamicPPL.tovec(x) == vec(x.UL) end end From 6e57822346549ecda4744233740d0e96370a5b6d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 17:02:06 +0100 Subject: [PATCH 169/182] Fixed docstring --- src/utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.jl b/src/utils.jl index 29c7ed104..5e30baa5e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -388,7 +388,7 @@ end """ collect_maybe(x) -Return `collect(x)` if `x` is an array, otherwise return `x`. +Return `x` if `x` is an array, otherwise return `collect(x)`. """ collect_maybe(x) = collect(x) collect_maybe(x::AbstractArray) = x From 841215f72017dc0ed8310e8f7b7596aa3fa16706 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 17:09:04 +0100 Subject: [PATCH 170/182] Update src/extract_priors.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/extract_priors.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index 634740875..6b0dfd1e4 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -110,7 +110,8 @@ julia> length(extract_priors(rng, model)[@varname(x)]) 9 ``` """ -extract_priors(args::Union{Model,AbstractVarInfo}...) = extract_priors(Random.default_rng(), args...) +extract_priors(args::Union{Model,AbstractVarInfo}...) = + extract_priors(Random.default_rng(), args...) function extract_priors(rng::Random.AbstractRNG, model::Model) context = PriorExtractorContext(SamplingContext(rng)) evaluate!!(model, VarInfo(), context) From 78b2083ac0cc2adb00c514aa9b261d0f726ecd6e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jul 2024 22:25:30 +0100 Subject: [PATCH 171/182] Bump minor version since this is a breaking change --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 78acb2566..2deb163b8 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DynamicPPL" uuid = "366bfd00-2699-11ea-058f-f148b4cae6d8" -version = "0.28" +version = "0.29" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" From bab63e1f716a6f792e0f95e17db1f0e739b71504 Mon Sep 17 00:00:00 2001 From: Xianda Sun <5433119+sunxd3@users.noreply.github.com> Date: Tue, 30 Jul 2024 22:51:25 +0800 Subject: [PATCH 172/182] Apply suggestions from code review Co-authored-by: Markus Hauru --- docs/src/internals/transformations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 67ff3c68c..d948290ec 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -25,7 +25,7 @@ For a large family of constraints encountered in practice, it is indeed possible In DynamicPPL.jl, this is often referred to as *linking* (a term originating in the statistics literature) and is done using transformations from [Bijectors.jl](https://github.com/TuringLang/Bijectors.jl). -For example, the above model could be transformed into (the following psuedo-code; it's not working code): +For example, the above model could be transformed into (the following pseudo-code; it's not working code): ```julia @model function demo() @@ -214,7 +214,7 @@ Unfortunately, this is not possible in general. Consider for example the followi end ``` -Here the variable `x` has is constrained to be on the domain `(m, Inf)`, where `m` is sampled according to a `Normal`. +Here the variable `x` is constrained to be in the domain `(m, Inf)`, where `m` is sampled according to a `Normal`. ```@example transformations-internal model = demo_dynamic_constraint() From 6997019c8e945da4e820771631ad0b69c68650ac Mon Sep 17 00:00:00 2001 From: Xianda Sun <5433119+sunxd3@users.noreply.github.com> Date: Tue, 30 Jul 2024 22:53:55 +0800 Subject: [PATCH 173/182] Update src/varinfo.jl Co-authored-by: Tor Erlend Fjelde --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index de5abd146..2670397d9 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2045,7 +2045,7 @@ end function values_from_metadata(md::Metadata) return ( - # `copy` to avoid accidentaly mutation of internal representation. + # `copy` to avoid accidentally mutation of internal representation. vn => copy( from_internal_transform(md, vn, getdist(md, vn))(getindex_internal(md, vn)) ) for vn in md.vns From 9dc7f027a44a9cfd663a8f4fe49866eb0add7855 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jul 2024 00:32:19 +0100 Subject: [PATCH 174/182] Apply suggestions from code review --- src/utils.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 554d3af3f..9877de7aa 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -226,14 +226,14 @@ invlink_transform(dist) = inverse(link_transform(dist)) ##################################################### # Useful transformation going from the flattened representation. -struct FromVec{Sz} <: Bijectors.Bijector - sz::Sz +struct FromVec{Size} <: Bijectors.Bijector + size::Size end FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) # TODO: Should we materialize the `reshape`? -(f::FromVec)(x) = reshape(x, f.sz) +(f::FromVec)(x) = reshape(x, f.size) (f::FromVec{Tuple{}})(x) = only(x) # TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. From c0f9923f8fe026ff69d311f00e0b5a412c3dd5ed Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jul 2024 00:35:48 +0100 Subject: [PATCH 175/182] Apply suggestions from code review --- test/simple_varinfo.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 74dcdd842..5ce112941 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -224,7 +224,6 @@ @testset "Static transformation" begin model = DynamicPPL.TestUtils.demo_static_transformation() - priors = extract_priors(model) varinfos = DynamicPPL.TestUtils.setup_varinfos( model, DynamicPPL.TestUtils.rand_prior_true(model), [@varname(s), @varname(m)] From 9056928e8fa85aa4a99436293c87b1ccd465d2c7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 6 Aug 2024 09:41:53 +0100 Subject: [PATCH 176/182] Update src/extract_priors.jl Co-authored-by: Xianda Sun <5433119+sunxd3@users.noreply.github.com> --- src/extract_priors.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index 6b0dfd1e4..dd5aeeb04 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -119,7 +119,6 @@ function extract_priors(rng::Random.AbstractRNG, model::Model) end """ - extract_priors(model::Model, varinfo::AbstractVarInfo) Extract the priors from a model. From e43dd1bbfa03df326c3c3df8ec0637a511ebe90a Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 6 Aug 2024 21:24:07 +0100 Subject: [PATCH 177/182] Added fix for product distributions of targets with changing support + tests --- src/abstract_varinfo.jl | 10 ---------- src/utils.jl | 12 +++++++++++- test/model.jl | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 5852472c0..7ddd09b2e 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -743,16 +743,6 @@ function unflatten(sampler::AbstractSampler, varinfo::AbstractVarInfo, ::Abstrac return unflatten(varinfo, sampler, θ) end -# # NOTE: Necessary to handle product distributions of `Dirichlet` and similar. -# function with_logabsdet_jacobian_and_reconstruct( -# f::Bijectors.Inverse{<:Bijectors.SimplexBijector}, dist, y -# ) -# (d, ns...) = size(dist) -# yreshaped = reshape(y, d - 1, ns...) -# x, logjac = with_logabsdet_jacobian(f, yreshaped) -# return x, logjac -# end - """ to_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) diff --git a/src/utils.jl b/src/utils.jl index 9877de7aa..9233c20ff 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -274,6 +274,16 @@ distribution `dist` to the original representation compatible with `dist`. from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ FromVec(size(dist)) +""" + from_vec_transform(f, size::Tuple) + +Return the transformation from the vector representation of a realization of size `size` to original representation. + +This is useful when the transformation alters the size of the realization, in which case we need to account for the +size of the realization after pushed through the transformation. +""" +from_vec_transform(f, sz) = from_vec_transform_for_size(Bijectors.output_size(f, sz)) + """ from_linked_vec_transform(dist::Distribution) @@ -285,8 +295,8 @@ By default, this is just `invlink_transform(dist) ∘ from_vec_transform(dist)`. See also: [`DynamicPPL.invlink_transform`](@ref), [`DynamicPPL.from_vec_transform`](@ref). """ function from_linked_vec_transform(dist::Distribution) - f_vec = from_vec_transform(dist) f_invlink = invlink_transform(dist) + f_vec = from_vec_transform(inverse(f_invlink), size(dist)) return f_invlink ∘ f_vec end diff --git a/test/model.jl b/test/model.jl index b354279cf..6afa0e119 100644 --- a/test/model.jl +++ b/test/model.jl @@ -414,4 +414,21 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true # confused and call it the way you are meant to call `a_model`. @test_throws MethodError instance(1.0) end + + @testset "Product distribution with changing support" begin + @model function product_dirichlet() + x ~ product_distribution(fill(Dirichlet(4), 2, 3)) + end + model = product_dirichlet() + + varinfos = [ + DynamicPPL.untyped_varinfo(model), + DynamicPPL.typed_varinfo(model) + ] + @testset "$(varinfo)" for varinfo in varinfos + varinfo_linked = DynamicPPL.link(model, varinfo) + varinfo_linked_result = last(DynamicPPL.evaluate!!(model, deepcopy(varinfo_linked), DefaultContext())) + @test getlogp(varinfo_linked) == getlogp(varinfo_linked_result) + end + end end From a7673fd31e3028fd28eb323b7541939b2775f8db Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 6 Aug 2024 22:40:12 +0100 Subject: [PATCH 178/182] Addeed tests for product of distributions with dynamic support --- src/context_implementations.jl | 1 - src/simple_varinfo.jl | 10 ++++++++++ src/utils.jl | 2 +- test/model.jl | 10 ++++++---- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/context_implementations.jl b/src/context_implementations.jl index f45b2bc5d..78740e4f0 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -602,7 +602,6 @@ function set_val!( ) @assert size(val) == size(vns) foreach(CartesianIndices(val)) do ind - dist = dists isa AbstractArray ? dists[ind] : dists setindex!!(vi, tovec(val[ind]), vns[ind]) end return val diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index db0d3b0f4..d8afb9cec 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -248,6 +248,16 @@ function SimpleVarInfo{T}( return SimpleVarInfo(values, convert(T, getlogp(vi))) end +function untyped_simple_varinfo(model::Model) + varinfo = SimpleVarInfo(OrderedDict()) + return last(evaluate!!(model, varinfo, SamplingContext())) +end + +function typed_simple_varinfo(model::Model) + varinfo = SimpleVarInfo{Float64}() + return last(evaluate!!(model, varinfo, SamplingContext())) +end + unflatten(svi::SimpleVarInfo, spl::AbstractSampler, x::AbstractVector) = unflatten(svi, x) function unflatten(svi::SimpleVarInfo, x::AbstractVector) logp = getlogp(svi) diff --git a/src/utils.jl b/src/utils.jl index 9233c20ff..9ddeb6247 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -304,7 +304,7 @@ end function from_linked_vec_transform(dist::LKJCholesky) return inverse(Bijectors.VecCholeskyBijector(dist.uplo)) end -from_linked_vec_transform(dist::LKJ) = inverse(Bijectors.VecCorrBijector()) +from_linked_vec_transform(::LKJ) = inverse(Bijectors.VecCorrBijector()) """ to_vec_transform(x) diff --git a/test/model.jl b/test/model.jl index 6afa0e119..616dd7d1e 100644 --- a/test/model.jl +++ b/test/model.jl @@ -417,16 +417,18 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true @testset "Product distribution with changing support" begin @model function product_dirichlet() - x ~ product_distribution(fill(Dirichlet(4), 2, 3)) + x ~ product_distribution(fill(Dirichlet(ones(4)), 2, 3)) end model = product_dirichlet() varinfos = [ DynamicPPL.untyped_varinfo(model), - DynamicPPL.typed_varinfo(model) + DynamicPPL.typed_varinfo(model), + DynamicPPL.typed_simple_varinfo(model), + DynamicPPL.untyped_simple_varinfo(model), ] - @testset "$(varinfo)" for varinfo in varinfos - varinfo_linked = DynamicPPL.link(model, varinfo) + @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos + varinfo_linked = DynamicPPL.link(varinfo, model) varinfo_linked_result = last(DynamicPPL.evaluate!!(model, deepcopy(varinfo_linked), DefaultContext())) @test getlogp(varinfo_linked) == getlogp(varinfo_linked_result) end From e8d4c9694054c2e9d3eb847b06aa0a69446b9d4b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 6 Aug 2024 22:49:26 +0100 Subject: [PATCH 179/182] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/model.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/model.jl b/test/model.jl index 616dd7d1e..6959b400b 100644 --- a/test/model.jl +++ b/test/model.jl @@ -417,7 +417,7 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true @testset "Product distribution with changing support" begin @model function product_dirichlet() - x ~ product_distribution(fill(Dirichlet(ones(4)), 2, 3)) + return x ~ product_distribution(fill(Dirichlet(ones(4)), 2, 3)) end model = product_dirichlet() @@ -429,7 +429,9 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true ] @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos varinfo_linked = DynamicPPL.link(varinfo, model) - varinfo_linked_result = last(DynamicPPL.evaluate!!(model, deepcopy(varinfo_linked), DefaultContext())) + varinfo_linked_result = last( + DynamicPPL.evaluate!!(model, deepcopy(varinfo_linked), DefaultContext()) + ) @test getlogp(varinfo_linked) == getlogp(varinfo_linked_result) end end From d7c224e9a6d4ed5b449ccc50e481417a4fbd8175 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 14 Aug 2024 13:49:03 +0100 Subject: [PATCH 180/182] Empty commit to trigger CI From a0a8761f3e04e93e5a7a21698c1ef4755256950d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 14 Aug 2024 14:35:47 +0100 Subject: [PATCH 181/182] Update test/model.jl Co-authored-by: Markus Hauru --- test/model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/model.jl b/test/model.jl index 6959b400b..60a8d2461 100644 --- a/test/model.jl +++ b/test/model.jl @@ -432,7 +432,7 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true varinfo_linked_result = last( DynamicPPL.evaluate!!(model, deepcopy(varinfo_linked), DefaultContext()) ) - @test getlogp(varinfo_linked) == getlogp(varinfo_linked_result) + @test getlogp(varinfo_linked) ≈ getlogp(varinfo_linked_result) end end end From a8812e9e7585b2fd8f0b58f9ad8a95c956ebfa6c Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 21 Aug 2024 10:24:00 +0100 Subject: [PATCH 182/182] Increase HTML page size threshold for docs --- docs/make.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index 65d43d524..ebf15df06 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -14,7 +14,9 @@ DocMeta.setdocmeta!(DynamicPPL, :DocTestSetup, :(using DynamicPPL); recursive=tr makedocs(; sitename="DynamicPPL", - format=Documenter.HTML(), + # The API index.html page is fairly large, and violates the default HTML page size + # threshold of 200KiB, so we double that. + format=Documenter.HTML(; size_threshold=2^10 * 400), modules=[DynamicPPL], pages=[ "Home" => "index.md",