From 7d2ba341809eb296b92505da3d2ee0cf9fd89d95 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 24 May 2023 15:37:00 -0600 Subject: [PATCH 01/12] New branch for julia interface pull request --- .gitignore | 1 + Makefile | 3 + braid/Makefile | 6 +- braid/braid.c | 79 +-- braid/braid.h | 10 +- braid/braid.jl/XBraid.jl | 732 +++++++++++++++++++++++++++ braid/braid.jl/status.jl | 173 +++++++ braid/braid.jl/wrapper_functions.jl | 310 ++++++++++++ braid/braid_test.c | 176 ++++++- braid/braid_test.h | 52 ++ drivers/julia/drive-TaylorGreen.jl | 329 ++++++++++++ drivers/julia/drive-advdiff-theta.jl | 151 ++++++ drivers/julia/drive-burgers.jl | 334 ++++++++++++ drivers/julia/drive-kflow.jl | 443 ++++++++++++++++ examples/julia/ex-01.jl | 69 +++ makefile.inc | 11 +- 16 files changed, 2817 insertions(+), 62 deletions(-) create mode 100644 braid/braid.jl/XBraid.jl create mode 100644 braid/braid.jl/status.jl create mode 100644 braid/braid.jl/wrapper_functions.jl create mode 100644 drivers/julia/drive-TaylorGreen.jl create mode 100644 drivers/julia/drive-advdiff-theta.jl create mode 100644 drivers/julia/drive-burgers.jl create mode 100644 drivers/julia/drive-kflow.jl create mode 100644 examples/julia/ex-01.jl diff --git a/.gitignore b/.gitignore index 4211a325..df34340c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ wrapperTests.txt # binaries # as these are build-dependent, binaries should not be committed *.o +*.so *.a ex-01 ex-01-adjoint diff --git a/Makefile b/Makefile index 12f13736..4bca0295 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,9 @@ all: braid examples braid: cd braid; $(MAKE) +ifeq ($(shared),yes) + cd braid; $(MAKE) libbraid.so +endif examples: ./braid/libbraid.a cd examples; $(MAKE) diff --git a/braid/Makefile b/braid/Makefile index 09ce70eb..df720c9f 100644 --- a/braid/Makefile +++ b/braid/Makefile @@ -89,8 +89,12 @@ libbraid.a: $(BRAID_HEADERS) $(BRAID_OBJ) ar cruv libbraid.a $(BRAID_OBJ) ranlib libbraid.a +libbraid.so: $(BRAID_HEADERS) $(BRAID_OBJ) + @echo "Building" $@ "..." + $(MPICC) $(SEQFLAGS) $(CFLAGS) $(SHAREDFLAGS) $(BRAID_OBJ) -o $@ + all: libbraid.a clean: - rm -f *.o libbraid.a + rm -f *.o libbraid.a libbraid.so diff --git a/braid/braid.c b/braid/braid.c index 1c187b27..8afb2e0c 100644 --- a/braid/braid.c +++ b/braid/braid.c @@ -493,7 +493,6 @@ braid_Destroy(braid_Core core) braid_App app = _braid_CoreElt(core, app); braid_Int nlevels = _braid_CoreElt(core, nlevels); _braid_Grid **grids = _braid_CoreElt(core, grids); - braid_Int cfactor = _braid_GridElt(grids[0], cfactor); braid_Int gupper = _braid_CoreElt(core, gupper); braid_Int richardson = _braid_CoreElt(core, richardson); braid_Int est_error = _braid_CoreElt(core, est_error); @@ -509,24 +508,54 @@ braid_Destroy(braid_Core core) _braid_TFree(_braid_CoreElt(core, tnorm_a)); _braid_TFree(_braid_CoreElt(core, rdtvalues)); - /* Destroy stored Lyapunov exponents */ - if (_braid_CoreElt(core, lyap_exp)) + if (grids[0] != NULL) { + braid_Int cfactor = _braid_GridElt(grids[0], cfactor); + /* Destroy stored Lyapunov exponents */ + if (_braid_CoreElt(core, lyap_exp)) + { + + braid_Int npoints; + if (nlevels == 1) + { + npoints = _braid_GridElt(grids[0], iupper) - _braid_GridElt(grids[0], ilower); + } + else + { + npoints = _braid_GridElt(grids[0], ncpoints); + } + for (braid_Int i = 0; i < npoints; i++) + { + _braid_TFree(_braid_CoreElt(core, local_exponents)[i]); + } + _braid_TFree(_braid_CoreElt(core, local_exponents)); + } + - braid_Int npoints; - if (nlevels == 1) + /* Free last time step, if set */ + if ( (_braid_CoreElt(core, storage) < 0) && !(_braid_IsCPoint(gupper, cfactor)) ) { - npoints = _braid_GridElt(grids[0], iupper) - _braid_GridElt(grids[0], ilower); + if (_braid_GridElt(grids[0], ulast) != NULL) + { + _braid_BaseFree(core, app, _braid_GridElt(grids[0], ulast)); + } } - else + + /* Destroy Richardson estimate structures */ + if ( richardson || est_error ) { - npoints = _braid_GridElt(grids[0], ncpoints); + _braid_TFree(_braid_CoreElt(core, dtk)); + if (est_error) + { + _braid_TFree(_braid_CoreElt(core, estimate)); + } } - for (braid_Int i = 0; i < npoints; i++) + + for (level = 0; level < nlevels; level++) { - _braid_TFree(_braid_CoreElt(core, local_exponents)[i]); + _braid_GridDestroy(core, grids[level]); } - _braid_TFree(_braid_CoreElt(core, local_exponents)); + } /* Destroy the optimization structure */ @@ -537,47 +566,19 @@ braid_Destroy(braid_Core core) _braid_TFree(_braid_CoreElt(core, optim)); } - /* Free last time step, if set */ - if ( (_braid_CoreElt(core, storage) < 0) && !(_braid_IsCPoint(gupper, cfactor)) ) - { - if (_braid_GridElt(grids[0], ulast) != NULL) - { - _braid_BaseFree(core, app, _braid_GridElt(grids[0], ulast)); - } - } - - /* Destroy Richardson estimate structures */ - if ( richardson || est_error ) - { - _braid_TFree(_braid_CoreElt(core, dtk)); - if (est_error) - { - _braid_TFree(_braid_CoreElt(core, estimate)); - } - } - - for (level = 0; level < nlevels; level++) - { - _braid_GridDestroy(core, grids[level]); - } - _braid_TFree(grids); - _braid_TFree(core); if (timer_file_stem != NULL) { _braid_TFree(timer_file_stem); } - } if (_braid_printfile != NULL) { fclose(_braid_printfile); } - - return _braid_error_flag; } diff --git a/braid/braid.h b/braid/braid.h index 3606ed0c..fb378ffd 100644 --- a/braid/braid.h +++ b/braid/braid.h @@ -57,15 +57,15 @@ extern "C" { */ /** Define Fortran name-mangling schema, there are four supported options, see braid_F90_iface.c */ -#define braid_FMANGLE 1 +#define braid_FMANGLE 0 /** Turn on the optional user-defined spatial coarsening and refinement functions */ #define braid_Fortran_SpatialCoarsen 0 /** Turn on the optional user-defined residual function */ -#define braid_Fortran_Residual 1 +#define braid_Fortran_Residual 0 /** Turn on the optional user-defined time-grid function */ -#define braid_Fortran_TimeGrid 1 +#define braid_Fortran_TimeGrid 0 /** Turn on the optional user-defined sync function */ -#define braid_Fortran_Sync 1 +#define braid_Fortran_Sync 0 /** @} */ @@ -605,7 +605,7 @@ braid_PrintTimers(braid_Core core /**< braid_Core (_braid_Core) struc **/ braid_Int braid_ResetTimer(braid_Core core /**< braid_Core (_braid_Core) struct*/ -); + ); /** * After Drive() finishes, this function can be called to write out the convergence history diff --git a/braid/braid.jl/XBraid.jl b/braid/braid.jl/XBraid.jl new file mode 100644 index 00000000..285342d2 --- /dev/null +++ b/braid/braid.jl/XBraid.jl @@ -0,0 +1,732 @@ +#=BHEADER********************************************************************** + * Copyright (c) 2013, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * This file is part of XBraid. For support, post issues to the XBraid Github page. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License (as published by the Free Software + * Foundation) version 2.1 dated February 1999. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the terms and conditions of the GNU General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + ***********************************************************************EHEADER=# + +module XBraid +# Not sure we want to export anything... +# export Init, Drive, etc. + +# BEGIN MODULE XBraid +using Serialization: serialize, deserialize, Serializer # used to pack arbitrary julia objects into buffers +using LinearAlgebra: norm2 +using MPI + +include("status.jl") +include("wrapper_functions.jl") + +#= + | For this to work, XBraid must be compiled as a shared library. + | $ make braid shared=yes + | and the Fortran flags braid_Fortran_SpatialCoarsen, braid_Fortran_Residual, + | braid_Fortran_TimeGrid, and braid_Fortran_Sync must all be set to zero in + | braid.h before compiling. (TODO: figure out why this is...) + =# +libbraid = joinpath(dirname(@__DIR__), "libbraid.so") + +c_stdout = Libc.FILE(Libc.RawFD(1), "w") # corresponds to C standard output +function malloc_null_double_ptr(T::Type) + pp = Base.Libc.malloc(sizeof(Ptr{Cvoid})) + pp = reinterpret(Ptr{Ptr{T}}, pp) + return pp +end + +# This can contain anything (TODO: do I even need this?) +mutable struct BraidVector{T} + user_vector::T + VecType::Type +end +BraidVector(u::T) where T = BraidVector(u, T) + +OptionalFunction = Union{Function, Nothing} +""" +This is an internal structure that is not generally exposed to the user. +This enables the user interface to only use memory safe function calls. +User defined data structures that are not time-dependent can be declared in +the global scope, or they can be packed into a user defined object and passed to +Init(), in which case the object will be passed as the first argument in step, sum, spatialnorm, and access. +_app.user_app is passed as the first argument to some user defined function. +Stores any time-independent data the user may need to compute a time-step. +Large, preallocate arrays should be packed into this object, since operations +involving globally scoped variables are slower than local variables. +struct braid_App end + +This may be manually contructed in order to pass to test functions. + +julia> app = BraidApp(my_app, MPI.COMM_WORLD, MPI.COMM_WORLD, my_step, my_init, my_sum, my_norm, my_access); + +julia> testSpatialNorm(app, 0.); + """ +mutable struct BraidApp + user_app::Any # user defined app data structure + + comm::MPI.Comm # global mpi communicator + comm_t::MPI.Comm # temporal mpi communicator + + # user functions + step::Function + init::Function + sum::Function + spatialnorm::Function + access::Function + + basis_init::OptionalFunction + inner_prod::OptionalFunction + + # dictionary to store globally scoped references to allocated braid_vector objects + ref_ids::IdDict{UInt64, BraidVector} + bufsize::Integer # expected serialized size of user's vector + bufsize_lyap::Integer + user_AppType::Type + user_VecType::Type + user_BasType::Type +end + +# some default functions assuming the user's vector is a julia array +function default_sum!(app, α::Real, x::AbstractArray, β::Real, y::AbstractArray) + y .= α .* x .+ β .* y + return +end + +function default_norm(app, u::AbstractArray) + return norm2(u) +end + +function default_inner_prod(app, u::AbstractArray, v::AbstractArray) + return u ⋅ v +end + +# default constructor +function BraidApp(app, comm::MPI.Comm, comm_t::MPI.Comm, step::Function, init::Function, sum::Function, norm::Function, access::Function, basis_init::Function, inner_prod::Function) + BraidApp(app, comm, comm_t, step, init, sum, norm, access, basis_init, inner_prod, IdDict(), 0, 0, Nothing, Nothing, Nothing) +end + +function BraidApp(app, comm::MPI.Comm, comm_t::MPI.Comm, step::Function, init::Function, sum::Function, norm::Function, access::Function) + BraidApp(app, comm, comm_t, step, init, sum, norm, access, nothing, nothing, IdDict(), 0, 0, Nothing, Nothing, Nothing) +end + +function BraidApp(app, comm::MPI.Comm, step::Function, init::Function, sum::Function, norm::Function, access::Function) + BraidApp(app, comm, comm, step, init, sum, norm, access) +end + +function BraidApp(app, comm::MPI.Comm, step, init, access) + BraidApp(app, comm, comm, step, init, default_sum!, default_norm, access) +end + +function BraidApp(app, comm::MPI.Comm, step, init, access, basis_init) + BraidApp(app, comm, comm, step, init, default_sum!, default_norm, access, basis_init, default_inner_prod) +end + +""" +Stores all the information needed to run XBraid. Create this object with Init(), then pass this to Drive() to run XBraid. + +julia> core = XBraid.Init(MPI.COMM_WORLD, MPI.COMM_WORLD, + +julia> XBraid.Drive(core); + +""" +mutable struct BraidCore + # internal values + _braid_core::Ptr{Cvoid} + _braid_app::BraidApp + tstart::Float64 + tstop::Float64 + ntime::Int32 + function BraidCore(_braid_core, _braid_app, tstart, tstop, ntime) + x = new(_braid_core, _braid_app, tstart, tstop, ntime) + finalizer(x) do core + # @async println("Destroying BraidCore $(core._braid_core)") + @ccall libbraid.braid_Destroy(core._braid_core::Ptr{Cvoid})::Cint + end + end +end + +""" +Used to add a reference to the vector u to the IdDict stored in the app. +this keeps all newly allocated braid_vectors in the global scope, preventing them from being garbage collected. +""" +function _register_vector(app::BraidApp, u::BraidVector) + app.ref_ids[objectid(u)] = u +end + +""" +Used to remove a reference to the vector u to the IdDict stored in the app. +""" +function _deregister_vector(app::BraidApp, u::BraidVector) + id = objectid(u)::UInt64 + pop!(app.ref_ids, id) +end + +function postInitPrecompile(app::BraidApp) + #= + # This is a hack to make sure every processor knows how big the user's vector will be. + # We can also take this time to precompile the user's step function so it doesn't happen + # in serial on the coarse grid. + =# + GC.enable(false) # disable garbage collection + app_ptr = pointer_from_objref(app)::Ptr{Cvoid} + pp = malloc_null_double_ptr(Cvoid) + + _jl_init!(app_ptr, 0., pp) + u_ptr = unsafe_load(pp) + u = unsafe_pointer_to_objref(u_ptr)::BraidVector + VecType = typeof(u.user_vector) + AppType = typeof(app.user_app) + + !precompile(app.step, (AppType, Ptr{Cvoid}, VecType, VecType, Float64, Float64)) && println("failed to precompile step") + !precompile(app.spatialnorm, (AppType, VecType)) && println("failed to precompile norm") + !precompile(app.sum, (AppType, Float64, VecType, Float64, VecType)) && println("failed to precompile sum") + if app.access !== nothing + !precompile(app.access, (AppType, Ptr{Cvoid}, VecType)) && println("failed to precompile access") + end + + # some julia functions that can be precompiled + precompile(deepcopy, (BraidVector{VecType},)) + precompile(unsafe_store!, (Ptr{Float64}, Float64,)) + precompile(Tuple{typeof(Base.getproperty), BraidVector{VecType}, Symbol}) + precompile(Tuple{typeof(Base.unsafe_store!), Ptr{Int32}, Int64}) + precompile(Tuple{typeof(deserialize), Serializer{Base.GenericIOBuffer{Array{UInt8, 1}}}, DataType}) + precompile(Tuple{typeof(Base.unsafe_wrap), Type{Array{UInt8, 1}}, Ptr{UInt8}, Int64}) + precompile(Tuple{Type{NamedTuple{(:read, :write, :maxsize), T} where T<:Tuple}, Tuple{Bool, Bool, Int64}}) + + _jl_free!(app_ptr, u_ptr) + Base.Libc.free(pp) + GC.enable(true) # re-enable garbage collection + + app.user_AppType = AppType + app.user_VecType = VecType +end + +function deltaPrecompile(app::BraidApp) + GC.enable(false) + app_ptr = pointer_from_objref(app)::Ptr{Cvoid} + pp = malloc_null_double_ptr(Cvoid) + AppType = app.user_AppType + VecType = app.user_VecType + + _jl_init_basis!(app_ptr, 0., Int32(0), pp) + ψ_ptr = unsafe_load(pp) + ψ = unsafe_pointer_to_objref(ψ_ptr) + BasType = typeof(ψ.user_vector) + println("precompiling user Delta functions") + !precompile(app.step, (AppType, Ptr{Cvoid}, VecType, VecType, Float64, Float64, Vector{BasType})) && println("failed to compile step_du") + !precompile(app.inner_prod, (AppType, VecType, VecType)) && println("failed to compile inner_prod u⋅u") + !precompile(app.inner_prod, (AppType, BasType, VecType)) && println("failed to compile inner_prod ψ⋅u") + !precompile(app.inner_prod, (AppType, VecType, BasType)) && println("failed to compile inner_prod u⋅ψ") + !precompile(app.inner_prod, (AppType, BasType, BasType)) && println("failed to compile inner_prod ψ⋅ψ") + + _jl_free!(app_ptr, ψ_ptr) + Base.Libc.free(pp) + GC.enable(true) + + app.user_BasType = BasType +end + +""" +Create a new BraidCore object. The BraidCore object is used to run the XBraid solver, and is destroyed when the object is garbage collected. +""" +function Init(comm_world::MPI.Comm, comm_t::MPI.Comm, + tstart::Real, tstop::Real, ntime::Integer, + step::Function, init::Function, sum::Function, spatialnorm::Function, access::OptionalFunction; + app = nothing +)::BraidCore + _app = BraidApp(app, comm_world, comm_t, step, init, sum, spatialnorm, access) + _core_ptr = malloc_null_double_ptr(Cvoid) + + GC.@preserve _core_ptr begin + @ccall libbraid.braid_Init( + _app.comm::MPI.MPI_Comm, _app.comm_t::MPI.MPI_Comm, + tstart::Cdouble, tstop::Cdouble, ntime::Cint, + _app::Ref{BraidApp}, _c_step::Ptr{Cvoid}, _c_init::Ptr{Cvoid}, _c_clone::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, + _core_ptr::Ptr{Ptr{Cvoid}}, + )::Cint + end + + _core = unsafe_load(_core_ptr) + Base.Libc.free(_core_ptr) + + return BraidCore(_core, _app, tstart, tstop, ntime) +end + +function Init(comm_world::MPI.Comm, tstart::Real, tstop::Real, ntime::Integer, step::Function, init::Function, access::OptionalFunction; app=nothing) + Init(comm_world, comm_world, tstart, tstop, ntime, step, init, default_sum!, default_norm, access; app=app) +end + +""" +Warmup the BraidCore object. XBraid.Drive calls this function automatically, but it can be called manually to precompile all user functions. + +This function is not necessary for XBraid to run, but it can significantly reduce the time to run the first time step. Note that this function calls all user functions, so if any user functions have side-effects, this may behave unexpectedly + +julia> XBraid.Warmup(core) + +Julia> XBraid.Drive(core; warmup=false) + +See also: XBraid.Drive +""" +function Warmup(core::BraidCore) + _app = core._braid_app + # precompile all user functions by calling them from braid_Warmup + # if any user functions have side-effects, this may behave unexpectedly + fdt = (core.tstop - core.tstart) / (core.ntime+1) + cdt = 2fdt + + if (_app.basis_init !== nothing) && (_app.inner_prod !== nothing) + @ccall libbraid.braid_Warmup( + _app::Ref{BraidApp}, _app.comm::MPI.MPI_Comm, core.tstart::Cdouble, fdt::Cdouble, cdt::Cdouble, + _c_init::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, + C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid}, _c_step::Ptr{Cvoid}, + _c_init_basis::Ptr{Cvoid}, _c_inner_prod::Ptr{Cvoid} + )::Cint + else + @ccall libbraid.braid_Warmup( + _app::Ref{BraidApp}, _app.comm::MPI.MPI_Comm, core.tstart::Cdouble, fdt::Cdouble, cdt::Cdouble, + _c_init::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, + C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid}, _c_step::Ptr{Cvoid}, + C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid} + )::Cint + end + nothing +end + +""" +Wraps the XBraid braid_Drive function. This function is called by XBraid.Drive, and should not be called directly. +""" +function _Drive(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_Drive(core._braid_core::Ptr{Cvoid})::Cint + end +end + +""" +Run the XBraid solver. This function calls XBraid.Warmup by default, but this can be disabled by setting warmup=false, in which case a less expensive (but less effective) option is used to precompile the user functions. +""" +function Drive(core::BraidCore; warmup=true) + if warmup + Warmup(core) + else + _app = core._braid_app + # cheaper precompile option, but less effective + begin + postInitPrecompile(_app) + if (_app.basis_init !== nothing) && (_app.inner_prod !== nothing) + deltaPrecompile(_app) + end + end + end + _Drive(core) +end + +function PrintStats(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_PrintStats(core._braid_core::Ptr{Cvoid})::Cint + end +end + +function SetTimerFile(core::BraidCore, filestem::String) + len = @ccall strlen(filestem::Cstring)::Cint + GC.@preserve core begin + @ccall libbraid.braid_SetTimerFile(core._braid_core::Ptr{Cvoid}, len::Cint, filestem::Cstring)::Cint + end +end + +function PrintTimers(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_PrintTimers(core._braid_core::Ptr{Cvoid})::Cint + end +end + +function ResetTimer(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_ResetTimer(core._braid_core::Ptr{Cvoid})::Cint + end +end + +function SetTimings(core::BraidCore, timing_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetTimings(core._braid_core::Ptr{Cvoid}, timing_level::Cint)::Cint + end +end + +function WriteConvHistory(core::BraidCore, filename::String) + GC.@preserve core begin + @ccall libbraid.braid_WriteConvHistory(core._braid_core::Ptr{Cvoid}, filename::Cstring)::Cint + end +end + +function SetMaxLevels(core::BraidCore, max_levels::Integer) + @assert max_levels > 0 "Max. levels must be an integer greater than 0" + GC.@preserve core begin + @ccall libbraid.braid_SetMaxLevels(core._braid_core::Ptr{Cvoid}, max_levels::Cint)::Cint + end +end + +function SetIncrMaxLevels(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetIncrMaxLevels(core._braid_core::Ptr{Cvoid})::Cint + end +end + +function SetSkip(core::BraidCore, skip::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetSkip(core._braid_core::Ptr{Cvoid}, skip::Cint)::Cint + end +end + + +function SetRefine(core::BraidCore, refine::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetSkip(core._braid_core::Ptr{Cvoid}, refine::Cint)::Cint + end +end + +function SetMaxRefinements(core::BraidCore, max_refinements::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetMaxRefinements(core._braid_core::Ptr{Cvoid}, max_refinements::Cint)::Cint + end +end + + +function SetTPointsCutoff(core::BraidCore, tpoints_cutoff::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetTPointsCutoff(core._braid_core::Ptr{Cvoid}, tpoints_cutoff::Cint)::Cint + end +end + + +function SetMinCoarse(core::BraidCore, min_coarse::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetMinCoarse(core._braid_core::Ptr{Cvoid}, min_coarse::Cint)::Cint + end +end + +function SetRelaxOnlyCG(core::BraidCore, relax_only_cg::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetRelaxOnlyCG(core._braid_core::Ptr{Cvoid}, relax_only_cg::Cint)::Cint + end +end + +function SetAbsTol(core::BraidCore, atol::Real) + GC.@preserve core begin + @ccall libbraid.braid_SetAbsTol(core._braid_core::Ptr{Cvoid}, atol::Cdouble)::Cint + end +end + +function SetRelTol(core::BraidCore, rtol::Real) + GC.@preserve core begin + @ccall libbraid.braid_SetRelTol(core._braid_core::Ptr{Cvoid}, rtol::Cdouble)::Cint + end +end + +function SetNRelax(core::BraidCore, level::Integer, nrelax::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetNRelax(core._braid_core::Ptr{Cvoid}, level::Cint, nrelax::Cint)::Cint + end +end + +function SetCRelaxWt(core::BraidCore, level::Integer, Cwt::Real) + GC.@preserve core begin + @ccall libbraid.braid_SetCRelaxWt(core._braid_core::Ptr{Cvoid}, level::Cint, Cwt::Cdouble)::Cint + end +end + +function SetCFactor(core::BraidCore, level::Integer, cfactor::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetCFactor(core._braid_core::Ptr{Cvoid}, level::Cint, cfactor::Cint)::Cint + end +end + +function SetMaxIter(core::BraidCore, max_iter::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetMaxIter(core._braid_core::Ptr{Cvoid}, max_iter::Cint)::Cint + end +end + +function SetFMG(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetFMG(core._braid_core::Ptr{Cvoid})::Cint + end +end + +function SetNFMG(core::BraidCore, k::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetNFMG(core._braid_core::Ptr{Cvoid}, k::Cint)::Cint + end +end + +function SetNFMGVcyc(core::BraidCore, nfmg_Vcyc::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetNFMGVcyc(core._braid_core::Ptr{Cvoid}, nfmg_Vcyc::Cint)::Cint + end +end + +function SetStorage(core::BraidCore, storage::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetStorage(core._braid_core::Ptr{Cvoid}, storage::Cint)::Cint + end +end + +function SetTemporalNorm(core::BraidCore, tnorm::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetTemporalNorm(core._braid_core::Ptr{Cvoid}, tnorm::Cint)::Cint + end +end + +function SetPeriodic(core::BraidCore, periodic::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetPeriodic(core._braid_core::Ptr{Cvoid}, periodic::Cint)::Cint + end +end + +function SetPrintLevel(core::BraidCore, print_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetPrintLevel(core._braid_core::Ptr{Cvoid}, print_level::Cint)::Cint + end +end + +function SetFileIOLevel(core::BraidCore, io_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetFileIOLevel(core._braid_core::Ptr{Cvoid}, io_level::Cint)::Cint + end +end + +function SetPrintFile(core::BraidCore, filename::String) + GC.@preserve core begin + @ccall libbraid.braid_SetPrintFile(core._braid_core::Ptr{Cvoid}, filename::Cstring)::Cint + end +end + +function SetDefaultPrintFile(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetDefaultPrintFile(core._braid_core::Ptr{Cvoid})::Cint + end +end + +""" +Set access level for XBraid. This controls how often the user's access function is called. + + - 0: Never call access function + - 1: Call access function only when XBraid is finished + - 2: Call access function at every XBraid iteration, on every level + +Default is 1. +""" +function SetAccessLevel(core::BraidCore, access_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetAccessLevel(core._braid_core::Ptr{Cvoid}, access_level::Cint)::Cint + end +end + +function SetFinalFCRelax(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetFinalFCRelax(core._braid_core::Ptr{Cvoid})::Cint + end +end + +function GetNumIter(core::BraidCore) + niter = Ref{Cint}(0) + GC.@preserve core begin + @ccall libbraid.braid_GetNumIter(core._braid_core::Ptr{Cvoid}, niter::Ref{Cint})::Cint + end + return niter[] +end + +function GetRNorms(core::BraidCore) + nrequest = Ref{Cint}() + nrequest[] = GetNumIter(core) + rnorms = zeros(nrequest[]) + GC.@preserve core begin + @ccall libbraid.braid_GetRNorms(core._braid_core::Ptr{Cvoid}, nrequest::Ref{Cint}, rnorms::Ref{Cdouble})::Cint + end + nrequest[] == 0 && return Float64[] + return rnorms[1:nrequest[]] +end + +function GetNLevels(core::BraidCore) + nlevels = Ref(0) + GC.@preserve core begin + @ccall libbraid.braid_GetNLevels(core._braid_core::Ptr{Cvoid}, nlevels::Ref{Cint})::Cint + end + return nlevels[] +end + +function SetSeqSoln(core::BraidCore, seq_soln::Bool) + GC.@preserve core begin + @ccall libbraid.braid_GetNLevels(core._braid_core::Ptr{Cvoid}, seq_soln::Cint)::Cint + end +end + +function SetRichardsonEstimation(core::BraidCore, est_error::Bool, richardson::Bool, local_order::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetRichardsonEstimation(core._braid_core::Ptr{Cvoid}, est_error::Cint, richardson::Cint, local_order::Cint)::Cint + end +end + +function SetDeltaCorrection(core::BraidCore, rank::Integer, basis_init::Function, inner_prod::Function) + core._braid_app.basis_init = basis_init + core._braid_app.inner_prod = inner_prod + # deltaPrecompile(core._braid_app) + + GC.@preserve core begin + @ccall libbraid.braid_SetDeltaCorrection(core._braid_core::Ptr{Cvoid}, rank::Cint, _c_init_basis::Ptr{Cvoid}, _c_inner_prod::Ptr{Cvoid})::Cint + end +end + +function SetDeferDelta(core::BraidCore, level::Integer, iter::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetDeferDelta(core._braid_core::Ptr{Cvoid}, level::Cint, iter::Cint)::Cint + end +end + +function SetLyapunovEstimation(core::BraidCore; relax::Bool = false, cglv::Bool = true, exponents::Bool = false) + GC.@preserve core begin + @ccall libbraid.braid_SetLyapunovEstimation(core._braid_core::Ptr{Cvoid}, relax::Cint, cglv::Cint, exponents::Cint)::Cint + end +end + +# Still missing spatial coarsening, sync, residual, and adjoint + +# braid_test +function testInitAccess(app::BraidApp, t::Real, outputFile::Libc.FILE) + pass = @ccall libbraid.braid_TestInitAccess( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + )::Cint + return pass > 0 + + # println("Serialized size of user vector: $(app.bufsize)") + # println("Check output for objects not properly freed:") + # println(app.ref_ids) +end +testInitAccess(app::BraidApp, t::Real, outputFile::IO) = testInitAccess(app, t, Libc.FILE(outputFile)) +testInitAccess(app::BraidApp, t::Real) = testInitAccess(app, t, c_stdout) + +function testClone(app::BraidApp, t::Real, outputFile::Libc.FILE) + pass = @ccall libbraid.braid_TestClone( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + )::Cint + return pass > 0 +end +testClone(app::BraidApp, t::Real, outputFile::IO) = testClone(app, t, Libc.FILE(outputFile)) +testClone(app::BraidApp, t::Real) = testClone(app, t, c_stdout) + +function testSum(app::BraidApp, t::Real, outputFile::Libc.FILE) + pass = @ccall libbraid.braid_TestSum( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + )::Cint + return pass > 0 +end +testSum(app::BraidApp, t::Real, outputFile::IO) = testSum(app, t, Libc.FILE(outputFile)) +testSum(app::BraidApp, t::Real) = testSum(app, t, c_stdout) + +function testSpatialNorm(app::BraidApp, t::Real, outputFile::Libc.FILE) + pass = @ccall libbraid.braid_TestSpatialNorm( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + _c_norm::Ptr{Cvoid}, + )::Cint + return pass > 0 +end +testSpatialNorm(app::BraidApp, t::Real, outputFile::IO) = testSpatialNorm(app, t, Libc.FILE(outputFile)) +testSpatialNorm(app::BraidApp, t::Real) = testSpatialNorm(app, t, c_stdout) + +function testBuf(app::BraidApp, t::Real, outputFile::Libc.FILE) + pass = @ccall libbraid.braid_TestBuf( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + _c_norm::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, + _c_bufpack::Ptr{Cvoid}, + _c_bufunpack::Ptr{Cvoid}, + )::Cint + return pass > 0 + # print('\n') + # println("Check output for objects not freed:") + # println(app.ref_ids) +end +testBuf(app::BraidApp, t::Real, outputFile::IO) = testBuf(app, t, Libc.FILE(outputFile)) +testBuf(app::BraidApp, t::Real) = testBuf(app, t, c_stdout) + +function testDelta(app::BraidApp, t::Real, dt::Real, rank::Integer, outputFile::Libc.FILE) + app.basis_init::Function + app.inner_prod::Function + pass = @ccall libbraid.braid_TestDelta( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + dt::Cdouble, + rank::Cint, + _c_init::Ptr{Cvoid}, + _c_init_basis::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, + _c_bufpack::Ptr{Cvoid}, + _c_bufunpack::Ptr{Cvoid}, + _c_inner_prod::Ptr{Cvoid}, + _c_step::Ptr{Cvoid}, + )::Cint + return pass > 0 + # println("Check output for objects not freed:") + # println(app.ref_ids) +end +testDelta(app::BraidApp, t::Real, dt::Real, rank::Integer, outputFile::IO) = testDelta(app, t, dt, rank, Libc.FILE(outputFile)) +testDelta(app::BraidApp, t::Real, dt::Real, rank::Integer) = testDelta(app, t, dt, rank, c_stdout) + +# END MODULE XBraid +end diff --git a/braid/braid.jl/status.jl b/braid/braid.jl/status.jl new file mode 100644 index 00000000..76a974b2 --- /dev/null +++ b/braid/braid.jl/status.jl @@ -0,0 +1,173 @@ +#=BHEADER********************************************************************** + * Copyright (c) 2013, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * This file is part of XBraid. For support, post issues to the XBraid Github page. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License (as published by the Free Software + * Foundation) version 2.1 dated February 1999. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the terms and conditions of the GNU General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + ***********************************************************************EHEADER=# + +# should these automatically qeuery the braid status for some commonly used values? +struct StepStatus + ptr::Ptr{Cvoid} +end + +struct AccessStatus + ptr::Ptr{Cvoid} +end + +# @ccall requires these type annotations to be known at compile time, so macros won't work here +function status_GetT(status::Union{StepStatus, AccessStatus}) + t = Ref{Cdouble}() + @ccall libbraid.braid_StatusGetT(status.ptr::Ptr{Cvoid}, t::Ref{Cdouble})::Cint + return t[] +end + +function status_GetTStop(status::StepStatus) + tstop = Ref{Cdouble}() + @ccall libbraid.braid_StatusGetTStop(status.ptr::Ptr{Cvoid}, tstop::Ref{Cdouble})::Cint + return tstop[] +end + +function status_GetTstartTstop(status::StepStatus) + tstart, tstop = Ref{Cdouble}(), Ref{Cdouble}() + @ccall libbraid.braid_StepStatusGetTstartTstop(status.ptr::Ptr{Cvoid}, tstart::Ref{Cdouble}, tstop::Ref{Cdouble})::Cint + return tstart[], tstop[] +end + +function status_GetTIndex(status::Union{StepStatus, AccessStatus}) + ti = Ref{Cint}() + @ccall libbraid.braid_StatusGetTIndex(status.ptr::Ptr{Cvoid}, ti::Ref{Cint})::Cint + return ti[] +end + +function status_GetLevel(status::Union{StepStatus, AccessStatus}) + level = Ref{Cint}() + @ccall libbraid.braid_StatusGetLevel(status.ptr::Ptr{Cvoid}, level::Ref{Cint})::Cint + return level[] +end + +function status_GetWrapperTest(status::Union{StepStatus, AccessStatus}) + wtest = Ref{Cint}() + @ccall libbraid.braid_StatusGetWrapperTest(status.ptr::Ptr{Cvoid}, wtest::Ref{Cint})::Cint + return (wtest[] > 0) +end + +function status_GetIter(status::Union{StepStatus, AccessStatus}) + iter = Ref{Cint}() + @ccall libbraid.braid_StatusGetIter(status.ptr::Ptr{Cvoid}, iter::Ref{Cint})::Cint + return iter[] +end + +function status_GetNLevels(status::Union{StepStatus, AccessStatus}) + nlevels = Ref{Cint}() + @ccall libbraid.braid_StatusGetNLevels(status.ptr::Ptr{Cvoid}, nlevels::Ref{Cint})::Cint + return nlevels[] +end + +function status_GetNTPoints(status::Union{StepStatus, AccessStatus}) + ntpoints = Ref{Cint}() + @ccall libbraid.braid_StatusGetNTPoints(status.ptr::Ptr{Cvoid}, ntpoints::Ref{Cint})::Cint + return ntpoints[] +end + +function status_GetResidual(status::Union{StepStatus, AccessStatus}) + res = Ref{Cdouble}() + @ccall libbraid.braid_StatusGetResidual(status.ptr::Ptr{Cvoid}, res::Ref{Cdouble})::Cint + return res[] +end + +function status_GetDone(status::Union{StepStatus, AccessStatus}) + done = Ref{Cint}() + @ccall libbraid.braid_StatusGetDone(status.ptr::Ptr{Cvoid}, done::Ref{Cint})::Cint + return (done[] > 0) +end + +function status_GetTILD(status::Union{StepStatus, AccessStatus}) + time = Ref{Cdouble}() + iter = Ref{Cint}() + level = Ref{Cint}() + done = Ref{Cint}() + @ccall libbraid.braid_StatusGetTILD(status.ptr::Ptr{Cvoid}, time::Ref{Cdouble}, iter::Ref{Cint}, level::Ref{Cint}, done::Ref{Cint})::Cint + return time[], iter[], level[], (done[] > 0) +end + +function status_GetCallingFunction(status::Union{StepStatus, AccessStatus}) + func = Ref{Cint}() + @ccall libbraid.braid_StatusGetCallingFunction(status.ptr::Ptr{Cvoid}, func::Ref{Cint})::Cint + return func[] +end + +function status_GetTol(status::StepStatus) + tol = Ref{Cdouble}() + @ccall libbraid.braid_StatusGetTol(status.ptr::Ptr{Cvoid}, tol::Ref{Cdouble})::Cint + return tol[] +end + +function status_GetRNorms(status::StepStatus) + iters = status_GetIter(status) + nrequest = Ref{Cint}(iters) + norms = zeros(Cdouble, iters) + @ccall libbraid.braid_StatusGetRNorms(status.ptr::Ptr{Cvoid}, nrequest::Ref{Cint}, norms::Ref{Cdouble})::Cint + return norms +end + +function status_GetProc(status::Union{StepStatus, AccessStatus}) + proc = Ref{Cint}() + @ccall libbraid.braid_StatusGetProc(status.ptr::Ptr{Cvoid}, proc::Ref{Cint})::Cint + return proc[] +end + +function status_GetDeltaRank(status::Union{StepStatus, AccessStatus}) + rank = Ref{Cint}() + @ccall libbraid.braid_StatusGetDeltaRank(status.ptr::Ptr{Cvoid}, rank::Ref{Cint})::Cint + return rank[] +end + +# These are special cases +function status_GetLocalLyapExponents(status::AccessStatus) + rank = status_GetDeltaRank(status) + rank < 1 && return [] + + exps = zeros(rank) + num_retrieved = Ref(rank) + @ccall libbraid.braid_StatusGetLocalLyapExponents(status.ptr::Ptr{Cvoid}, exps::Ref{Cdouble}, num_retrieved::Ref{Cint})::Cint + return exps +end + +function status_GetBasisVectors(status::Union{StepStatus, AccessStatus}) + rank = status_GetDeltaRank(status) + Ψ = [] + rank < 1 && return Ψ + + for i in 1:rank + pp = malloc_null_double_ptr(Cvoid) + GC.@preserve pp begin + @ccall libbraid.braid_StatusGetBasisVec(status.ptr::Ptr{Cvoid}, pp::Ptr{Ptr{Cvoid}}, (i-1)::Cint)::Cint + end + p = unsafe_load(pp) + if p !== C_NULL + φ = unsafe_pointer_to_objref(p)::BraidVector + push!(Ψ, φ.user_vector) + end + Base.Libc.free(pp) + end + + return Ψ +end + +function status_SetRFactor(status::StepStatus, rfactor::Real) + @ccall libbraid.braid_StatusSetRFactor(status.ptr::Ptr{Cvoid}, rfactor::Cdouble)::Cint +end diff --git a/braid/braid.jl/wrapper_functions.jl b/braid/braid.jl/wrapper_functions.jl new file mode 100644 index 00000000..60acf5ca --- /dev/null +++ b/braid/braid.jl/wrapper_functions.jl @@ -0,0 +1,310 @@ +#=BHEADER********************************************************************** + * Copyright (c) 2013, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * This file is part of XBraid. For support, post issues to the XBraid Github page. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License (as published by the Free Software + * Foundation) version 2.1 dated February 1999. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the terms and conditions of the GNU General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + ***********************************************************************EHEADER=# + +""" +Displays a caught error message, including stacktrace, without throwing +""" +function stacktrace_warn(msg::String, err) + err_msg = sprint(showerror, err) + trace = sprint((io,v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) + @warn "$(msg):\n$(err_msg)\n$(trace)" +end + +""" +Where'd all my data go? +This extends Base to include a buffer which throws away the data written to it +(useful for measuring the serialized size of an object) +""" +mutable struct BlackHoleBuffer <: IO + ptr::Int +end +BlackHoleBuffer() = BlackHoleBuffer(0) + +function Base.read(from::BlackHoleBuffer, T::Type{UInt8}) + throw(ArgumentError("BlackHoleBuffer is not readable)")) +end +function Base.write(to::BlackHoleBuffer, x::UInt8) + to.ptr += 1 + return 1 +end +function Base.write(to::BlackHoleBuffer, x::Array{T}) where T + to.ptr += sizeof(x) + return sizeof(x) +end + +# these are internal functions which directly interface with XBraid + +function _jl_step!(_app::Ptr{Cvoid}, + _ustop::Ptr{Cvoid}, + _fstop::Ptr{Cvoid}, + _u::Ptr{Cvoid}, + _status::Ptr{Cvoid})::Cint + # println("step") + app = unsafe_pointer_to_objref(_app)::BraidApp + u = unsafe_pointer_to_objref(_u)::BraidVector + ustop = unsafe_pointer_to_objref(_ustop)::BraidVector + status = StepStatus(_status) + + tstart, tstop = status_GetTstartTstop(status) + delta_rank = status_GetDeltaRank(status) + + # call the user's function + try + # residual option + if _fstop !== C_NULL + fstop = unsafe_pointer_to_objref(_fstop)::BraidVector + app.step(app.user_app, status, u.user_vector, ustop.user_vector, fstop.user_vector, tstart, tstop) + # Delta correction + elseif delta_rank > 0 + basis_vecs = status_GetBasisVectors(status) + app.step(app.user_app, status, u.user_vector, ustop.user_vector, tstart, tstop, basis_vecs) + # Default + else + app.step(app.user_app, status, u.user_vector, ustop.user_vector, tstart, tstop) + end + catch err + stacktrace_warn("Error in user Step", err) + end + + return 0 +end +precompile(_jl_step!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) +_c_step = @cfunction(_jl_step!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) + + +function _jl_init!(_app::Ptr{Cvoid}, t::Cdouble, u_ptr::Ptr{Ptr{Cvoid}})::Cint + # println("init") + app = unsafe_pointer_to_objref(_app)::BraidApp + # initialize u and register a reference with IdDict + u = nothing + try + u = BraidVector(app.init(app.user_app, t)) + catch err + stacktrace_warn("Error in user Init", err) + return 1 + end + _register_vector(app, u) + + unsafe_store!(u_ptr, pointer_from_objref(u)) + + # store max size of all initialized vectors + if app.bufsize == 0 + # This serializes u but doesn't actually + # store the data + buffer = BlackHoleBuffer() + serialize(buffer, u) + app.bufsize = buffer.ptr + sizeof(Int) + end + + if app.user_VecType == Nothing + app.user_VecType = u.VecType + end + + # TODO: figure out how to put an upper bound on the size of the serialized object + # without serializing it first + # u_size = Base.summarysize(Ref(u)) + 9 + # if u_size > app.bufsize + # app.bufsize = u_size + # end + + return 0 +end +_c_init = @cfunction(_jl_init!, Cint, (Ptr{Cvoid}, Cdouble, Ptr{Ptr{Cvoid}})) + +function _jl_init_basis!(_app::Ptr{Cvoid}, t::Cdouble, index::Cint, u_ptr::Ptr{Ptr{Cvoid}})::Cint + # println("init_basis") + app = unsafe_pointer_to_objref(_app)::BraidApp + u = nothing + try + u = BraidVector(app.basis_init(app.user_app, t, index)) + catch err + stacktrace_warn("Error in user InitBasis", err) + return 1 + end + + _register_vector(app, u) + unsafe_store!(u_ptr, pointer_from_objref(u)) + + # store max size of all initialized vectors + if app.bufsize == 0 + buffer = BlackHoleBuffer() + serialize(buffer, u) + app.bufsize_lyap = buffer.ptr + sizeof(Int) + end + + # store type of initialized vector + if app.user_BasType == Nothing + app.user_BasType = u.VecType + end + + return 0 +end +_c_init_basis = @cfunction(_jl_init_basis!, Cint, (Ptr{Cvoid}, Cdouble, Cint, Ptr{Ptr{Cvoid}})) + +function _jl_clone!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, v_ptr::Ptr{Ptr{Cvoid}})::Cint + # println("clone") + app = unsafe_pointer_to_objref(_app)::BraidApp + u = unsafe_pointer_to_objref(_u)::BraidVector + # initialize v, and copy u into v + v = nothing + try + v = deepcopy(u) + catch err + stacktrace_warn("Clone error", err) + return 1 + end + + # then register v with IdDict and store in v_ptr + _register_vector(app, v) + unsafe_store!(v_ptr, pointer_from_objref(v)) + + return 0 +end +precompile(_jl_clone!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}})) +_c_clone = @cfunction(_jl_clone!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}})) + +function _jl_free!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid})::Cint + # println("free") + app = unsafe_pointer_to_objref(_app)::BraidApp + u = unsafe_pointer_to_objref(_u)::BraidVector + # removing the global reference to u will cause it to be garbage collected + _deregister_vector(app, u) + return 0 +end +precompile(_jl_free!, (Ptr{Cvoid}, Ptr{Cvoid})) +_c_free = @cfunction(_jl_free!, Cint, (Ptr{Cvoid}, Ptr{Cvoid})) + +function _jl_sum!(_app::Ptr{Cvoid}, + alpha::Cdouble, _x::Ptr{Cvoid}, + beta::Cdouble, _y::Ptr{Cvoid})::Cint + app = unsafe_pointer_to_objref(_app)::BraidApp + x = unsafe_pointer_to_objref(_x)::BraidVector + y = unsafe_pointer_to_objref(_y)::BraidVector + try + app.sum(app.user_app, alpha, x.user_vector, beta, y.user_vector) + catch err + stacktrace_warn("Error in user Sum", err) + end + return 0 +end +precompile(_jl_sum!, (Ptr{Cvoid}, Cdouble, Ptr{Cvoid}, Cdouble, Ptr{Cvoid})) +_c_sum = @cfunction(_jl_sum!, Cint, (Ptr{Cvoid}, Cdouble, Ptr{Cvoid}, Cdouble, Ptr{Cvoid})) + +function _jl_norm!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, norm_ptr::Ptr{Cdouble})::Cint + app = unsafe_pointer_to_objref(_app)::BraidApp + u = unsafe_pointer_to_objref(_u)::BraidVector + norm = NaN + try + norm = app.spatialnorm(app.user_app, u.user_vector) + catch err + stacktrace_warn("Error in user Norm", err) + end + unsafe_store!(norm_ptr, norm) + + return 0 +end +precompile(_jl_norm!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) +_c_norm = @cfunction(_jl_norm!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) + +function _jl_inner_prod!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _v::Ptr{Cvoid}, norm_ptr::Ptr{Cdouble})::Cint + app = unsafe_pointer_to_objref(_app)::BraidApp + u = unsafe_pointer_to_objref(_u)::BraidVector + v = unsafe_pointer_to_objref(_v)::BraidVector + prod = NaN + try + prod = app.inner_prod(app.user_app, u.user_vector, v.user_vector) + catch err + stacktrace_warn("Error in user InnerProd", err) + end + unsafe_store!(norm_ptr, prod) + + return 0 +end +precompile(_jl_inner_prod!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) +_c_inner_prod = @cfunction(_jl_inner_prod!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) + +function _jl_access!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _status::Ptr{Cvoid})::Cint + app = unsafe_pointer_to_objref(_app)::BraidApp + status = AccessStatus(_status) + + if !isnothing(app.access) + u = unsafe_pointer_to_objref(_u)::BraidVector + try + app.access(app.user_app, status, u.user_vector) + catch err + stacktrace_warn("Error in user Access", err) + end + end + + return 0 +end +precompile(_jl_access!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) +_c_access = @cfunction(_jl_access!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) + +function _jl_bufsize!(_app::Ptr{Cvoid}, size_ptr::Ptr{Cint}, status::Ptr{Cvoid})::Cint + app = unsafe_pointer_to_objref(_app) + unsafe_store!(size_ptr, app.bufsize) + return 0 +end +_c_bufsize = @cfunction(_jl_bufsize!, Cint, (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cvoid})) + +function _jl_bufpack!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, status::Ptr{Cvoid})::Cint + app = unsafe_pointer_to_objref(_app)::BraidApp + u = unsafe_pointer_to_objref(_u)::BraidVector + buff_arr = unsafe_wrap(Vector{UInt8}, Base.unsafe_convert(Ptr{UInt8}, _buffer), app.bufsize) + buffer = IOBuffer(buff_arr, read=true, write=true, maxsize=app.bufsize) + + # store u in buffer + seek(buffer, sizeof(Int)) + serialize(buffer, u) + + # also store the total size of the buffer + seek(buffer, 0) + write(buffer, buffer.size) + + # tell braid the written size + @ccall libbraid.braid_BufferStatusSetSize(status::Ptr{Cvoid}, buffer.size::Cdouble)::Cint + + return 0 +end +precompile(_jl_bufpack!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) +_c_bufpack = @cfunction(_jl_bufpack!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) + +function _jl_bufunpack!(_app::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, u_ptr::Ptr{Ptr{Cvoid}}, status::Ptr{Cvoid})::Cint + @assert _buffer !== C_NULL "tried to unpack null buffer" + app = unsafe_pointer_to_objref(_app)::BraidApp + # get size of buffer we are unpacking: + header = reinterpret(Ptr{Int}, _buffer) + bufsize = unsafe_load(header)::Int + buff_arr = unsafe_wrap(Vector{UInt8}, Base.unsafe_convert(Ptr{UInt8}, _buffer), bufsize) + buffer = IOBuffer(buff_arr, read=true, write=true, maxsize=bufsize) + seek(buffer, sizeof(Int)) + + # unpack the buffer into a new julia object, then register with IdDict + u = deserialize(buffer)::BraidVector{app.user_VecType} + _register_vector(app, u) + + # store u in provided pointer + unsafe_store!(u_ptr, pointer_from_objref(u)) + return 0 +end +precompile(_jl_bufunpack!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}}, Ptr{Cvoid})) +_c_bufunpack = @cfunction(_jl_bufunpack!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}}, Ptr{Cvoid})) diff --git a/braid/braid_test.c b/braid/braid_test.c index d15c6a4a..ac44a60c 100644 --- a/braid/braid_test.c +++ b/braid/braid_test.c @@ -39,6 +39,10 @@ braid_TestInitAccess( braid_App app, braid_PtFcnAccess myaccess, braid_PtFcnFree myfree) { + if (!fp) + { + fp = stdout; + } braid_Vector u ; braid_Status status = _braid_CTAlloc(_braid_Status, 1); @@ -84,6 +88,10 @@ braid_TestClone( braid_App app, braid_PtFcnFree myfree, braid_PtFcnClone clone) { + if (!fp) + { + fp = stdout; + } braid_Vector u, v; braid_Status status = _braid_CTAlloc(_braid_Status, 1); @@ -143,6 +151,10 @@ braid_TestSum( braid_App app, braid_PtFcnClone clone, braid_PtFcnSum sum ) { + if (!fp) + { + fp = stdout; + } braid_Vector u, v; braid_Status status = _braid_CTAlloc(_braid_Status, 1); @@ -215,6 +227,10 @@ braid_TestSpatialNorm( braid_App app, braid_PtFcnSum sum, braid_PtFcnSpatialNorm spatialnorm) { + if (!fp) + { + fp = stdout; + } braid_Vector u, v, w; braid_Real result1, result2; @@ -377,6 +393,10 @@ braid_TestInnerProd( braid_App app, braid_PtFcnSum sum, braid_PtFcnInnerProd inner_prod) { + if (!fp) + { + fp = stdout; + } braid_Vector u, v; braid_Real result1, result2; @@ -503,6 +523,10 @@ braid_TestBuf( braid_App app, braid_PtFcnBufPack bufpack, braid_PtFcnBufUnpack bufunpack) { + if (!fp) + { + fp = stdout; + } braid_Vector u, v; braid_Real result1; @@ -602,6 +626,10 @@ braid_TestCoarsenRefine( braid_App app, braid_PtFcnSCoarsen coarsen, braid_PtFcnSRefine refine) { + if (!fp) + { + fp = stdout; + } braid_Vector u, v, w, uc, vc, wc; braid_Real result1; @@ -781,6 +809,10 @@ braid_TestResidual( braid_App app, braid_PtFcnResidual residual, braid_PtFcnStep step) { + if (!fp) + { + fp = stdout; + } braid_Vector u, unext, ustop, fstop; braid_Real result1; @@ -924,6 +956,10 @@ braid_TestAll( braid_App app, braid_PtFcnStep step) { + if (!fp) + { + fp = stdout; + } braid_Int myid_x, flag = 0, correct = 1; MPI_Comm_rank( comm_x, &myid_x ); @@ -1024,9 +1060,12 @@ braid_TestDelta(braid_App app, braid_PtFcnInnerProd myinner_prod, braid_PtFcnStep mystep) { + if (!fp) + { + fp = stdout; + } - braid_Vector u; - braid_Vector v; + braid_Vector u, v, w; braid_Basis A, B; // braid_Int actual_rank = rank; braid_Status status = _braid_CTAlloc(_braid_Status, 1); @@ -1120,7 +1159,6 @@ braid_TestDelta(braid_App app, myinner_prod(app, B->userVecs[i], B->userVecs[i], &prod); if ((prod < wiggle) && (prod > -wiggle)) { - _braid_ParFprintfFlush(fp, myid_x, " braid_TestInitBasis: B has linearly dependent columns!\n"); _braid_ParFprintfFlush(fp, myid_x, " braid_TestInitBasis: Test 2 Failed\n"); correct = 0; @@ -1152,6 +1190,8 @@ braid_TestDelta(braid_App app, } _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: v = clone(u) \n"); myclone(app, u, &v); + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: w = clone(u) \n"); + myclone(app, u, &w); _braid_StepStatusInit(t, t + dt, 0, wiggle, 0, 0, 0, 0, braid_ASCaller_Residual, B, sstatus); _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: v = step(v)\n"); @@ -1159,25 +1199,34 @@ braid_TestDelta(braid_App app, mystep(app, u, NULL, v, sstatus); braid_Real eps = sqrt(wiggle); - _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: u = step(u + eps*A_0) \n"); _braid_StepStatusInit(t, t + dt, 0, wiggle, 0, 0, 0, 0, braid_ASCaller_Residual, NULL, sstatus); - mysum(app, eps, A->userVecs[0], 1., u); - mystep(app, u, NULL, u, sstatus); - - _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: v = B_0 - (u - v)/eps\n"); - _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: (v = step_dv(u)*A_0 - (step(u + eps*A_0) - step(u))/eps) \n"); - mysum(app, 1/eps, u, -1/eps, v); - mysum(app, 1., B->userVecs[0], -1., v); - _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: result = inner_prod(v, v) \n"); - braid_Real result; - myinner_prod(app, v, v, &result); - _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: actual output: result approx. %1.2e \n", result); - _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: expected output: positive, near zero \n\n"); + for (braid_Int i = 0; i < rank; i++) + { + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: for i = %d \n", i); + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: w = step(u + eps*A[i]) \n"); + + mysum(app, 1., u, 0., w); + mysum(app, eps, A->userVecs[i], 1., w); + mystep(app, w, NULL, w, sstatus); + + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: w = B[i] - (w - v)/eps\n"); + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: (w = step_dv(u)*A[i] - (step(u + eps*A[i]) - step(u))/eps) \n"); + mysum(app, -1/eps, v, 1/eps, w); + mysum(app, 1., B->userVecs[i], -1., w); + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: result = inner_prod(w, w) \n"); + braid_Real result; + myinner_prod(app, w, w, &result); + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: actual output: result approx. %1.2e \n", result); + _braid_ParFprintfFlush(fp, myid_x, " braid_TestStepDiff: expected output: positive, near zero \n\n"); + } + _braid_ParFprintfFlush(fp, myid_x, "Finished braid_TestStepDiff\n"); /* Free variables */ _braid_ParFprintfFlush(fp, myid_x, " braid_TestDelta: free(v) \n"); myfree(app, v); + _braid_ParFprintfFlush(fp, myid_x, " braid_TestDelta: free(w) \n"); + myfree(app, w); _braid_ParFprintfFlush(fp, myid_x, " braid_TestDelta: free(B) \n"); for (braid_Int i = 0; i < rank; i++) { @@ -1257,6 +1306,7 @@ braid_TestDelta(braid_App app, } _braid_ParFprintfFlush(fp, myid_x, " braid_TestBufBasis: inner_prod(v, v)\n"); + braid_Real result; myinner_prod(app, v, v, &result); if (result > wiggle || _braid_isnan(result)) { @@ -1321,4 +1371,98 @@ braid_TestDelta(braid_App app, _braid_ParFprintfFlush(fp, myid_x, "Finished braid_TestDelta\n"); return correct; +} + +braid_Int +braid_Warmup(braid_App app, + MPI_Comm comm_x, + braid_Real t, + braid_Real fdt, + braid_Real cdt, + braid_PtFcnInit init, + braid_PtFcnAccess access, + braid_PtFcnFree myfree, + braid_PtFcnClone clone, + braid_PtFcnSum sum, + braid_PtFcnSpatialNorm spatialnorm, + braid_PtFcnBufSize bufsize, + braid_PtFcnBufPack bufpack, + braid_PtFcnBufUnpack bufunpack, + braid_PtFcnSCoarsen coarsen, + braid_PtFcnSRefine refine, + braid_PtFcnStep step, + braid_PtFcnInitBasis init_basis, + braid_PtFcnInnerProd innerprod) +{ + braid_Vector u, v; + braid_Status status = _braid_CTAlloc(_braid_Status, 1); + braid_AccessStatus astatus = (braid_AccessStatus)status; + + /* init/access */ + _braid_AccessStatusInit(t, 0, 0.0, 0, 0, 0, 0, 0, 1, -1, NULL, astatus); + init(app, t, &u); + if(access != NULL) + { + access(app, u, astatus); + } + + braid_Int size; + braid_Real throwaway; + void *buffer; + + /* bufsize, bufpack, bufunpack */ + braid_BufferStatus bstatus = (braid_BufferStatus)status; + _braid_BufferStatusInit(0, 0, 0, 0, bstatus); + bufsize(app, &size, bstatus); + buffer = malloc(size); + _braid_StatusElt(bstatus, size_buffer) = size; + bufpack(app, u, buffer, bstatus); + bufunpack(app, buffer, &v, bstatus); + free(buffer); + + /* free, clone, sum, spatialnorm */ + myfree(app, v); + clone(app, u, &v); + sum(app, 1.0, u, -1.0, v); + spatialnorm(app, v, &throwaway); + + /* step */ + braid_StepStatus sstatus = (braid_StepStatus)status; + _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, NULL, sstatus); + step(app, v, NULL, u, sstatus); + + /* coarsen, refine */ + if ( (coarsen != NULL) && (refine != NULL) ) + { + braid_CoarsenRefStatus cstatus = (braid_CoarsenRefStatus)status; + + _braid_CoarsenRefStatusInit(t, t-fdt, t+fdt, t-cdt, t+cdt, 0, 0, 0, 0, cstatus); + myfree(app, v); + coarsen(app, u, &v, cstatus); + myfree(app, u); + refine(app, v, &u, cstatus); + } + + /* Delta functions: init_basis, innerprod, step with basis */ + if ( (init_basis != NULL) && (innerprod != NULL)) + { + braid_Basis B; + B = _braid_TAlloc(_braid_Basis, 1); + B->rank = 1; + B->userVecs = _braid_TAlloc(braid_Vector, 1); + init_basis(app, t, 0, &(B->userVecs[0])); + innerprod(app, u, B->userVecs[0], &throwaway); + _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, B, sstatus); + step(app, v, NULL, u, sstatus); + + myfree(app, B->userVecs[0]); + _braid_TFree(B); + } + + /* Free variables */ + myfree(app, u); + myfree(app, v); + _braid_StatusDestroy(status); + + return _braid_error_flag; } \ No newline at end of file diff --git a/braid/braid_test.h b/braid/braid_test.h index 378fd5b1..44d250fc 100644 --- a/braid/braid_test.h +++ b/braid/braid_test.h @@ -122,6 +122,32 @@ braid_TestSpatialNorm( braid_App app, /**< User defined App ); +/** + * Test the inner_prod function.\n + * A vector is initialized at time *t1*, then the vector is normalized under the + * norm induced by inner_prod. A second vector is initialized at time *t2*, and the + * Gram Schmidt process removes the component of the second vector along the + * direction of the first. The test is inconclusive unless both vectors are nonzero + * and not orthogonal. + * + * - Returns 0 if the tests fail + * - Returns 1 if the tests pass + * - Check the log messages to see details of which tests failed. + **/ +braid_Int +braid_TestInnerProd( braid_App app, /**< User defined App structure */ + MPI_Comm comm_x, /**< Spatial communicator */ + FILE *fp, /**< File pointer (could be stdout or stderr) for log messages */ + braid_Real t1, /**< Time value used to initialize the 1st vector */ + braid_Real t2, /**< Time value used to initialize the 2nd vector (t1 != t2) */ + braid_PtFcnInit init, /**< Initialize a braid_Vector on finest temporal grid */ + braid_PtFcnFree free, /**< Free a braid_Vector */ + braid_PtFcnSum sum, /**< Compute vector sum of two braid_Vectors */ + braid_PtFcnInnerProd inner_prod /**< Compute inner product of two braid_Vectors */ + ); + + + /** * Test the inner_prod function.\n * A vector is initialized at time *t1*, then the vector is normalized under the @@ -282,6 +308,32 @@ braid_TestDelta(braid_App app, /**< User defined App stru braid_PtFcnStep mystep /**< Compute a time step with a braid_Vector */ ); +/** + * Warmup calls all user functions once with sensible arguments. + * This is especially useful for the Julia wrapper, since Julia is JIT compiled, + * and this force compiles all the functions used in braid_Drive. + */ +braid_Int +braid_Warmup( braid_App app, /**< User defined App structure */ + MPI_Comm comm_x, /**< Spatial communicator */ + braid_Real t, /**< Time value to initialize test vectors with*/ + braid_Real fdt, /**< Fine time step value that you spatially coarsen from */ + braid_Real cdt, /**< Coarse time step value that you coarsen to */ + braid_PtFcnInit init, /**< Initialize a braid_Vector on finest temporal grid*/ + braid_PtFcnAccess access, /**< Allows access to XBraid and current braid_Vector (can be NULL for no writing)*/ + braid_PtFcnFree free, /**< Free a braid_Vector*/ + braid_PtFcnClone clone, /**< Clone a braid_Vector */ + braid_PtFcnSum sum, /**< Compute vector sum of two braid_Vectors */ + braid_PtFcnSpatialNorm spatialnorm, /**< Compute norm of a braid_Vector, this is a norm only over space */ + braid_PtFcnBufSize bufsize, /**< Computes size in bytes for one braid_Vector MPI buffer */ + braid_PtFcnBufPack bufpack, /**< Packs MPI buffer to contain one braid_Vector */ + braid_PtFcnBufUnpack bufunpack, /**< Unpacks MPI buffer into a braid_Vector */ + braid_PtFcnSCoarsen coarsen, /**< Spatially coarsen a vector. If NULL, skipped.*/ + braid_PtFcnSRefine refine, /**< Spatially refine a vector. If NULL, skipped.*/ + braid_PtFcnStep step, /**< Compute a time step with a braid_Vector */ + braid_PtFcnInitBasis init_basis, /**< Initialize a basis vector at time t and spatial index i. If NULL, skipped */ + braid_PtFcnInnerProd innerprod /**< Compute the inner product between two braid_Vectors. If NULL, skipped. */ + ); /** @}*/ #ifdef __cplusplus diff --git a/drivers/julia/drive-TaylorGreen.jl b/drivers/julia/drive-TaylorGreen.jl new file mode 100644 index 00000000..aa0534ac --- /dev/null +++ b/drivers/julia/drive-TaylorGreen.jl @@ -0,0 +1,329 @@ +using ForwardDiff, DiffResults, LinearAlgebra, PreallocationTools +using IterativeSolvers, LinearMaps, SparseArrays, Interpolations +using MPI, BenchmarkTools +using Plots + +include("../../braid/braid.jl/XBraid.jl") +using .XBraid + +MPI.Init() +comm = MPI.COMM_WORLD + +Float = Float64 + +""" +struct to package preallocated caches for storing temp coords and velocity +as well as the sparse poisson matrix needed by project_incompressible!() +""" +struct TGApp + x_d::DiffCache + u_d::DiffCache + ϕ_d::DiffCache + Δ_s::SparseMatrixCSC + solution + lyapunov_vecs + lyapunov_exps + times +end + +# default constructor +function TGApp(x::AbstractArray, u::AbstractArray, ϕ::AbstractArray, Δ::SparseMatrixCSC) + TGApp(DiffCache(x), DiffCache(u), DiffCache(ϕ), Δ, [], [], [], []) +end + +""" +reshapes 1D array into vector of 3D arrays, with no allocation +""" +function get_views(u) + @views begin + u_x = reshape(u[1:nₓ^3], (nₓ, nₓ, nₓ)) + u_y = reshape(u[nₓ^3+1:2*nₓ^3], (nₓ, nₓ, nₓ)) + u_z = reshape(u[2*nₓ^3+1:end], (nₓ, nₓ, nₓ)) + end + return [u_x, u_y, u_z] +end + +# some helpers for doing modulo arithmetic with CartesianIndex +Base.mod1(I::CartesianIndex, J::CartesianIndex) = CartesianIndex(broadcast(mod1, Tuple(I), Tuple(J))) +δ(I::CartesianIndex{N}, dim) where N = CartesianIndex(ntuple(i -> i == dim ? 1 : 0, N)) + +function TaylorGreen!(u, k = 1) + kx = trunc(Int64, k/2) + k%2 + ky = trunc(Int64, k/2) + 1 + kz = trunc(Int64, k/2) + 1 + u_x, u_y, u_z = u + x, y, z = coordinates + @. u_x = sin(kx * x) * cos(ky * y) * cos(kz * z) + @. u_y = -cos(kx * x) * sin(ky * y) * cos(kz * z) + @. u_z = 0.0 + return +end + +""" +finite difference approx first derivative +of A wrt dimension *dim* +""" +function ∂(A, dim; h = Δx) + # This works for A of arbitrary dimension + out = similar(A) + R = CartesianIndices(A) + bound = last(R) + @views @inbounds for I ∈ R + # second order + I⁻ = mod1(I - δ(I, dim), bound) + I⁺ = mod1(I + δ(I, dim), bound) + out[I] = 1 / 2h * (A[I⁺] - A[I⁻]) + end + return out +end + +# divergence, curl, gradient, and laplacian +∇_dot(V) = +sum(enumerate(V)) do (i, V_i) + ∂(V_i, i) +end + +function ∇X(V) + out_arr = zeros(3 * nₓ^3) + out = get_views(out_arr) + Vx, Vy, Vz = V + x, y, z = 1:3 + out[x] .= ∂(Vz, y) - ∂(Vy, z) + out[y] .= ∂(Vx, z) - ∂(Vz, x) + out[z] .= ∂(Vy, x) - ∂(Vx, y) + return out +end + +∇(F) = [∂(F, i) for i ∈ 1:3] +# Δ(F) = ∇_dot(∇(F)) # this is asymmetric... + +function my_poisson(f_arr) + out_arr = similar(f_arr) + F = reshape(f_arr, (nₓ, nₓ, nₓ)) + out = reshape(out_arr, (nₓ, nₓ, nₓ)) + R = CartesianIndices(F) + bound = last(R) + @inbounds for I ∈ R + out[I] = -6 * F[I] + @inbounds for dim ∈ 1:3 + I⁺ = mod1(I + δ(I, dim), bound) + I⁻ = mod1(I - δ(I, dim), bound) + out[I] += F[I⁺] + F[I⁻] + end + out[I] *= 1 / Δx^2 + end + # need a single point condition u(0,0,0) = 0., else + # this system is singular (ones(nₓ^3) nullspace) + out[first(R)] += F[first(R)] + return out_arr +end + +function my_poisson_mat(nₓ) + flat_index(I::CartesianIndex) = I[1] + (I[2] - 1) * nₓ + (I[3] - 1) * nₓ^2 + rowinds = Array{Int}([]) + colinds = Array{Int}([]) + nzs = Array{Float}([]) + function make_connection!(I_r, I_c, nz) + push!(rowinds, flat_index(I_r)) + push!(colinds, flat_index(I_c)) + push!(nzs, nz) + end + diag = -6 / Δx^2 + offd = 1 / Δx^2 + bound = CartesianIndex((nₓ, nₓ, nₓ)) + for I in CartesianIndices((1:nₓ, 1:nₓ, 1:nₓ)) + make_connection!(I, I, diag) + for dim ∈ 1:3 + I⁺ = mod1(I + δ(I, dim), bound) + I⁻ = mod1(I - δ(I, dim), bound) + make_connection!(I, I⁺, offd) + make_connection!(I, I⁻, offd) + end + end + # point condition u(0,0,0) = 0., else system is singular + nzs[1] += 1.0 + return sparse(rowinds, colinds, nzs) +end + +""" +compute self advection of u +""" +function advect_semi_lagrangian!(app::TGApp, u_arr, Δt) + # get cached arrays (either real or dual, depending on typeof(u)) + coords_new_arr = get_tmp(app.x_d, u_arr) + u_new_arr = get_tmp(app.u_d, u_arr) + + u = get_views(u_arr) + u_new = get_views(u_new_arr) + + # (1) trace each grid point backwards along u + @. coords_new_arr = coords_arr - Δt * u_arr + coords_new = get_views(coords_new_arr) + + # (2) interpolate the velocity field at the new grid points + x_range = range(0., 2π - Δx, nₓ) + for i in 1:3 + itp = interpolate(u[i], BSpline(Linear(Periodic()))) + sitp = scale(itp, x_range, x_range, x_range) + extp = extrapolate(sitp, Periodic()) + + u_new[i] .= extp.(coords_new...) + end + u_arr .= u_new_arr + return u_arr +end + +""" +Solve Poisson problem for incompressible velocity +Δϕ = ∇⋅u +u = u - ∇ϕ +""" +function project_incompressible!(app::TGApp, u_arr) + u = get_views(u_arr) + ϕ_arr = get_tmp(app.ϕ_d, u_arr) + ϕ_arr .= 0. + rhs = vec(∇_dot(u)) + cg!(ϕ_arr, app.Δ_s, rhs; abstol=1/2*Δx^2) + # cg!(ϕ_arr, app.Δ_s, rhs) + + ϕ = reshape(ϕ_arr, (nₓ, nₓ, nₓ)) + for i in 1:3 + u[i] .-= ∂(ϕ, i) + end + return u_arr +end + +function base_step(app::TGApp, u_arr, Δt) + # print("Advection : ") + advect_semi_lagrangian!(app, u_arr, Δt) + # diffuse_backward_Euler!(u_arr, Δt) # numerical diffusion may be enough + # print("Projection: ") + project_incompressible!(app, u_arr) + return u_arr +end + +nₓ = 64 +Δx = 2π / nₓ + +function my_init(app, t) + u_arr = zeros(Float, 3 * nₓ^3) + u = get_views(u_arr) + TaylorGreen!(u) + return u_arr +end + +function my_basis_init(app, t, i) + ψ_arr = zeros(Float, 3 * nₓ^3) + ψ = get_views(ψ_arr) + TaylorGreen!(ψ, i + 1) + ψ_arr .+= Δx^3*randn(Float, 3 * nₓ^3) + return ψ_arr +end + +function my_step!(app, status, u, ustop, tstart, tstop) + u = base_step(app, u, tstop - tstart) + return +end + +function my_step!(app, status, u, ustop, tstart, tstop, Ψ) + Δt = tstop - tstart + rank = length(Ψ) + Ψ_new = reduce(hcat, Ψ) + perturb(r) = base_step(app, u + r' * Ψ, Δt) + + result = DiffResults.DiffResult(u, Ψ_new) + result = ForwardDiff.jacobian!(result, perturb, zeros(rank)) + for i in 1:rank + Ψ[i] .= Ψ_new[:, i] + end + return +end + +function my_sum!(app, a, x, b, y) + @. y = a * x + b * y +end + +function my_access(app, status, u) + index = XBraid.status_GetTIndex(status) + lyap_exps = XBraid.status_GetLocalLyapExponents(status) + lyap_vecs = XBraid.status_GetBasisVectors(status) + if length(lyap_vecs) > 0 + push!(app.lyapunov_vecs, deepcopy(reduce(hcat, lyap_vecs))) + push!(app.lyapunov_exps, lyap_exps) + end + + push!(app.solution, deepcopy(u)) + push!(app.times, index) +end + +my_norm(app, u) = Δx^3 * LinearAlgebra.norm2(u) +my_innerprod(app, u, v) = u' * v + +coords_arr = zeros(Float, 3 * nₓ^3) +coordinates = get_views(coords_arr) +x_bound = 2π - Δx +x_range = range(0, x_bound, nₓ) +coordinates[1] .= [x for x ∈ x_range, y ∈ x_range, z ∈ x_range] +coordinates[2] .= [y for x ∈ x_range, y ∈ x_range, z ∈ x_range] +coordinates[3] .= [z for x ∈ x_range, y ∈ x_range, z ∈ x_range]; + +function test() + # preallocations + x_d = zeros(Float, 3 * nₓ^3) + u_d = zeros(Float, 3 * nₓ^3) + ϕ_d = zeros(Float, nₓ^3) + Δ = my_poisson_mat(nₓ) + my_app = TGApp(x_d, u_d, ϕ_d, Δ) + + # test user routines: + test_app = XBraid.BraidApp( + my_app, comm, comm, + my_step!, my_init, + my_sum!, my_norm, my_access, + my_basis_init, my_innerprod) + + XBraid.testInitAccess(test_app, 0.0) + XBraid.testClone(test_app, 0.0) + XBraid.testSpatialNorm(test_app, 0.0) + XBraid.testBuf(test_app, 0.0) + @time XBraid.testDelta(test_app, 0.0, 0.1, 3) + + curlz = ∇X(get_views(my_app.solution[1]))[3] + heatmap(curlz[:, :, 1]) +end + +function main(;ml=1, tstop=4., ntime=32, delta_rank=0) + # preallocations + x_d = zeros(Float, 3 * nₓ^3) + u_d = zeros(Float, 3 * nₓ^3) + ϕ_d = zeros(Float, nₓ^3) + Δ = my_poisson_mat(nₓ) + my_app = TGApp(x_d, u_d, ϕ_d, Δ) + + # theme(:dracula) + tstart = 0.0 + + core = XBraid.Init( + comm, comm, tstart, tstop, ntime, + my_step!, my_init, my_sum!, my_norm, my_access; app = my_app, + )::XBraid.BraidCore + + if delta_rank > 0 + XBraid.SetDeltaCorrection(core, delta_rank, my_basis_init, my_innerprod) + XBraid.SetLyapunovEstimation(core; exponents=true) + end + XBraid.SetMaxLevels(core, ml) + XBraid.SetMaxIter(core, 10) + XBraid.SetCFactor(core, -1, 2) + XBraid.SetAccessLevel(core, 1) + XBraid.SetNRelax(core, -1, 0) + XBraid.SetAbsTol(core, 1e-6) + + @time XBraid.Drive(core) + + p = sortperm(my_app.times) + curlz_start = ∇X(get_views(my_app.solution[p][1]))[3] + curlz_end = ∇X(get_views(my_app.solution[p][end]))[3] + plot(heatmap(curlz_start[1, :, :]), heatmap(curlz_end[1, :, :]), size=(900,400)) + return my_app +end \ No newline at end of file diff --git a/drivers/julia/drive-advdiff-theta.jl b/drivers/julia/drive-advdiff-theta.jl new file mode 100644 index 00000000..add1d651 --- /dev/null +++ b/drivers/julia/drive-advdiff-theta.jl @@ -0,0 +1,151 @@ +using ToeplitzMatrices, LinearAlgebra +using Plots +using MPI +using IterativeSolvers + +include("../../braid/braid.jl/XBraid.jl") +using .XBraid + +struct AdvDifApp + nx::Int + Δx::Float64 + A::Circulant + I::SymmetricToeplitz + useTheta::Bool + useRich::Bool + cf::Int + order::Int + solTf::Vector{Vector{Float64}} + residuals::Vector{Float64} +end +# default constructor +function AdvDifApp(nx::Integer, Δx::Real, ν::Real, α::Real, useTheta::Bool, useRich::Bool, cf::Integer, order::Integer) + I = SymmetricToeplitz([1, zeros(nx-1)...]) + # Δ = (1/Δx^2)SymmetricToeplitz([-2, 1, zeros(nx-2)...]) + # ∇ = (1/Δx)Toeplitz([0, -1/2, zeros(nx-2)...], [0, 1/2, zeros(nx-2)...]) + Δ = (1/Δx^2)Circulant([-2, 1, zeros(nx-3)..., 1]) + ∇ = (1/Δx)Circulant([0, -1/2, zeros(nx-3)..., 1/2]) + # ∇ = (1/Δx)Circulant([-1, 1, zeros(nx-2)...]) + A = ν*Δ - α*∇ + AdvDifApp(nx, Δx, A, I, useTheta, useRich, cf, order, [], []) +end + +function my_init(app, t) + t == 0.0 && return sin.(range(0., 2π-app.Δx, app.nx)) + return randn(app.nx) +end + +function backward_euler!(app, u, Δt) + u .= (app.I - Δt * app.A) \ u + # u .= cg(app.I - Δt * app.A, u) + return u +end + +function θ_euler!(app, u, Δt, m) + θ = (m - 1)/2m + u .= (app.I - (1-θ)Δt * app.A) \ ((app.I + θ*Δt * app.A) * u) + return u +end + +function SDIRK2!(app, u, Δt) + a = 1/√2 + k1 = Δt*((app.I - (1-a) * Δt * app.A) \ (app.A*u)) + k2 = Δt*((app.I - (1-a) * Δt * app.A) \ (app.A*(u + (2a-1)*k1))) + u .+= 1/2*(k1 + k2) + return u +end + +function θSDIRK2!(app, u, Δt, m) + α = √2√(m*(m-1))/2m + θ = (m^2 -3m - √2√(m*(m-1)))/(2m^2 - 4m) + k1 = Δt*((app.I - (1-α) * Δt * app.A) \ (app.A*u)) + k2 = Δt*((app.I - (1-α) * Δt * app.A) \ (app.A*(u + (2α-1)*k1))) + u .+= (θ)k1 + (1-θ)k2 + return u +end + +function base_step!(app, u, Δt, level) + if app.order == 2 + if app.useTheta + throw("Theta method not implemented for order 2") + end + return SDIRK2!(app, u, Δt) + end + + if app.useTheta && level > 0 + return θSDIRK2!(app, u, Δt, app.cf^level) + # return θ_euler!(app, u, Δt, app.cf^level) + end + + return backward_euler!(app, u, Δt) +end + +function my_step!(app, status, u, ustop, tstart, tstop) + Δt = tstop - tstart + level = XBraid.status_GetLevel(status) + if !app.useRich || level == 0 + return base_step!(app, u, Δt, level) + end + m = app.cf ^ level + p = app.order + if app.useTheta + p += 1 + end + θ = 2^p * (m^p - 1) / (m^p * (2^p - 1)) + u_sub = deepcopy(u) + base_step!(app, u_sub, Δt/2, level) + base_step!(app, u_sub, Δt/2, level) + base_step!(app, u, Δt, level) + @. u = θ*u_sub + (1-θ)*u + return u +end + +function my_access(app, status, u) + XBraid.status_GetWrapperTest(status) && return + ti = XBraid.status_GetTIndex(status) + ntime = XBraid.status_GetNTPoints(status) + if ti == ntime + push!(app.solTf, deepcopy(u)) + end +end + +function test(;ν=1., α=1., order=1) + MPI.Init() + comm = MPI.COMM_WORLD + + my_app = AdvDifApp(512, 2π/513, ν, α, false, false, 2, order) + app = XBraid.BraidApp(my_app, comm, my_step!, my_init, my_access) + + XBraid.testInitAccess(app, 0.) + XBraid.testSpatialNorm(app, 0.) + u = my_init(my_app, 0.) + plot(u, label="u0", legend=:bottomright) + my_step!(my_app, nothing, u, nothing, 0., .1) + plot!(u, label="u1") +end + + +function main(;tstop=2π, ntime=512, nx=512, ν=1., α=1., useTheta=false, useRich=false, cf=2, ml=2, order=1) + MPI.Init() + comm = MPI.COMM_WORLD + + Δx = 2π / nx + app = AdvDifApp(nx, Δx, ν, α, useTheta, useRich, cf, order) + + core = XBraid.Init(comm, 0.0, tstop, ntime, my_step!, my_init, my_access; app=app) + + XBraid.SetPrintLevel(core, 2) + XBraid.SetMaxLevels(core, ml) + XBraid.SetCFactor(core, -1, app.cf) + XBraid.SetAccessLevel(core, 1) + XBraid.SetNRelax(core, -1, 1) + XBraid.SetAbsTol(core, 1e-8) + XBraid.SetMaxIter(core, 40) + XBraid.SetSkip(core, false) + + XBraid.Drive(core) + + append!(app.residuals, XBraid.GetRNorms(core)) + + return app +end \ No newline at end of file diff --git a/drivers/julia/drive-burgers.jl b/drivers/julia/drive-burgers.jl new file mode 100644 index 00000000..8225e22b --- /dev/null +++ b/drivers/julia/drive-burgers.jl @@ -0,0 +1,334 @@ +using LinearAlgebra, PreallocationTools, ForwardDiff, DiffResults +using MPI, Interpolations, IterativeSolvers +using SparseArrays, ToeplitzMatrices +using FFTW +using Plots +using LinearAlgebra: norm2 +using Random: seed! +# theme(:dracula) + +include("../../braid/braid.jl/XBraid.jl") +using .XBraid + +MPI.Init() +comm = MPI.COMM_WORLD + +struct BurgerApp + x::Vector{Float64} + κ::Frequencies{Float64} + μ::Float64 + cf::Int + useTheta::Bool + # preallocated caches that support ForwardDiff: + x_d::DiffCache + y_d::DiffCache + # storage for solution values + solution::Vector{Vector{Float64}} + lyap_vecs::Vector{Union{Matrix{Float64}, Vector{Float64}}} + lyap_exps::Vector{Vector{Float64}} + times::Vector{Int} + # pre-planned fourier transform + P̂ +end + +function BurgerApp(x::Vector{<:Real}, κ::Frequencies{<:Real}, μ::Real, cf::Integer, useTheta::Bool, x_cache::AbstractArray, y_cache::AbstractArray) + BurgerApp(x, κ, μ, cf, useTheta, DiffCache(x_cache), DiffCache(y_cache), [], [], [], [], plan_fft(x)) +end + +# system parameters (globally scoped) +const lengthScale = 2π +const nₓ = 1024 +const Δx = lengthScale / nₓ + +function stencil_to_circulant(stencil::Vector{T}, nx::Integer) where T + @assert isodd(length(stencil)) "stencil should have odd number of entries" + stencilWidth = length(stencil) + center = ceil(Int, length(stencil)/2) + columnView = @view(stencil[end:-1:1]) + v = [columnView[center:end]; zeros(T, nx-stencilWidth); columnView[1:center-1]] + return Circulant(v) +end + +function euler_mat(Δt::Float64; μ::Float64=.00001) + # diffusion + # A = spdiagm(-1 => ones(nₓ-1), 0 => -2*ones(nₓ), 1 => ones(nₓ-1)) + # A[1, end] = 1 + # A[end, 1] = 1 + # A = sparse(I, (nₓ, nₓ)) - Δt/Δx^2 * μ * A + + # KS operator + ∇⁴ = [1, -4, 6, -4, 1] + ∇² = [0, 1, -2, 1, 0] + I = [0, 0, 1, 0, 0] + stencil = @. I - Δt*(-1/Δx^4 * ∇⁴ - 1/Δx^2 * ∇²) + # stencil = @. I - Δt(-μ/Δx^4 * ∇⁴) + stencil_to_circulant(stencil, nₓ) +end + +function semi_lagrangian!(burger::BurgerApp, y, v, Δt) + x_back = get_tmp(burger.x_d, y) + y_intp = get_tmp(burger.y_d, y) + + # initialize interpolation + itp = interpolate(y, BSpline(Linear(Periodic()))) + sitp = scale(itp, range(0., lengthScale - Δx, nₓ)) + extp = extrapolate(sitp, Periodic()) + + # forward euler + x_back .= burger.x .- Δt * v + + # backward euler + # max_newton = 1000 + # av_iters = 0. + # for i ∈ eachindex(x_back) + # for iter ∈ 1:max_newton + # p = extp(x_back[i]) + # p_prime = Interpolations.gradient(extp, x_back[i])[] + # x_back[i] -= (x_back[i] + Δt * p - burger.x[i]) / (1 + Δt * p_prime) + # if abs(x_back[i] + Δt * p - burger.x[i]) < 1e-10 + # # av_iters += iter/nₓ + # break + # end + # end + # end + # println(av_iters) + + # y_intp .= interp_periodic.(x_back, Ref(y)) + y_intp .= extp.(x_back) + y .= y_intp + return y +end + +function diffuse_beuler!(burger::BurgerApp, y, Δt::Float64; init_guess=nothing) + y_tmp = get_tmp(burger.y_d, y) + y_tmp .= y + + if init_guess !== nothing + y .= init_guess + end + + cg!(y, euler_mat(Δt), y_tmp) + return y +end + +function diffuse_fft!(burger::BurgerApp, y::AbstractArray, Δt::Real; init_guess=nothing) + P̂ = burger.P̂ + κ = burger.κ + # y .= real(P̂ \ (exp.(Δt*(κ.^2 - κ.^4)))) + ŷ = P̂ * y + @. ŷ *= exp(-burger.μ * Δt * κ^2) # standard diffusion + # @. ŷ *= exp(Δt*(κ^2 - κ^4)) # KS-equation operator + y .= real(P̂ \ ŷ) + return y +end + +function extractPartials(y::Vector{ForwardDiff.Dual{T,V,P}}) where {T,V,P} + ps = zeros(V, size(y)..., P) + for i ∈ eachindex(y), j ∈ 1:P + @inbounds ps[i, j] = ForwardDiff.partials(y[i], j) + end + return ps +end + +function fillDualArray!(y::Vector{ForwardDiff.Dual{T,V,P}}, vs, ps) where {T,V,P} + checkbounds(ps, firstindex(y), 1:P) + for i ∈ eachindex(y) + @inbounds y[i] = ForwardDiff.Dual{T}(vs[i], ntuple(j -> @inbounds(ps[i, j]), P)) + end +end + +# this enables ForwardDiff through the FFT where it normally doesn't work +function diffuse_fft!(burger::BurgerApp, y::Vector{ForwardDiff.Dual{T,V,P}}, Δt::Real; μ=1e-4) where {T, V, P} + vs = ForwardDiff.value.(y) + ps = extractPartials(y) + diffuse_fft!(burger, vs, Δt; μ=μ) + map(eachcol(ps)) do p diffuse_fft!(burger, p, Δt; μ=μ) end + fillDualArray!(y, vs, ps) + return y +end + +# user routines: +function my_init(burger::BurgerApp, t::Float64) + # sin + u = sin.(2π/lengthScale * burger.x) + # gaussian + # u = exp.(-(burger.x .- lengthScale/2).^2 / (lengthScale/5)^2) + # if t == 0 + # seed!(1234) + # u .+= 1e-1randn(nₓ) .* u + # end + # @. u = exp(-(burger.x - lengthScale/2)^2) + return u +end + +function my_basis_init(burger::BurgerApp, t::Float64, k::Int32) + x = burger.x + ψ = similar(x) + if k % 2 == 0 + @. ψ = cos(k/2*x) + else + @. ψ = sin((k+1)/2*x) + end + return ψ +end + +function base_step!(burger::BurgerApp, u, Δt::Float64; init_guess=nothing) + u_mid = deepcopy(u) + semi_lagrangian!(burger, u_mid, u, Δt/2) + semi_lagrangian!(burger, u, u_mid, Δt) # u(∇ u) + # semi_lagrangian!(burger, u, sin.(burger.x), Δt) # ∇ u + # semi_lagrangian!(burger, u, u, Δt) # u(∇ u) + if burger.μ > 0. + diffuse_fft!(burger, u, Δt) # Δu + end +end + +function my_step!( + burger::BurgerApp, status::XBraid.StepStatus, + u, ustop, tstart::Float64, tstop::Float64 +) + Δt = tstop - tstart + level = XBraid.status_GetLevel(status) + if !burger.useTheta || level == 0 + base_step!(burger, u, Δt; init_guess=ustop) + else + # richardson based θ method + m = burger.cf^level + θ = 2(m - 1)/m + # p = 0.86 + # p = 1. + # θ = 2^p * (m^p - 1) / (m^p * (2^p - 1)) + # θ = 2 + u_sub = deepcopy(u) + base_step!(burger, u_sub, Δt/2; init_guess=(ustop .- u)./2) + base_step!(burger, u_sub, Δt/2; init_guess=ustop) + base_step!(burger, u, Δt; init_guess=u_sub) + @. u = θ*u_sub + (1-θ)*u + end + return u +end + +function my_step!(burger::BurgerApp, status::XBraid.StepStatus, u, ustop, tstart::Float64, tstop::Float64, Ψ) + rank = length(Ψ) + Ψ_new = reduce(hcat, Ψ) + # perturb(r) = base_step!(burger, u + r' * Ψ, Δt) + perturb(r) = my_step!(burger, status, u + r' * Ψ, ustop, tstart, tstop) + + result = DiffResults.DiffResult(u, Ψ_new) + result = ForwardDiff.jacobian!(result, perturb, zeros(rank)) + for i in eachindex(Ψ) + Ψ[i] .= Ψ_new[:, i] + end +end + +function my_sum!(burger, a, x, b, y) + @. y = a*x + b*y +end + +function my_access(burger::BurgerApp, status::XBraid.AccessStatus, u) + XBraid.status_GetWrapperTest(status) && return + ti = XBraid.status_GetTIndex(status) + push!(burger.solution, deepcopy(u)) + push!(burger.times, ti) + if XBraid.status_GetDeltaRank(status) > 0 + Ψ = XBraid.status_GetBasisVectors(status) + λ = XBraid.status_GetLocalLyapExponents(status) + push!(burger.lyap_vecs, deepcopy(reduce(hcat, Ψ))) + push!(burger.lyap_exps, deepcopy(λ)) + end +end + +my_norm(burger, u) = LinearAlgebra.norm2(u) +my_innerprod(burger, u, v) = u' * v + +# test user routines: +function test() + x_new = zeros(nₓ) + y_new = zeros(nₓ) + x = Array(range(0., lengthScale-Δx, nₓ)) + κ = 2π/lengthScale .* fftfreq(nₓ, nₓ) + burger = BurgerApp(x, κ, 0., 4, false, x_new, y_new); + test_app = XBraid.BraidApp( + burger, comm, comm, + my_step!, my_init, + my_sum!, my_norm, my_access, + my_basis_init, my_innerprod) + + open("drive-burgers.test.out", "w") do file + cfile = Libc.FILE(file) + XBraid.testBuf(test_app, 0.0, cfile) + XBraid.testSpatialNorm(test_app, 0.0, cfile) + XBraid.testDelta(test_app, 0.0, 0.1, 3, cfile) + end + # plot!(burger.lyap_vecs[1][1]) + plot(burger.solution[1]) +end + +function main(;tstop=π, ntime=128, deltaRank=0, useTheta=false, fmg=false, ml=1, cf=4, saveGif=false, maxiter=30, μ=0.) + x_new = zeros(nₓ) + y_new = zeros(nₓ) + x = Array(range(0., lengthScale-Δx, nₓ)) + κ = 2π/lengthScale .* fftfreq(nₓ, nₓ) + burger = BurgerApp(x, κ, μ, 4, useTheta, x_new, y_new); + + tstart = 0.0 + core = XBraid.Init( + comm, comm, tstart, tstop, ntime, + my_step!, my_init, my_sum!, my_norm, my_access; app=burger + ) + + if deltaRank > 0 + XBraid.SetDeltaCorrection(core, deltaRank, my_basis_init, my_innerprod) + XBraid.SetLyapunovEstimation(core; exponents=true) + end + + # println("Wrapper test:") + # @time test() + + XBraid.SetMaxIter(core, maxiter) + XBraid.SetMaxLevels(core, ml) + XBraid.SetCFactor(core, -1, cf) + XBraid.SetAccessLevel(core, 1) + XBraid.SetNRelax(core, -1, 1) + XBraid.SetAbsTol(core, 1e-6) + XBraid.SetSkip(core, false) + if fmg + XBraid.SetFMG(core) + XBraid.SetNFMG(core, 1) + end + + XBraid.SetTimings(core, 2) + XBraid.Warmup(core) + XBraid.Drive(core; warmup=false) + XBraid.PrintTimers(core) + + if deltaRank > 0 + exponents = sum(burger.lyap_exps) + exponents = MPI.Allreduce(exponents, (+), comm) + exponents ./= tstop + if MPI.Comm_rank(comm) == 0 + println("exponents: ", exponents) + end + end + + if saveGif + for (ti, u) in zip(burger.times, burger.solution) + plt = plot(burger.x, u; ylim=(-1.5, 1.5)) + savefig(plt, "burgers_gif/$(lpad(ti, 6, "0")).png") + end + MPI.Barrier(comm) + + if MPI.Comm_rank(comm) == 0 + println("animating...") + fnames = [lpad(i, 6, "0") for i in 0:ntime] + anim = Animation("burgers_gif", fnames) + Plots.buildanimation(anim, "burgers_gif/anim.gif", fps=30) + end + end + + return burger, XBraid.GetRNorms(core) +end + +# test() +# main(); +nothing \ No newline at end of file diff --git a/drivers/julia/drive-kflow.jl b/drivers/julia/drive-kflow.jl new file mode 100644 index 00000000..902bee34 --- /dev/null +++ b/drivers/julia/drive-kflow.jl @@ -0,0 +1,443 @@ +using ForwardDiff, DiffResults, LinearAlgebra, PreallocationTools +using Interpolations, FFTW +using MPI, Base.Threads +using BenchmarkTools, Plots, LaTeXStrings +using Statistics: mean +using Random: seed! +using LinearAlgebra: norm + +include("../../braid/braid.jl/XBraid.jl") +using .XBraid + +MPI.Init() +comm = MPI.COMM_WORLD + +Float = Float64 + +""" +struct containing everything needed internally for the simulation +""" +struct KFlowApp + cf::Int + useTheta::Bool + deltaRank::Int + coords::Array{Float, 3} + κ::Frequencies{Float} + k_num::Int + ℜ::Float + solution::Vector{Array{Float, 3}} + solTf::Vector{Array{Float, 3}} + lyapunov_vecs::Vector{Vector{Array{Float, 3}}} + lyapunov_exps::Vector{Vector{Float}} + times::Vector{Int} + x_d::DiffCache{Array{Float, 3}} + u_d::DiffCache{Array{Float, 3}} + ϕ_d::DiffCache{Array{Complex{Float}, 2}} + P̂::FFTW.FFTWPlan +end + +# default constructor +function KFlowApp(cf::Integer, useTheta::Bool, deltaRank::Integer, k_num::Integer, ℜ::Real) + coords = zeros(Float, nₓ, nₓ, 2) + x_range = range(0, lengthScale - Δx, nₓ) + @views @inbounds for (i, x) ∈ enumerate(x_range), (j, y) ∈ enumerate(x_range) + coords[i, j, 1] = x + coords[i, j, 2] = y + end + κ = 2π / lengthScale .* fftfreq(nₓ, nₓ) + # preallocations + x_d = zeros(Float, nₓ, nₓ, 2) + u_d = zeros(Float, nₓ, nₓ, 2) + ϕ_d = zeros(Complex{Float}, nₓ, nₓ) + KFlowApp(cf, useTheta, deltaRank, coords, κ, k_num, ℜ, [], [], [], [], [], DiffCache(x_d), DiffCache(u_d), DiffCache(ϕ_d), plan_fft(coords, [1, 2])) +end + +oneball(n) = [(n + 1 - i, i) for i ∈ 1:n] +wavenumbers(shell) = reduce(hcat, [oneball(i) for i ∈ 1:shell]) +is_in_shell(i, shell) = i <= trunc(Int, shell * (shell + 1) / 2) + +""" +compute ith wavenumber for a 2D scalar field +""" +function get_wavenumber2D(i) + shell = 1 + for _ ∈ 1:i + is_in_shell(i, shell) && break + shell += 1 + end + shell_start = trunc(Int, shell * (shell - 1) / 2) + rel_i = i - shell_start + return oneball(shell)[rel_i] +end + +""" +compute the ith Fourier mode of a 1D scalar field +""" +function fourierMode1D(x::Real, k::Integer) + k % 2 == 1 && return cos(trunc(k / 2) * 2π / lengthScale * x) + k % 2 == 0 && return sin(trunc((k + 1) / 2) * 2π / lengthScale * x) +end + +""" +compute the ith Fourier mode of a 2D scalar field +""" +function fourierMode2D!(app::KFlowApp, u_field, i::Integer) + kx, ky = get_wavenumber2D(i) + @views x, y = app.coords[:, :, 1], app.coords[:, :, 2] + @. u_field = fourierMode1D(x, kx) * fourierMode1D(y, ky) +end + +""" +compute the ith Fourier mode of a 2D vector field +""" +function fourierMode2DVec!(app::KFlowApp, u, i::Integer) + @views ux, uy = u[:, :, 1], u[:, :, 2] + kx, ky = get_wavenumber2D(i) + fourierMode2D!(app, ux, kx) + fourierMode2D!(app, uy, ky) +end + +""" +set the initial condition to a Taylor-Green vortex +""" +function TaylorGreen!(app::KFlowApp, u, k = 1) + coords = app.coords + kx = trunc(Int, k / 2) + k % 2 + ky = trunc(Int, k / 2) + 1 + @views u_x, u_y = u[:, :, 1], u[:, :, 2] + @views x, y = coords[:, :, 1], coords[:, :, 2] + @. u_x = sin(kx * x) * cos(ky * y) + @. u_y = -cos(kx * x) * sin(ky * y) + return +end + +""" +compute the forcing term +""" +function kolmogorovForce!(app::KFlowApp, u, Δt; μ = 32.) + k = app.k_num + # Fₖ(x, y) = (sin(ky), 0) + @views @. u[:, :, 1] += μ * Δt * sin(k * app.coords[:, :, 2]) + return u +end + +""" +compute self advection of u +""" +function advect_semi_lagrangian!(app::KFlowApp, u::AbstractArray, v::AbstractArray, Δt::Real) + # get cached arrays (either real or dual, depending on typeof(u)) + coords_new = get_tmp(app.x_d, u) + u_new = get_tmp(app.u_d, u) + + # (1) trace each grid point backwards along u + @. coords_new = app.coords - Δt * v + + # (2) interpolate the velocity field at the new grid points + x_range = range(0.0, lengthScale - Δx, nₓ) + for i in 1:2 + itp = interpolate(u[:, :, i], BSpline(Linear(Periodic()))) + sitp = scale(itp, x_range, x_range) + extp = extrapolate(sitp, Periodic()) + + u_new[:, :, i] .= @views extp.(coords_new[:, :, 1], coords_new[:, :, 2]) + end + u .= u_new + return u +end + +""" +Solve Poisson problem for incompressible velocity field +Δϕ = ∇⋅u +u = u - ∇ϕ +""" +function project_incompressible!(app::KFlowApp, u::AbstractArray, Δt::Real) + κ = get_tmp(app.x_d, u) + ϕ = get_tmp(app.ϕ_d, u) + P̂ = app.P̂ + ℜ = app.ℜ + for (i, κ_x) ∈ enumerate(app.κ), (j, κ_y) ∈ enumerate(app.κ) + κ[i, j, 1] = κ_x + κ[i, j, 2] = κ_y + end + û = P̂ * u + @views κ_x, κ_y = κ[:, :, 1], κ[:, :, 2] + @views û_x, û_y = û[:, :, 1], û[:, :, 2] + # diffusion + @. û_x *= exp(-Δt / ℜ * (κ_x^2 + κ_y^2)) + @. û_y *= exp(-Δt / ℜ * (κ_x^2 + κ_y^2)) + + # KS equation: uₜ + u(∇u) = - Δu - Δ²u + # @. û_x *= exp(Δt*(κ_x^2 + κ_y^2 - (κ_x^2 + κ_y^2)^2)) + # @. û_y *= exp(Δt*(κ_x^2 + κ_y^2 - (κ_x^2 + κ_y^2)^2)) + + # incompressible projection + # ϕ = inv(Δ)∇⋅u + @. ϕ = -(1.0im * κ_x * û_x + 1.0im * κ_y * û_y) / (κ_x^2 + κ_y^2) + ϕ[1, 1] = 0.0 + 0.0im # pressure is unique up to a constant + # u = u - ∇ϕ + @. û_x -= 1.0im * κ_x * ϕ + @. û_y -= 1.0im * κ_y * ϕ + + u .= real(P̂ \ û) + return u +end + +function extractPartials(u::AbstractArray{ForwardDiff.Dual{T, V, P}}) where {T, V, P} + ps = zeros(V, size(u)..., P) + for I ∈ CartesianIndices(u), j ∈ 1:P + @inbounds ps[I, j] = ForwardDiff.partials(u[I], j) + end + return ps +end + +function fillArrayOfDuals!(u::AbstractArray{ForwardDiff.Dual{T, V, P}}, vs::AbstractArray{V}, ps::AbstractArray{V}) where {T, V, P} + checkbounds(ps, first(CartesianIndices(u)), 1:P) # make inbounds safe + for I ∈ CartesianIndices(u) + @inbounds u[I] = ForwardDiff.Dual{T}(vs[I], ntuple(j -> @inbounds(ps[I, j]), P)) + end +end + +# this enables ForwardDiff through the FFT where it normally doesn't work +function project_incompressible!(app::KFlowApp, u::AbstractArray{ForwardDiff.Dual{T, V, P}}, Δt::Real) where {T, V, P} + vs = ForwardDiff.value.(u) + ps = extractPartials(u) + project_incompressible!(app, vs, Δt) + map(eachslice(ps, dims = 4)) do p + project_incompressible!(app, p, Δt) + end + fillArrayOfDuals!(u, vs, ps) + return u +end + +function ∇_dot(app, u) + κ = get_tmp(app.x_d, u) + for (i, κ_x) ∈ enumerate(app.κ), (j, κ_y) ∈ enumerate(app.κ) + κ[i, j, 1] = κ_x + κ[i, j, 2] = κ_y + end + @views κx, κy = κ[:, :, 1], κ[:, :, 2] + û = app.P̂ * u + out = @. 1.0im * κx * û[:, :, 1] + 1.0im * κy * û[:, :, 2] + return real(ifft(out)) +end + +function ∇X(app::KFlowApp, V) + κ = get_tmp(app.x_d, V) + for (i, κ_x) ∈ enumerate(app.κ), (j, κ_y) ∈ enumerate(app.κ) + κ[i, j, 1] = κ_x + κ[i, j, 2] = κ_y + end + f = get_tmp(app.ϕ_d, V) + V̂ = app.P̂ * V + @views V̂x, V̂y = V̂[:, :, 1], V̂[:, :, 2] + @views κx, κy = κ[:, :, 1], κ[:, :, 2] + out = @. 1.0im * κx * V̂y - 1.0im * κy * V̂x + return real(ifft(out)) +end + +function spectralInterpolation(f, targetSize) + size_f = size(f)[1] # assume square + targetSize > size_f || return f + npad = floor(Int, targetSize / 2 - size_f / 2) + f̂ = fftshift(fft(f)) + C = eltype(f̂) + f̂_interp = [ + zeros(C, npad, npad) zeros(C, npad, size_f) zeros(C, npad, npad) + zeros(C, size_f, npad) f̂ zeros(C, size_f, npad) + zeros(C, npad, npad) zeros(C, npad, size_f) zeros(C, npad, npad) + ] + return real(ifft(ifftshift(f̂_interp))) +end + +function base_step!(app::KFlowApp, u::AbstractArray, Δt::Float) + kolmogorovForce!(app, u, Δt) + advect_semi_lagrangian!(app, u, u, Δt) + project_incompressible!(app, u, Δt) + return u +end + +function my_init(app::KFlowApp, t::Float) + seed!(1) + # u = zeros(Float, nₓ, nₓ, 2) + u = randn(Float, nₓ, nₓ, 2) + # TaylorGreen!(app, u) + project_incompressible!(app, u, 1e-1app.ℜ) + u[:, :, 1] .-= mean(u[:, :, 1]) + u[:, :, 2] .-= mean(u[:, :, 2]) + u ./= my_norm(app, u) + return u +end + +function my_basis_init(app::KFlowApp, t::Float, i::Integer) + ψ = zeros(Float, nₓ, nₓ, 2) + fourierMode2DVec!(app, ψ, i + 1) + return ψ +end + +function my_step!( + app::KFlowApp, status::Ptr{Cvoid}, + u::AbstractArray, ustop::AbstractArray, + tstart::Real, tstop::Real, +) + Δt = tstop - tstart + level = XBraid.status_GetLevel(status) + iter = XBraid.status_GetIter(status) + if !app.useTheta || level == 0 + u = base_step!(app, u, Δt) + else + # richardson based θ method + m = app.cf ^ level + p = 1 + θ = 2^p * (m^p - 1) / (m^p * (2^p - 1)) + # θ = 2 + u_sub = deepcopy(u) + base_step!(app, u_sub, Δt / 2) + base_step!(app, u_sub, Δt / 2) + base_step!(app, u, Δt) + @. u = θ * u_sub + (1 - θ) * u + end + return u +end + +function my_step!( + app::KFlowApp, status::Ptr{Cvoid}, + u::AbstractArray, ustop::AbstractArray, + tstart::Real, tstop::Real, + Ψ::Vector{Any} +) + rank = length(Ψ) + Ψ_new = reduce((a, b) -> cat(a, b, dims = 4), Ψ) + perturb(r) = my_step!(app, status, u + r' * Ψ, ustop, tstart, tstop) + + result = DiffResults.DiffResult(u, Ψ_new) + result = ForwardDiff.jacobian!(result, perturb, zeros(rank)) + for i in 1:rank + Ψ[i] .= Ψ_new[:, :, :, i] + end + return +end + +function my_sum!(app, a, x, b, y) + @. y = a * x + b * y +end + +function my_access(app::KFlowApp, status, u) + XBraid.status_GetWrapperTest(status) && return + ntime = XBraid.status_GetNTPoints(status) + index = XBraid.status_GetTIndex(status) + level, done = XBraid.status_GetLevel(status), XBraid.status_GetDone(status) + + if level == 0 && done && index % app.cf == 0 + push!(app.solution, deepcopy(u)) + push!(app.times, index) + rank = XBraid.status_GetDeltaRank(status) + if rank > 0 + exps = XBraid.status_GetLocalLyapExponents(status) + vecs = XBraid.status_GetBasisVectors(status) + push!(app.lyapunov_exps, exps) + push!(app.lyapunov_vecs, deepcopy(vecs)) + end + end + + if level == 0 && index == ntime-1 + # last time step + push!(app.solTf, deepcopy(u)) + end +end + +my_norm(app, u) = LinearAlgebra.normInf(u) +my_innerprod(app, u, v) = u ⋅ v + +const lengthScale = 2π +const nₓ = 99 +const Δx = lengthScale / nₓ + +function test() + my_app = KFlowApp(2, false, 3, 2, 1600) + + test_app = XBraid.BraidApp( + my_app, comm, comm, + my_step!, my_init, + my_sum!, my_norm, my_access, + my_basis_init, my_innerprod) + + XBraid.testInitAccess(test_app, 0.0) + XBraid.testClone(test_app, 0.0) + XBraid.testSpatialNorm(test_app, 0.0) + XBraid.testBuf(test_app, 0.0) + XBraid.testDelta(test_app, 0.0, 0.1, my_app.deltaRank) + + # curl = ∇X(my_app, my_app.solution[1]) + # heatmap(curl') +end + +function main(;tstop=20.0, ntime=512, deltaRank=0, ml=1, cf=2, maxiter=10, fcf=1, relaxLyap=false, savegif=false, useTheta=false, k=4, ℜ=1600, deferDelta=(1, 1)) + my_app = KFlowApp(cf, useTheta, deltaRank, k, ℜ) + + tstart = 0.0 + + core = XBraid.Init( + comm, comm, tstart, tstop, ntime, + my_step!, my_init, my_sum!, my_norm, my_access; app = my_app, + ) + + if deltaRank > 0 + XBraid.SetDeltaCorrection(core, deltaRank, my_basis_init, my_innerprod) + # XBraid.SetDeferDelta(core, deferDelta...) + XBraid.SetLyapunovEstimation(core; relax = relaxLyap, exponents = true) + end + XBraid.SetMaxLevels(core, ml) + XBraid.SetMaxIter(core, maxiter) + XBraid.SetCFactor(core, -1, cf) + XBraid.SetNRelax(core, -1, fcf) + XBraid.SetAccessLevel(core, 2) + XBraid.SetAbsTol(core, 1e-6) + + XBraid.SetTimings(core, 2) + XBraid.Drive(core) + XBraid.PrintTimers(core) + + p = sortperm(my_app.times) + + if deltaRank > 0 + exponents = sum(my_app.lyapunov_exps) + exponents = MPI.Allreduce(exponents, (+), comm) + exponents ./= tstop - tstart + if MPI.Comm_rank(comm) == 0 + println("exponents: ", exponents) + end + end + + # if deltaRank > 0 + # movingaverage(g, n) = [i < n ? mean(g[begin:i]) : mean(g[i-n+1:i]) for i in eachindex(g)] + # println("Lyap Exps: ", sum(my_app.lyapunov_exps) ./ tstop) + # exps = reduce(hcat, my_app.lyapunov_exps[p])' + # nt = size(exps)[1] + # Δt = tstop / nt + # exps = movingaverage.([exps[:, i] ./ Δt for i ∈ 1:size(exps)[2]], length(exps[:, 1])) + # exps_plot = plot(exps; legend = false) + # savefig(exps_plot, "lyapunov_exps.png") + # end + + if savegif + preprocess(app, u) = spectralInterpolation(∇X(app, u), 128)' + heatmapArgs = Dict(:ticks => false, :colorbar => false, :aspect_ratio => :equal) + # anim = @animate for i in p[1:cf:end] + count = 1 + for i in p + u = my_app.solution[i] + plots = [heatmap(preprocess(my_app, u); heatmapArgs...)] + for j in 1:min(8, deltaRank) + ψⱼ = my_app.lyapunov_vecs[i][j] + push!(plots, heatmap(preprocess(my_app, ψⱼ); heatmapArgs...)) + end + fig = plot(plots...; size = (600, 600)) + savefig(fig, "kflow_gif/beamer/kolmo_$(nₓ)_$(ntime)_ml$(ml)-$(count).png") + count += 1 + end + # gif(anim, "kflow_gif/kolmo_$(nₓ)_$(ntime)_ml$(ml).gif", fps = 20) + end + + + return my_app, core +end diff --git a/examples/julia/ex-01.jl b/examples/julia/ex-01.jl new file mode 100644 index 00000000..aa281308 --- /dev/null +++ b/examples/julia/ex-01.jl @@ -0,0 +1,69 @@ +# main ex-01 +# include("XBraid.jl") +include("../../braid/braid.jl/XBraid.jl") +using .XBraid +using MPI + +MPI.Init() +comm = MPI.COMM_WORLD + +#= +using 1-element vector here since standard Float64 +values are immutable and don't have stable addresses. +=# +function my_init(app, t) + t == 0.0 && return [1.0] + return [0.456] +end + +# This must mutate u in place +function my_step!(app, status, u, ustop, tstart, tstop) + # backward Euler + u[] = 1.0 / (1.0 + tstop - tstart) * u[] +end + +#= +called by the solver to give access to +solution values. +=# +function my_access(app, status, u) + XBraid.status_GetWrapperTest(status) && return + t = XBraid.status_GetT(status) + ti = XBraid.status_GetTIndex(status) + print("t: $(t[]),\tu: $(u[])\n") +end + +#= +all other usual wrapper functions are either not required, +or are set to reasonable defaults for operating on julia Arrays. 😎 +=# + +test = false +if test + test_app = XBraid.BraidApp(nothing, comm, my_step!, my_init, my_access) + + open("ex_01.test.out", "w") do file + cfile = Libc.FILE(file) + XBraid.testInitAccess(test_app, 0.0, cfile) + XBraid.testClone(test_app, 0.0, cfile) + XBraid.testSum(test_app, 0.0, cfile) + XBraid.testSpatialNorm(test_app, 0.0, cfile) + XBraid.testBuf(test_app, 0.0, cfile) + end + MPI.Barrier(comm) +end + +ntime = 10 +tstart = 0.0 +tstop = tstart + ntime / 2.0; +core = XBraid.Init(comm, tstart, tstop, ntime, my_step!, my_init, my_access) + +XBraid.SetPrintLevel(core, 2) +XBraid.SetMaxLevels(core, 2) +XBraid.SetAbsTol(core, 1.e-6) +XBraid.SetCFactor(core, -1, 2) + +XBraid.Drive(core) + +# no need for braid_Destroy(core), julia will take care of it :) +MPI.Finalize() \ No newline at end of file diff --git a/makefile.inc b/makefile.inc index 0b246039..e9dad806 100644 --- a/makefile.inc +++ b/makefile.inc @@ -22,10 +22,11 @@ # #EHEADER********************************************************************** -# Three compile time options +# four compile time options # make debug=yes|no # make valgrind=yes|no # make sequential=yes|no +# make shared=yes|no # Was DEBUG specified? ifeq ($(debug),no) @@ -60,6 +61,7 @@ ifeq ($(findstring tux408,$(HOSTNAME)),tux408) MPICXX = mpicxx MPIF90 = mpif90 LFLAGS = -lm + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -lm -fPIC CXXFLAGS = -g -Wall -Wno-reorder -lm -fPIC @@ -80,6 +82,7 @@ else ifeq ($(shell uname -s), Darwin) MPICXX = mpicxx MPIF90 = mpif90 LFLAGS = -lm -lstdc++ + SHAREDFLAGS = -shared -undefined dynamic_lookup ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -fPIC CXXFLAGS = -g -Wall -fPIC @@ -95,6 +98,7 @@ else ifeq ($(findstring cab,$(HOSTNAME)),cab) MPICXX = mpiicpc MPIF90 = mpif90 LFLAGS = -lm + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -fPIC CXXFLAGS = -g -Wall -fPIC @@ -110,6 +114,7 @@ else ifeq ($(findstring vulcan,$(HOSTNAME)),vulcan) MPICXX = mpixlcxx MPIF90 = mpixlf90 LFLAGS = -lm + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -fPIC CXXFLAGS = -g -Wall -fPIC @@ -126,6 +131,7 @@ else ifeq ($(findstring cadaverous,$(HOSTNAME)),cadaverous) MPICXX = /usr/bin/mpicxx MPIF90 = /usr/bin/mpif90 LFLAGS = -lm + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -fPIC CXXFLAGS = -g -Wall -fPIC @@ -140,6 +146,7 @@ else ifeq ($(shell uname -s),Linux) MPICXX = mpicxx MPIF90 = mpif90 LFLAGS = -lm + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -fPIC CXXFLAGS = -g -Wall -fPIC @@ -157,6 +164,7 @@ else MPICXX = mpicxx MPIF90 = mpif90 LFLAGS = -lm + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -fPIC CXXFLAGS = -g -Wall -fPIC @@ -174,6 +182,7 @@ ifeq ($(sequential),YES) MPICXX = g++ MPIF90 = gfortran LFLAGS = -lm + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -D braid_SEQUENTIAL CXXFLAGS = -g -Wall -D braid_SEQUENTIAL From bd1f08423ef740dcaf16b81ddbea53b4cee9d6f9 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 24 May 2023 18:52:19 -0600 Subject: [PATCH 02/12] Added README for julia example directory --- examples/julia/README.md | 35 +++++++++++++++++++++++++++++++++++ examples/julia/ex-01.jl | 21 +++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 examples/julia/README.md diff --git a/examples/julia/README.md b/examples/julia/README.md new file mode 100644 index 00000000..97fdc1a9 --- /dev/null +++ b/examples/julia/README.md @@ -0,0 +1,35 @@ + + +To run examples in this directory: +1. build braid as a shared library: + + shell> make braid shared=yes + +2. install and configure MPI.jl: https://juliaparallel.org/MPI.jl/v0.20/ + + julia> using Pkg; Pkg.add(["MPI, MPIPreferences]") + julia> using MPIPreferences; MPIPreferences.use_system_binary() + +3. run from the command line: + + shell> mpirun -n 4 ex-01.jl \ No newline at end of file diff --git a/examples/julia/ex-01.jl b/examples/julia/ex-01.jl index aa281308..808fb227 100644 --- a/examples/julia/ex-01.jl +++ b/examples/julia/ex-01.jl @@ -1,3 +1,24 @@ +#=BHEADER********************************************************************** + * Copyright (c) 2013, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * This file is part of XBraid. For support, post issues to the XBraid Github page. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License (as published by the Free Software + * Foundation) version 2.1 dated February 1999. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the terms and conditions of the GNU General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + ***********************************************************************EHEADER=# + # main ex-01 # include("XBraid.jl") include("../../braid/braid.jl/XBraid.jl") From b984ff9f2437da1f488d5166ddbc35938d82ccd7 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Thu, 25 May 2023 13:51:01 -0600 Subject: [PATCH 03/12] Moved braid_Warmup function from braid_test to util --- braid/braid_test.c | 94 -------------------------------------------- braid/braid_test.h | 26 ------------- braid/util.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++ braid/util.h | 27 ++++++++++++- 4 files changed, 123 insertions(+), 121 deletions(-) diff --git a/braid/braid_test.c b/braid/braid_test.c index ac44a60c..2f218a53 100644 --- a/braid/braid_test.c +++ b/braid/braid_test.c @@ -1372,97 +1372,3 @@ braid_TestDelta(braid_App app, return correct; } - -braid_Int -braid_Warmup(braid_App app, - MPI_Comm comm_x, - braid_Real t, - braid_Real fdt, - braid_Real cdt, - braid_PtFcnInit init, - braid_PtFcnAccess access, - braid_PtFcnFree myfree, - braid_PtFcnClone clone, - braid_PtFcnSum sum, - braid_PtFcnSpatialNorm spatialnorm, - braid_PtFcnBufSize bufsize, - braid_PtFcnBufPack bufpack, - braid_PtFcnBufUnpack bufunpack, - braid_PtFcnSCoarsen coarsen, - braid_PtFcnSRefine refine, - braid_PtFcnStep step, - braid_PtFcnInitBasis init_basis, - braid_PtFcnInnerProd innerprod) -{ - braid_Vector u, v; - braid_Status status = _braid_CTAlloc(_braid_Status, 1); - braid_AccessStatus astatus = (braid_AccessStatus)status; - - /* init/access */ - _braid_AccessStatusInit(t, 0, 0.0, 0, 0, 0, 0, 0, 1, -1, NULL, astatus); - init(app, t, &u); - if(access != NULL) - { - access(app, u, astatus); - } - - braid_Int size; - braid_Real throwaway; - void *buffer; - - /* bufsize, bufpack, bufunpack */ - braid_BufferStatus bstatus = (braid_BufferStatus)status; - _braid_BufferStatusInit(0, 0, 0, 0, bstatus); - bufsize(app, &size, bstatus); - buffer = malloc(size); - _braid_StatusElt(bstatus, size_buffer) = size; - bufpack(app, u, buffer, bstatus); - bufunpack(app, buffer, &v, bstatus); - free(buffer); - - /* free, clone, sum, spatialnorm */ - myfree(app, v); - clone(app, u, &v); - sum(app, 1.0, u, -1.0, v); - spatialnorm(app, v, &throwaway); - - /* step */ - braid_StepStatus sstatus = (braid_StepStatus)status; - _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, NULL, sstatus); - step(app, v, NULL, u, sstatus); - - /* coarsen, refine */ - if ( (coarsen != NULL) && (refine != NULL) ) - { - braid_CoarsenRefStatus cstatus = (braid_CoarsenRefStatus)status; - - _braid_CoarsenRefStatusInit(t, t-fdt, t+fdt, t-cdt, t+cdt, 0, 0, 0, 0, cstatus); - myfree(app, v); - coarsen(app, u, &v, cstatus); - myfree(app, u); - refine(app, v, &u, cstatus); - } - - /* Delta functions: init_basis, innerprod, step with basis */ - if ( (init_basis != NULL) && (innerprod != NULL)) - { - braid_Basis B; - B = _braid_TAlloc(_braid_Basis, 1); - B->rank = 1; - B->userVecs = _braid_TAlloc(braid_Vector, 1); - init_basis(app, t, 0, &(B->userVecs[0])); - innerprod(app, u, B->userVecs[0], &throwaway); - _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, B, sstatus); - step(app, v, NULL, u, sstatus); - - myfree(app, B->userVecs[0]); - _braid_TFree(B); - } - - /* Free variables */ - myfree(app, u); - myfree(app, v); - _braid_StatusDestroy(status); - - return _braid_error_flag; -} \ No newline at end of file diff --git a/braid/braid_test.h b/braid/braid_test.h index 44d250fc..4db8add1 100644 --- a/braid/braid_test.h +++ b/braid/braid_test.h @@ -308,32 +308,6 @@ braid_TestDelta(braid_App app, /**< User defined App stru braid_PtFcnStep mystep /**< Compute a time step with a braid_Vector */ ); -/** - * Warmup calls all user functions once with sensible arguments. - * This is especially useful for the Julia wrapper, since Julia is JIT compiled, - * and this force compiles all the functions used in braid_Drive. - */ -braid_Int -braid_Warmup( braid_App app, /**< User defined App structure */ - MPI_Comm comm_x, /**< Spatial communicator */ - braid_Real t, /**< Time value to initialize test vectors with*/ - braid_Real fdt, /**< Fine time step value that you spatially coarsen from */ - braid_Real cdt, /**< Coarse time step value that you coarsen to */ - braid_PtFcnInit init, /**< Initialize a braid_Vector on finest temporal grid*/ - braid_PtFcnAccess access, /**< Allows access to XBraid and current braid_Vector (can be NULL for no writing)*/ - braid_PtFcnFree free, /**< Free a braid_Vector*/ - braid_PtFcnClone clone, /**< Clone a braid_Vector */ - braid_PtFcnSum sum, /**< Compute vector sum of two braid_Vectors */ - braid_PtFcnSpatialNorm spatialnorm, /**< Compute norm of a braid_Vector, this is a norm only over space */ - braid_PtFcnBufSize bufsize, /**< Computes size in bytes for one braid_Vector MPI buffer */ - braid_PtFcnBufPack bufpack, /**< Packs MPI buffer to contain one braid_Vector */ - braid_PtFcnBufUnpack bufunpack, /**< Unpacks MPI buffer into a braid_Vector */ - braid_PtFcnSCoarsen coarsen, /**< Spatially coarsen a vector. If NULL, skipped.*/ - braid_PtFcnSRefine refine, /**< Spatially refine a vector. If NULL, skipped.*/ - braid_PtFcnStep step, /**< Compute a time step with a braid_Vector */ - braid_PtFcnInitBasis init_basis, /**< Initialize a basis vector at time t and spatial index i. If NULL, skipped */ - braid_PtFcnInnerProd innerprod /**< Compute the inner product between two braid_Vectors. If NULL, skipped. */ - ); /** @}*/ #ifdef __cplusplus diff --git a/braid/util.c b/braid/util.c index 2a1b15aa..3bc9b233 100644 --- a/braid/util.c +++ b/braid/util.c @@ -283,3 +283,100 @@ _braid_MPI_Wtime(braid_Core core, braid_Int timing_level) return -1.0; } } + +/*---------------------------------------------------------------------------- + *----------------------------------------------------------------------------*/ + +braid_Int +braid_Warmup(braid_App app, + MPI_Comm comm_x, + braid_Real t, + braid_Real fdt, + braid_Real cdt, + braid_PtFcnInit init, + braid_PtFcnAccess access, + braid_PtFcnFree myfree, + braid_PtFcnClone clone, + braid_PtFcnSum sum, + braid_PtFcnSpatialNorm spatialnorm, + braid_PtFcnBufSize bufsize, + braid_PtFcnBufPack bufpack, + braid_PtFcnBufUnpack bufunpack, + braid_PtFcnSCoarsen coarsen, + braid_PtFcnSRefine refine, + braid_PtFcnStep step, + braid_PtFcnInitBasis init_basis, + braid_PtFcnInnerProd innerprod) +{ + braid_Vector u, v; + braid_Status status = _braid_CTAlloc(_braid_Status, 1); + braid_AccessStatus astatus = (braid_AccessStatus)status; + + /* init/access */ + _braid_AccessStatusInit(t, 0, 0.0, 0, 0, 0, 0, 0, 1, -1, NULL, astatus); + init(app, t, &u); + if(access != NULL) + { + access(app, u, astatus); + } + + braid_Int size; + braid_Real throwaway; + void *buffer; + + /* bufsize, bufpack, bufunpack */ + braid_BufferStatus bstatus = (braid_BufferStatus)status; + _braid_BufferStatusInit(0, 0, 0, 0, bstatus); + bufsize(app, &size, bstatus); + buffer = malloc(size); + _braid_StatusElt(bstatus, size_buffer) = size; + bufpack(app, u, buffer, bstatus); + bufunpack(app, buffer, &v, bstatus); + free(buffer); + + /* free, clone, sum, spatialnorm */ + myfree(app, v); + clone(app, u, &v); + sum(app, 1.0, u, -1.0, v); + spatialnorm(app, v, &throwaway); + + /* step */ + braid_StepStatus sstatus = (braid_StepStatus)status; + _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, NULL, sstatus); + step(app, v, NULL, u, sstatus); + + /* coarsen, refine */ + if ( (coarsen != NULL) && (refine != NULL) ) + { + braid_CoarsenRefStatus cstatus = (braid_CoarsenRefStatus)status; + + _braid_CoarsenRefStatusInit(t, t-fdt, t+fdt, t-cdt, t+cdt, 0, 0, 0, 0, cstatus); + myfree(app, v); + coarsen(app, u, &v, cstatus); + myfree(app, u); + refine(app, v, &u, cstatus); + } + + /* Delta functions: init_basis, innerprod, step with basis */ + if ( (init_basis != NULL) && (innerprod != NULL)) + { + braid_Basis B; + B = _braid_TAlloc(_braid_Basis, 1); + B->rank = 1; + B->userVecs = _braid_TAlloc(braid_Vector, 1); + init_basis(app, t, 0, &(B->userVecs[0])); + innerprod(app, u, B->userVecs[0], &throwaway); + _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, B, sstatus); + step(app, v, NULL, u, sstatus); + + myfree(app, B->userVecs[0]); + _braid_TFree(B); + } + + /* Free variables */ + myfree(app, u); + myfree(app, v); + _braid_StatusDestroy(status); + + return _braid_error_flag; +} \ No newline at end of file diff --git a/braid/util.h b/braid/util.h index 8f6d2a8a..d9916699 100644 --- a/braid/util.h +++ b/braid/util.h @@ -127,6 +127,31 @@ _braid_GetNEntries(braid_Real *_array, braid_Real _braid_MPI_Wtime(braid_Core core, braid_Int timing_level); - +/** + * Warmup calls all user functions once with sensible arguments. + * This is especially useful for the Julia wrapper, since Julia is JIT compiled, + * and this force compiles all the functions used in braid_Drive. + */ +braid_Int +braid_Warmup( braid_App app, /**< User defined App structure */ + MPI_Comm comm_x, /**< Spatial communicator */ + braid_Real t, /**< Time value to initialize test vectors with*/ + braid_Real fdt, /**< Fine time step value that you spatially coarsen from */ + braid_Real cdt, /**< Coarse time step value that you coarsen to */ + braid_PtFcnInit init, /**< Initialize a braid_Vector on finest temporal grid*/ + braid_PtFcnAccess access, /**< Allows access to XBraid and current braid_Vector (can be NULL for no writing)*/ + braid_PtFcnFree free, /**< Free a braid_Vector*/ + braid_PtFcnClone clone, /**< Clone a braid_Vector */ + braid_PtFcnSum sum, /**< Compute vector sum of two braid_Vectors */ + braid_PtFcnSpatialNorm spatialnorm, /**< Compute norm of a braid_Vector, this is a norm only over space */ + braid_PtFcnBufSize bufsize, /**< Computes size in bytes for one braid_Vector MPI buffer */ + braid_PtFcnBufPack bufpack, /**< Packs MPI buffer to contain one braid_Vector */ + braid_PtFcnBufUnpack bufunpack, /**< Unpacks MPI buffer into a braid_Vector */ + braid_PtFcnSCoarsen coarsen, /**< Spatially coarsen a vector. If NULL, skipped.*/ + braid_PtFcnSRefine refine, /**< Spatially refine a vector. If NULL, skipped.*/ + braid_PtFcnStep step, /**< Compute a time step with a braid_Vector */ + braid_PtFcnInitBasis init_basis, /**< Initialize a basis vector at time t and spatial index i. If NULL, skipped */ + braid_PtFcnInnerProd innerprod /**< Compute the inner product between two braid_Vectors. If NULL, skipped. */ + ); #endif From 70a55ac3f5edcb738a1644d2bfe2e57a923edd26 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 31 May 2023 00:51:29 -0600 Subject: [PATCH 04/12] implemented adaptive theta method in julia added sync function to Julia wrapper moved julia utility functions into BraidUtils.jl moved wrapper_functions.jl into Wrappers.jl created new namespace for Status structures added calling_function constants to status.jl added GetCFactor function to braid_status.c --- braid/braid.jl/BraidUtils.jl | 54 + .../{wrapper_functions.jl => Wrappers.jl} | 228 +++-- braid/braid.jl/XBraid.jl | 952 +++++++++--------- braid/braid.jl/status.jl | 155 ++- braid/braid_status.c | 31 +- braid/braid_status.h | 12 + braid/util.c | 4 +- .../adaptive_theta/build_function_sundials.jl | 101 ++ .../julia/adaptive_theta/butcher-tables.jl | 260 +++++ .../adaptive_theta/drive-adaptive-theta.jl | 265 +++++ drivers/julia/drive-TaylorGreen.jl | 6 +- drivers/julia/drive-advdiff-theta.jl | 16 +- drivers/julia/drive-burgers.jl | 100 +- drivers/julia/drive-kflow.jl | 116 +-- drivers/julia/matrix_stencils.ipynb | 312 ++++++ examples/julia/ex-01.jl | 14 +- 16 files changed, 1849 insertions(+), 777 deletions(-) create mode 100644 braid/braid.jl/BraidUtils.jl rename braid/braid.jl/{wrapper_functions.jl => Wrappers.jl} (57%) create mode 100644 drivers/julia/adaptive_theta/build_function_sundials.jl create mode 100644 drivers/julia/adaptive_theta/butcher-tables.jl create mode 100644 drivers/julia/adaptive_theta/drive-adaptive-theta.jl create mode 100644 drivers/julia/matrix_stencils.ipynb diff --git a/braid/braid.jl/BraidUtils.jl b/braid/braid.jl/BraidUtils.jl new file mode 100644 index 00000000..9e6746a3 --- /dev/null +++ b/braid/braid.jl/BraidUtils.jl @@ -0,0 +1,54 @@ +module BraidUtils +export libbraid, c_stdout, BlackHoleBuffer, malloc_null_double_ptr, stacktrace_warn + +libbraid = joinpath(dirname(@__DIR__), "libbraid.so") +c_stdout = Libc.FILE(Libc.RawFD(1), "w") # corresponds to C standard output + + +""" +Allocates a valid pointer to a pointer of type T +""" +function malloc_null_double_ptr(T::Type) + pp = Base.Libc.malloc(sizeof(Ptr{Cvoid})) + pp = reinterpret(Ptr{Ptr{T}}, pp) + return pp +end + +""" +Displays a caught error message, including stacktrace, without throwing +""" +function stacktrace_warn(msg::String, err) + err_msg = sprint(showerror, err) + trace = sprint((io,v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) + @warn "$(msg):\n$(err_msg)\n$(trace)" +end + +""" +Where'd all my data go? +This extends Base to include a buffer which throws away the data written to it +(useful for measuring the serialized size of an object) +""" +mutable struct BlackHoleBuffer <: IO + ptr::Int +end +BlackHoleBuffer() = BlackHoleBuffer(0) + +function Base.read(from::BlackHoleBuffer, T::Type{UInt8}) + throw(ArgumentError("BlackHoleBuffer is not readable)")) +end +function Base.write(to::BlackHoleBuffer, x::UInt8) + to.ptr += 1 + return 1 +end +function Base.write(to::BlackHoleBuffer, x::Array{T}) where T + to.ptr += sizeof(x) + return sizeof(x) +end + +end # module BraidUtils + +# helper functions +isCPoint(i::Integer, cfactor::Integer)::Bool = ((i-1) % cfactor == 0) +isFPoint(i::Integer, cfactor::Integer)::Bool = !isCPoint(i, cfactor) +mapFineToCoarse(i::Integer, cfactor::Integer)::Integer = (i-1) ÷ cfactor + 1 +mapCoarseToFine(i::Integer, cfactor::Integer)::Integer = (i-1) * cfactor + 1 \ No newline at end of file diff --git a/braid/braid.jl/wrapper_functions.jl b/braid/braid.jl/Wrappers.jl similarity index 57% rename from braid/braid.jl/wrapper_functions.jl rename to braid/braid.jl/Wrappers.jl index 60acf5ca..5e712290 100644 --- a/braid/braid.jl/wrapper_functions.jl +++ b/braid/braid.jl/Wrappers.jl @@ -19,88 +19,100 @@ * ***********************************************************************EHEADER=# +module Wrappers + +using LinearAlgebra: norm2, dot +using Serialization: serialize, deserialize +using ..XBraid.BraidUtils +using ..XBraid: Status, BraidApp, BraidVector + +# useful helpers + """ -Displays a caught error message, including stacktrace, without throwing +Macro which wraps the function call in a try-catch block which prints the stacktrace without throwing, +since Julia functions should not throw errors when called from braid """ -function stacktrace_warn(msg::String, err) - err_msg = sprint(showerror, err) - trace = sprint((io,v) -> show(io, "text/plain", v), stacktrace(catch_backtrace())) - @warn "$(msg):\n$(err_msg)\n$(trace)" +macro braidWrapper(expr) + # this error is at compile time, and therefore fine + expr.head == :function || error("Not a function expression.") + funcname = expr.args[1].args[1].args[1] + user_funcname = split(string(funcname), "_")[end] + :( + function $(esc(funcname))(args...; kwargs...)::Cint + f = $expr + try + result = f(args...; kwargs...) + return result + catch err + stacktrace_warn("Error in user function " * $(esc(user_funcname)), err) + return 1 + end + end + ) end """ -Where'd all my data go? -This extends Base to include a buffer which throws away the data written to it -(useful for measuring the serialized size of an object) +Used to add a reference to the vector u to the IdDict stored in the app. +this keeps all newly allocated braid_vectors in the global scope, preventing them from being garbage collected. """ -mutable struct BlackHoleBuffer <: IO - ptr::Int +function _register_vector(app::BraidApp, u::BraidVector) + app.ref_ids[objectid(u)] = u end -BlackHoleBuffer() = BlackHoleBuffer(0) -function Base.read(from::BlackHoleBuffer, T::Type{UInt8}) - throw(ArgumentError("BlackHoleBuffer is not readable)")) -end -function Base.write(to::BlackHoleBuffer, x::UInt8) - to.ptr += 1 - return 1 -end -function Base.write(to::BlackHoleBuffer, x::Array{T}) where T - to.ptr += sizeof(x) - return sizeof(x) +""" +Used to remove a reference to the vector u to the IdDict stored in the app. +""" +function _deregister_vector(app::BraidApp, u::BraidVector) + id = objectid(u)::UInt64 + pop!(app.ref_ids, id) end # these are internal functions which directly interface with XBraid -function _jl_step!(_app::Ptr{Cvoid}, - _ustop::Ptr{Cvoid}, - _fstop::Ptr{Cvoid}, - _u::Ptr{Cvoid}, - _status::Ptr{Cvoid})::Cint + +# Step, Init, and Access are required + +@braidWrapper function _jl_step!(_app::Ptr{Cvoid}, + _ustop::Ptr{Cvoid}, + _fstop::Ptr{Cvoid}, + _u::Ptr{Cvoid}, + _status::Ptr{Cvoid} +)::Cint # println("step") app = unsafe_pointer_to_objref(_app)::BraidApp u = unsafe_pointer_to_objref(_u)::BraidVector ustop = unsafe_pointer_to_objref(_ustop)::BraidVector - status = StepStatus(_status) + status = Status.StepStatus(_status) - tstart, tstop = status_GetTstartTstop(status) - delta_rank = status_GetDeltaRank(status) + tstart, tstop = Status.getTstartTstop(status) + delta_rank = Status.getDeltaRank(status) # call the user's function - try - # residual option - if _fstop !== C_NULL - fstop = unsafe_pointer_to_objref(_fstop)::BraidVector - app.step(app.user_app, status, u.user_vector, ustop.user_vector, fstop.user_vector, tstart, tstop) + # residual option + if _fstop !== C_NULL + fstop = unsafe_pointer_to_objref(_fstop)::BraidVector + app.step(app.user_app, status, u.user_vector, ustop.user_vector, fstop.user_vector, tstart, tstop) # Delta correction - elseif delta_rank > 0 - basis_vecs = status_GetBasisVectors(status) - app.step(app.user_app, status, u.user_vector, ustop.user_vector, tstart, tstop, basis_vecs) + elseif delta_rank > 0 + basis_vecs = Status.getBasisVectors(status) + app.step(app.user_app, status, u.user_vector, ustop.user_vector, tstart, tstop, basis_vecs) # Default - else - app.step(app.user_app, status, u.user_vector, ustop.user_vector, tstart, tstop) - end - catch err - stacktrace_warn("Error in user Step", err) + else + app.step(app.user_app, status, u.user_vector, ustop.user_vector, tstart, tstop) end return 0 end precompile(_jl_step!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) _c_step = @cfunction(_jl_step!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) +export _c_step, _jl_step! - -function _jl_init!(_app::Ptr{Cvoid}, t::Cdouble, u_ptr::Ptr{Ptr{Cvoid}})::Cint +@braidWrapper function _jl_init!(_app::Ptr{Cvoid}, t::Cdouble, u_ptr::Ptr{Ptr{Cvoid}})::Cint # println("init") app = unsafe_pointer_to_objref(_app)::BraidApp + # initialize u and register a reference with IdDict - u = nothing - try - u = BraidVector(app.init(app.user_app, t)) - catch err - stacktrace_warn("Error in user Init", err) - return 1 - end + u = BraidVector(app.init(app.user_app, t)) _register_vector(app, u) unsafe_store!(u_ptr, pointer_from_objref(u)) @@ -118,27 +130,16 @@ function _jl_init!(_app::Ptr{Cvoid}, t::Cdouble, u_ptr::Ptr{Ptr{Cvoid}})::Cint app.user_VecType = u.VecType end - # TODO: figure out how to put an upper bound on the size of the serialized object - # without serializing it first - # u_size = Base.summarysize(Ref(u)) + 9 - # if u_size > app.bufsize - # app.bufsize = u_size - # end - return 0 end _c_init = @cfunction(_jl_init!, Cint, (Ptr{Cvoid}, Cdouble, Ptr{Ptr{Cvoid}})) +export _c_init, _jl_init! -function _jl_init_basis!(_app::Ptr{Cvoid}, t::Cdouble, index::Cint, u_ptr::Ptr{Ptr{Cvoid}})::Cint +@braidWrapper function _jl_init_basis!(_app::Ptr{Cvoid}, t::Cdouble, index::Cint, u_ptr::Ptr{Ptr{Cvoid}})::Cint # println("init_basis") app = unsafe_pointer_to_objref(_app)::BraidApp - u = nothing - try - u = BraidVector(app.basis_init(app.user_app, t, index)) - catch err - stacktrace_warn("Error in user InitBasis", err) - return 1 - end + # julia uses 1-based indexing + u = BraidVector(app.basis_init(app.user_app, t, index + 1)) _register_vector(app, u) unsafe_store!(u_ptr, pointer_from_objref(u)) @@ -158,19 +159,14 @@ function _jl_init_basis!(_app::Ptr{Cvoid}, t::Cdouble, index::Cint, u_ptr::Ptr{P return 0 end _c_init_basis = @cfunction(_jl_init_basis!, Cint, (Ptr{Cvoid}, Cdouble, Cint, Ptr{Ptr{Cvoid}})) +export _c_init_basis, _jl_init_basis! -function _jl_clone!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, v_ptr::Ptr{Ptr{Cvoid}})::Cint +@braidWrapper function _jl_clone!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, v_ptr::Ptr{Ptr{Cvoid}})::Cint # println("clone") app = unsafe_pointer_to_objref(_app)::BraidApp u = unsafe_pointer_to_objref(_u)::BraidVector # initialize v, and copy u into v - v = nothing - try - v = deepcopy(u) - catch err - stacktrace_warn("Clone error", err) - return 1 - end + v = deepcopy(u) # then register v with IdDict and store in v_ptr _register_vector(app, v) @@ -180,8 +176,9 @@ function _jl_clone!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, v_ptr::Ptr{Ptr{Cvoid}})::C end precompile(_jl_clone!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}})) _c_clone = @cfunction(_jl_clone!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}})) +export _c_clone, _jl_clone! -function _jl_free!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid})::Cint +@braidWrapper function _jl_free!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid})::Cint # println("free") app = unsafe_pointer_to_objref(_app)::BraidApp u = unsafe_pointer_to_objref(_u)::BraidVector @@ -191,82 +188,72 @@ function _jl_free!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid})::Cint end precompile(_jl_free!, (Ptr{Cvoid}, Ptr{Cvoid})) _c_free = @cfunction(_jl_free!, Cint, (Ptr{Cvoid}, Ptr{Cvoid})) +export _c_free, _jl_free! -function _jl_sum!(_app::Ptr{Cvoid}, - alpha::Cdouble, _x::Ptr{Cvoid}, - beta::Cdouble, _y::Ptr{Cvoid})::Cint +@braidWrapper function _jl_sum!(_app::Ptr{Cvoid}, + alpha::Cdouble, _x::Ptr{Cvoid}, + beta::Cdouble, _y::Ptr{Cvoid})::Cint app = unsafe_pointer_to_objref(_app)::BraidApp x = unsafe_pointer_to_objref(_x)::BraidVector y = unsafe_pointer_to_objref(_y)::BraidVector - try - app.sum(app.user_app, alpha, x.user_vector, beta, y.user_vector) - catch err - stacktrace_warn("Error in user Sum", err) - end + app.sum(app.user_app, alpha, x.user_vector, beta, y.user_vector) return 0 end precompile(_jl_sum!, (Ptr{Cvoid}, Cdouble, Ptr{Cvoid}, Cdouble, Ptr{Cvoid})) _c_sum = @cfunction(_jl_sum!, Cint, (Ptr{Cvoid}, Cdouble, Ptr{Cvoid}, Cdouble, Ptr{Cvoid})) +export _c_sum, _jl_sum! -function _jl_norm!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, norm_ptr::Ptr{Cdouble})::Cint +@braidWrapper function _jl_norm!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, norm_ptr::Ptr{Cdouble})::Cint app = unsafe_pointer_to_objref(_app)::BraidApp u = unsafe_pointer_to_objref(_u)::BraidVector - norm = NaN - try - norm = app.spatialnorm(app.user_app, u.user_vector) - catch err - stacktrace_warn("Error in user Norm", err) - end + norm = app.spatialnorm(app.user_app, u.user_vector) unsafe_store!(norm_ptr, norm) return 0 end precompile(_jl_norm!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) _c_norm = @cfunction(_jl_norm!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) +export _c_norm, _jl_norm! -function _jl_inner_prod!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _v::Ptr{Cvoid}, norm_ptr::Ptr{Cdouble})::Cint +@braidWrapper function _jl_inner_prod!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _v::Ptr{Cvoid}, norm_ptr::Ptr{Cdouble})::Cint app = unsafe_pointer_to_objref(_app)::BraidApp u = unsafe_pointer_to_objref(_u)::BraidVector v = unsafe_pointer_to_objref(_v)::BraidVector - prod = NaN - try - prod = app.inner_prod(app.user_app, u.user_vector, v.user_vector) - catch err - stacktrace_warn("Error in user InnerProd", err) - end + prod = app.inner_prod(app.user_app, u.user_vector, v.user_vector) unsafe_store!(norm_ptr, prod) return 0 end precompile(_jl_inner_prod!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) _c_inner_prod = @cfunction(_jl_inner_prod!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cdouble})) +export _c_inner_prod, _jl_inner_prod! -function _jl_access!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _status::Ptr{Cvoid})::Cint +@braidWrapper function _jl_access!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _status::Ptr{Cvoid})::Cint app = unsafe_pointer_to_objref(_app)::BraidApp - status = AccessStatus(_status) + status = Status.AccessStatus(_status) if !isnothing(app.access) u = unsafe_pointer_to_objref(_u)::BraidVector - try - app.access(app.user_app, status, u.user_vector) - catch err - stacktrace_warn("Error in user Access", err) - end + app.access(app.user_app, status, u.user_vector) end return 0 end precompile(_jl_access!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) _c_access = @cfunction(_jl_access!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) +export _c_access, _jl_access! -function _jl_bufsize!(_app::Ptr{Cvoid}, size_ptr::Ptr{Cint}, status::Ptr{Cvoid})::Cint +# buffer functions + +@braidWrapper function _jl_bufsize!(_app::Ptr{Cvoid}, size_ptr::Ptr{Cint}, status::Ptr{Cvoid})::Cint app = unsafe_pointer_to_objref(_app) unsafe_store!(size_ptr, app.bufsize) return 0 end _c_bufsize = @cfunction(_jl_bufsize!, Cint, (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cvoid})) +export _c_bufsize, _jl_bufsize! -function _jl_bufpack!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, status::Ptr{Cvoid})::Cint +@braidWrapper function _jl_bufpack!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, status::Ptr{Cvoid})::Cint app = unsafe_pointer_to_objref(_app)::BraidApp u = unsafe_pointer_to_objref(_u)::BraidVector buff_arr = unsafe_wrap(Vector{UInt8}, Base.unsafe_convert(Ptr{UInt8}, _buffer), app.bufsize) @@ -287,8 +274,9 @@ function _jl_bufpack!(_app::Ptr{Cvoid}, _u::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, sta end precompile(_jl_bufpack!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) _c_bufpack = @cfunction(_jl_bufpack!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) +export _c_bufpack, _jl_bufpack! -function _jl_bufunpack!(_app::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, u_ptr::Ptr{Ptr{Cvoid}}, status::Ptr{Cvoid})::Cint +@braidWrapper function _jl_bufunpack!(_app::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, u_ptr::Ptr{Ptr{Cvoid}}, status::Ptr{Cvoid})::Cint @assert _buffer !== C_NULL "tried to unpack null buffer" app = unsafe_pointer_to_objref(_app)::BraidApp # get size of buffer we are unpacking: @@ -308,3 +296,29 @@ function _jl_bufunpack!(_app::Ptr{Cvoid}, _buffer::Ptr{Cvoid}, u_ptr::Ptr{Ptr{Cv end precompile(_jl_bufunpack!, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}}, Ptr{Cvoid})) _c_bufunpack = @cfunction(_jl_bufunpack!, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Ptr{Cvoid}}, Ptr{Cvoid})) +export _c_bufunpack, _jl_bufunpack! + +# optional functions + +@braidWrapper function _jl_sync(_app::Ptr{Cvoid}, status::Ptr{Cvoid})::Cint + app = unsafe_pointer_to_objref(_app)::BraidApp + if app.sync !== nothing + app.sync(app.user_app, Status.SyncStatus(status)) + end + return 0 +end +precompile(_jl_sync, (Ptr{Cvoid}, Ptr{Cvoid})) +_c_sync = @cfunction(_jl_sync, Cint, (Ptr{Cvoid}, Ptr{Cvoid})) +export _c_sync, _jl_sync + +# default functions assuming the user's vector is a julia array +export default_sum!, default_norm, default_inner_prod + +function default_sum!(app, α::Real, x::AbstractArray, β::Real, y::AbstractArray) + y .= α .* x .+ β .* y +end +default_norm(app, u::AbstractArray) = norm2(u) +default_inner_prod(app, u::AbstractArray, v::AbstractArray) = dot(u, v) + + +end # module Wrappers \ No newline at end of file diff --git a/braid/braid.jl/XBraid.jl b/braid/braid.jl/XBraid.jl index 285342d2..bb8c5b5e 100644 --- a/braid/braid.jl/XBraid.jl +++ b/braid/braid.jl/XBraid.jl @@ -23,13 +23,12 @@ module XBraid # Not sure we want to export anything... # export Init, Drive, etc. -# BEGIN MODULE XBraid using Serialization: serialize, deserialize, Serializer # used to pack arbitrary julia objects into buffers -using LinearAlgebra: norm2 +using LinearAlgebra: norm2, dot using MPI -include("status.jl") -include("wrapper_functions.jl") +include("BraidUtils.jl") +using .BraidUtils #= | For this to work, XBraid must be compiled as a shared library. @@ -38,23 +37,16 @@ include("wrapper_functions.jl") | braid_Fortran_TimeGrid, and braid_Fortran_Sync must all be set to zero in | braid.h before compiling. (TODO: figure out why this is...) =# -libbraid = joinpath(dirname(@__DIR__), "libbraid.so") - -c_stdout = Libc.FILE(Libc.RawFD(1), "w") # corresponds to C standard output -function malloc_null_double_ptr(T::Type) - pp = Base.Libc.malloc(sizeof(Ptr{Cvoid})) - pp = reinterpret(Ptr{Ptr{T}}, pp) - return pp -end # This can contain anything (TODO: do I even need this?) mutable struct BraidVector{T} - user_vector::T - VecType::Type + user_vector::T + VecType::Type end -BraidVector(u::T) where T = BraidVector(u, T) +BraidVector(u::T) where {T} = BraidVector(u, T) + +OptionalFunction = Union{Function,Nothing} -OptionalFunction = Union{Function, Nothing} """ This is an internal structure that is not generally exposed to the user. This enables the user interface to only use memory safe function calls. @@ -74,64 +66,41 @@ julia> app = BraidApp(my_app, MPI.COMM_WORLD, MPI.COMM_WORLD, my_step, my_init, julia> testSpatialNorm(app, 0.); """ mutable struct BraidApp - user_app::Any # user defined app data structure + user_app::Any # user defined app data structure - comm::MPI.Comm # global mpi communicator - comm_t::MPI.Comm # temporal mpi communicator + comm::MPI.Comm # global mpi communicator + comm_t::MPI.Comm # temporal mpi communicator - # user functions - step::Function - init::Function - sum::Function - spatialnorm::Function - access::Function + # user functions + step::Function + init::Function + sum::Function + spatialnorm::Function + access::Function - basis_init::OptionalFunction - inner_prod::OptionalFunction - - # dictionary to store globally scoped references to allocated braid_vector objects - ref_ids::IdDict{UInt64, BraidVector} - bufsize::Integer # expected serialized size of user's vector - bufsize_lyap::Integer - user_AppType::Type - user_VecType::Type - user_BasType::Type -end - -# some default functions assuming the user's vector is a julia array -function default_sum!(app, α::Real, x::AbstractArray, β::Real, y::AbstractArray) - y .= α .* x .+ β .* y - return -end - -function default_norm(app, u::AbstractArray) - return norm2(u) -end + sync::OptionalFunction + basis_init::OptionalFunction + inner_prod::OptionalFunction -function default_inner_prod(app, u::AbstractArray, v::AbstractArray) - return u ⋅ v + # dictionary to store globally scoped references to allocated braid_vector objects + ref_ids::IdDict{UInt64,BraidVector} + bufsize::Integer # expected serialized size of user's vector + bufsize_lyap::Integer + user_AppType::Type + user_VecType::Type + user_BasType::Type end # default constructor -function BraidApp(app, comm::MPI.Comm, comm_t::MPI.Comm, step::Function, init::Function, sum::Function, norm::Function, access::Function, basis_init::Function, inner_prod::Function) - BraidApp(app, comm, comm_t, step, init, sum, norm, access, basis_init, inner_prod, IdDict(), 0, 0, Nothing, Nothing, Nothing) +function BraidApp(app, comm::MPI.Comm, comm_t::MPI.Comm, step::Function, init::Function, sum::Function, norm::Function, access::Function, sync::OptionalFunction, basis_init::OptionalFunction, inner_prod::OptionalFunction) + BraidApp(app, comm, comm_t, step, init, sum, norm, access, sync, basis_init, inner_prod, IdDict(), 0, 0, Nothing, Nothing, Nothing) end -function BraidApp(app, comm::MPI.Comm, comm_t::MPI.Comm, step::Function, init::Function, sum::Function, norm::Function, access::Function) - BraidApp(app, comm, comm_t, step, init, sum, norm, access, nothing, nothing, IdDict(), 0, 0, Nothing, Nothing, Nothing) +function BraidApp(app, comm::MPI.Comm, step, init, access; comm_t=comm, sum=default_sum!, spatialnorm=default_norm, sync=nothing, basis_init=nothing, inner_prod=default_inner_prod) + BraidApp(app, comm, comm_t, step, init, sum, spatialnorm, access, sync, basis_init, inner_prod) end -function BraidApp(app, comm::MPI.Comm, step::Function, init::Function, sum::Function, norm::Function, access::Function) - BraidApp(app, comm, comm, step, init, sum, norm, access) -end -function BraidApp(app, comm::MPI.Comm, step, init, access) - BraidApp(app, comm, comm, step, init, default_sum!, default_norm, access) -end - -function BraidApp(app, comm::MPI.Comm, step, init, access, basis_init) - BraidApp(app, comm, comm, step, init, default_sum!, default_norm, access, basis_init, default_inner_prod) -end """ Stores all the information needed to run XBraid. Create this object with Init(), then pass this to Drive() to run XBraid. @@ -142,132 +111,134 @@ julia> XBraid.Drive(core); """ mutable struct BraidCore - # internal values - _braid_core::Ptr{Cvoid} - _braid_app::BraidApp - tstart::Float64 - tstop::Float64 - ntime::Int32 - function BraidCore(_braid_core, _braid_app, tstart, tstop, ntime) - x = new(_braid_core, _braid_app, tstart, tstop, ntime) - finalizer(x) do core - # @async println("Destroying BraidCore $(core._braid_core)") - @ccall libbraid.braid_Destroy(core._braid_core::Ptr{Cvoid})::Cint - end - end -end - -""" -Used to add a reference to the vector u to the IdDict stored in the app. -this keeps all newly allocated braid_vectors in the global scope, preventing them from being garbage collected. -""" -function _register_vector(app::BraidApp, u::BraidVector) - app.ref_ids[objectid(u)] = u -end - -""" -Used to remove a reference to the vector u to the IdDict stored in the app. -""" -function _deregister_vector(app::BraidApp, u::BraidVector) - id = objectid(u)::UInt64 - pop!(app.ref_ids, id) -end + # internal values + _braid_core::Ptr{Cvoid} + _braid_app::BraidApp + tstart::Float64 + tstop::Float64 + ntime::Int32 + function BraidCore(_braid_core, _braid_app, tstart, tstop, ntime) + x = new(_braid_core, _braid_app, tstart, tstop, ntime) + finalizer(x) do core + # @async println("Destroying BraidCore $(core._braid_core)") + @ccall libbraid.braid_Destroy(core._braid_core::Ptr{Cvoid})::Cint + end + end +end + + +# import wrapper functions and status structures +include("Status.jl") +include("Wrappers.jl") +using .Status, .Wrappers function postInitPrecompile(app::BraidApp) - #= - # This is a hack to make sure every processor knows how big the user's vector will be. - # We can also take this time to precompile the user's step function so it doesn't happen - # in serial on the coarse grid. - =# - GC.enable(false) # disable garbage collection - app_ptr = pointer_from_objref(app)::Ptr{Cvoid} - pp = malloc_null_double_ptr(Cvoid) - - _jl_init!(app_ptr, 0., pp) - u_ptr = unsafe_load(pp) - u = unsafe_pointer_to_objref(u_ptr)::BraidVector - VecType = typeof(u.user_vector) - AppType = typeof(app.user_app) - - !precompile(app.step, (AppType, Ptr{Cvoid}, VecType, VecType, Float64, Float64)) && println("failed to precompile step") - !precompile(app.spatialnorm, (AppType, VecType)) && println("failed to precompile norm") - !precompile(app.sum, (AppType, Float64, VecType, Float64, VecType)) && println("failed to precompile sum") - if app.access !== nothing - !precompile(app.access, (AppType, Ptr{Cvoid}, VecType)) && println("failed to precompile access") - end - - # some julia functions that can be precompiled - precompile(deepcopy, (BraidVector{VecType},)) - precompile(unsafe_store!, (Ptr{Float64}, Float64,)) - precompile(Tuple{typeof(Base.getproperty), BraidVector{VecType}, Symbol}) - precompile(Tuple{typeof(Base.unsafe_store!), Ptr{Int32}, Int64}) - precompile(Tuple{typeof(deserialize), Serializer{Base.GenericIOBuffer{Array{UInt8, 1}}}, DataType}) - precompile(Tuple{typeof(Base.unsafe_wrap), Type{Array{UInt8, 1}}, Ptr{UInt8}, Int64}) - precompile(Tuple{Type{NamedTuple{(:read, :write, :maxsize), T} where T<:Tuple}, Tuple{Bool, Bool, Int64}}) - - _jl_free!(app_ptr, u_ptr) - Base.Libc.free(pp) - GC.enable(true) # re-enable garbage collection - - app.user_AppType = AppType - app.user_VecType = VecType + #= + # This is a hack to make sure every processor knows how big the user's vector will be. + # We can also take this time to precompile the user's step function so it doesn't happen + # in serial on the coarse grid. + =# + GC.enable(false) # disable garbage collection + app_ptr = pointer_from_objref(app)::Ptr{Cvoid} + pp = malloc_null_double_ptr(Cvoid) + + _jl_init!(app_ptr, 0.0, pp) + u_ptr = unsafe_load(pp) + u = unsafe_pointer_to_objref(u_ptr)::BraidVector + VecType = typeof(u.user_vector) + AppType = typeof(app.user_app) + + !precompile(app.step, (AppType, Status.StepStatus, VecType, VecType, Float64, Float64)) && println("failed to precompile step") + !precompile(app.spatialnorm, (AppType, VecType)) && println("failed to precompile norm") + !precompile(app.sum, (AppType, Float64, VecType, Float64, VecType)) && println("failed to precompile sum") + if app.access !== nothing + !precompile(app.access, (AppType, Status.AccessStatus, VecType)) && println("failed to precompile access") + end + + # some julia functions that can be precompiled + precompile(deepcopy, (BraidVector{VecType},)) + precompile(unsafe_store!, (Ptr{Float64}, Float64,)) + precompile(Tuple{typeof(Base.getproperty),BraidVector{VecType},Symbol}) + precompile(Tuple{typeof(Base.unsafe_store!),Ptr{Int32},Int64}) + precompile(Tuple{typeof(deserialize),Serializer{Base.GenericIOBuffer{Array{UInt8,1}}},DataType}) + precompile(Tuple{typeof(Base.unsafe_wrap),Type{Array{UInt8,1}},Ptr{UInt8},Int64}) + precompile(Tuple{Type{NamedTuple{(:read, :write, :maxsize),T} where T<:Tuple},Tuple{Bool,Bool,Int64}}) + + _jl_free!(app_ptr, u_ptr) + Base.Libc.free(pp) + GC.enable(true) # re-enable garbage collection + + app.user_AppType = AppType + app.user_VecType = VecType end function deltaPrecompile(app::BraidApp) - GC.enable(false) - app_ptr = pointer_from_objref(app)::Ptr{Cvoid} - pp = malloc_null_double_ptr(Cvoid) - AppType = app.user_AppType - VecType = app.user_VecType - - _jl_init_basis!(app_ptr, 0., Int32(0), pp) - ψ_ptr = unsafe_load(pp) - ψ = unsafe_pointer_to_objref(ψ_ptr) - BasType = typeof(ψ.user_vector) - println("precompiling user Delta functions") - !precompile(app.step, (AppType, Ptr{Cvoid}, VecType, VecType, Float64, Float64, Vector{BasType})) && println("failed to compile step_du") - !precompile(app.inner_prod, (AppType, VecType, VecType)) && println("failed to compile inner_prod u⋅u") - !precompile(app.inner_prod, (AppType, BasType, VecType)) && println("failed to compile inner_prod ψ⋅u") - !precompile(app.inner_prod, (AppType, VecType, BasType)) && println("failed to compile inner_prod u⋅ψ") - !precompile(app.inner_prod, (AppType, BasType, BasType)) && println("failed to compile inner_prod ψ⋅ψ") - - _jl_free!(app_ptr, ψ_ptr) - Base.Libc.free(pp) - GC.enable(true) - - app.user_BasType = BasType + GC.enable(false) + app_ptr = pointer_from_objref(app)::Ptr{Cvoid} + pp = malloc_null_double_ptr(Cvoid) + AppType = app.user_AppType + VecType = app.user_VecType + + _jl_init_basis!(app_ptr, 0.0, Int32(0), pp) + ψ_ptr = unsafe_load(pp) + ψ = unsafe_pointer_to_objref(ψ_ptr) + BasType = typeof(ψ.user_vector) + !precompile(app.step, (AppType, Status.StepStatus, VecType, VecType, Float64, Float64, Vector{BasType})) && println("failed to compile step_du") + !precompile(app.inner_prod, (AppType, VecType, VecType)) && println("failed to compile inner_prod u⋅u") + !precompile(app.inner_prod, (AppType, BasType, VecType)) && println("failed to compile inner_prod ψ⋅u") + !precompile(app.inner_prod, (AppType, VecType, BasType)) && println("failed to compile inner_prod u⋅ψ") + !precompile(app.inner_prod, (AppType, BasType, BasType)) && println("failed to compile inner_prod ψ⋅ψ") + + _jl_free!(app_ptr, ψ_ptr) + Base.Libc.free(pp) + GC.enable(true) + + app.user_BasType = BasType end """ Create a new BraidCore object. The BraidCore object is used to run the XBraid solver, and is destroyed when the object is garbage collected. """ -function Init(comm_world::MPI.Comm, comm_t::MPI.Comm, - tstart::Real, tstop::Real, ntime::Integer, - step::Function, init::Function, sum::Function, spatialnorm::Function, access::OptionalFunction; - app = nothing +function Init( + comm_world::MPI.Comm, + tstart::Real, + tstop::Real, + ntime::Integer, + step::Function, + init::Function, + access::Function; + sum = default_sum!, + spatialnorm = default_norm, + sync = nothing, + comm_t::MPI.Comm = comm_world, + app = nothing )::BraidCore - _app = BraidApp(app, comm_world, comm_t, step, init, sum, spatialnorm, access) - _core_ptr = malloc_null_double_ptr(Cvoid) - - GC.@preserve _core_ptr begin - @ccall libbraid.braid_Init( - _app.comm::MPI.MPI_Comm, _app.comm_t::MPI.MPI_Comm, - tstart::Cdouble, tstop::Cdouble, ntime::Cint, - _app::Ref{BraidApp}, _c_step::Ptr{Cvoid}, _c_init::Ptr{Cvoid}, _c_clone::Ptr{Cvoid}, - _c_free::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, - _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, - _core_ptr::Ptr{Ptr{Cvoid}}, - )::Cint - end - - _core = unsafe_load(_core_ptr) - Base.Libc.free(_core_ptr) - - return BraidCore(_core, _app, tstart, tstop, ntime) -end - -function Init(comm_world::MPI.Comm, tstart::Real, tstop::Real, ntime::Integer, step::Function, init::Function, access::OptionalFunction; app=nothing) - Init(comm_world, comm_world, tstart, tstop, ntime, step, init, default_sum!, default_norm, access; app=app) + _app = BraidApp(app, comm_world, comm_t, step, init, sum, spatialnorm, access, sync, nothing, nothing) + _core_ptr = malloc_null_double_ptr(Cvoid) + + GC.@preserve _core_ptr begin + @ccall libbraid.braid_Init( + _app.comm::MPI.MPI_Comm, _app.comm_t::MPI.MPI_Comm, + tstart::Cdouble, tstop::Cdouble, ntime::Cint, + _app::Ref{BraidApp}, _c_step::Ptr{Cvoid}, _c_init::Ptr{Cvoid}, _c_clone::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, + _core_ptr::Ptr{Ptr{Cvoid}}, + )::Cint + end + + _core = unsafe_load(_core_ptr) + Base.Libc.free(_core_ptr) + + if sync !== nothing + GC.@preserve _core begin + @ccall libbraid.braid_SetSync( + _core::Ptr{Cvoid}, _c_sync::Ptr{Cvoid} + )::Cint + end + end + + return BraidCore(_core, _app, tstart, tstop, ntime) end """ @@ -282,245 +253,243 @@ Julia> XBraid.Drive(core; warmup=false) See also: XBraid.Drive """ function Warmup(core::BraidCore) - _app = core._braid_app - # precompile all user functions by calling them from braid_Warmup - # if any user functions have side-effects, this may behave unexpectedly - fdt = (core.tstop - core.tstart) / (core.ntime+1) - cdt = 2fdt - - if (_app.basis_init !== nothing) && (_app.inner_prod !== nothing) - @ccall libbraid.braid_Warmup( - _app::Ref{BraidApp}, _app.comm::MPI.MPI_Comm, core.tstart::Cdouble, fdt::Cdouble, cdt::Cdouble, - _c_init::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, _c_free::Ptr{Cvoid}, - _c_clone::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, - _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, - C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid}, _c_step::Ptr{Cvoid}, - _c_init_basis::Ptr{Cvoid}, _c_inner_prod::Ptr{Cvoid} - )::Cint - else - @ccall libbraid.braid_Warmup( - _app::Ref{BraidApp}, _app.comm::MPI.MPI_Comm, core.tstart::Cdouble, fdt::Cdouble, cdt::Cdouble, - _c_init::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, _c_free::Ptr{Cvoid}, - _c_clone::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, - _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, - C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid}, _c_step::Ptr{Cvoid}, - C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid} - )::Cint - end - nothing + _app = core._braid_app + # precompile all user functions by calling them from braid_Warmup + # if any user functions have side-effects, this may behave unexpectedly + fdt = (core.tstop - core.tstart) / (core.ntime + 1) + cdt = 2fdt + + if (_app.basis_init !== nothing) && (_app.inner_prod !== nothing) + @ccall libbraid.braid_Warmup( + _app::Ref{BraidApp}, _app.comm::MPI.MPI_Comm, core.tstart::Cdouble, fdt::Cdouble, cdt::Cdouble, + _c_init::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, + C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid}, _c_step::Ptr{Cvoid}, + _c_init_basis::Ptr{Cvoid}, _c_inner_prod::Ptr{Cvoid} + )::Cint + else + @ccall libbraid.braid_Warmup( + _app::Ref{BraidApp}, _app.comm::MPI.MPI_Comm, core.tstart::Cdouble, fdt::Cdouble, cdt::Cdouble, + _c_init::Ptr{Cvoid}, _c_access::Ptr{Cvoid}, _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, _c_sum::Ptr{Cvoid}, _c_norm::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, _c_bufpack::Ptr{Cvoid}, _c_bufunpack::Ptr{Cvoid}, + C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid}, _c_step::Ptr{Cvoid}, + C_NULL::Ptr{Cvoid}, C_NULL::Ptr{Cvoid} + )::Cint + end + nothing end """ Wraps the XBraid braid_Drive function. This function is called by XBraid.Drive, and should not be called directly. """ function _Drive(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_Drive(core._braid_core::Ptr{Cvoid})::Cint - end + GC.@preserve core begin + @ccall libbraid.braid_Drive(core._braid_core::Ptr{Cvoid})::Cint + end end """ Run the XBraid solver. This function calls XBraid.Warmup by default, but this can be disabled by setting warmup=false, in which case a less expensive (but less effective) option is used to precompile the user functions. """ function Drive(core::BraidCore; warmup=true) - if warmup - Warmup(core) - else - _app = core._braid_app - # cheaper precompile option, but less effective - begin - postInitPrecompile(_app) - if (_app.basis_init !== nothing) && (_app.inner_prod !== nothing) - deltaPrecompile(_app) - end - end - end - _Drive(core) + if warmup + Warmup(core) + else + _app = core._braid_app + # cheaper precompile option, but less effective + begin + postInitPrecompile(_app) + if (_app.basis_init !== nothing) && (_app.inner_prod !== nothing) + deltaPrecompile(_app) + end + end + end + _Drive(core) end -function PrintStats(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_PrintStats(core._braid_core::Ptr{Cvoid})::Cint - end +function printStats(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_PrintStats(core._braid_core::Ptr{Cvoid})::Cint + end end -function SetTimerFile(core::BraidCore, filestem::String) - len = @ccall strlen(filestem::Cstring)::Cint - GC.@preserve core begin - @ccall libbraid.braid_SetTimerFile(core._braid_core::Ptr{Cvoid}, len::Cint, filestem::Cstring)::Cint - end +function setTimerFile(core::BraidCore, filestem::String) + len = @ccall strlen(filestem::Cstring)::Cint + GC.@preserve core begin + @ccall libbraid.braid_SetTimerFile(core._braid_core::Ptr{Cvoid}, len::Cint, filestem::Cstring)::Cint + end end -function PrintTimers(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_PrintTimers(core._braid_core::Ptr{Cvoid})::Cint - end +function printTimers(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_PrintTimers(core._braid_core::Ptr{Cvoid})::Cint + end end -function ResetTimer(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_ResetTimer(core._braid_core::Ptr{Cvoid})::Cint - end +function resetTimer(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_ResetTimer(core._braid_core::Ptr{Cvoid})::Cint + end end -function SetTimings(core::BraidCore, timing_level::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetTimings(core._braid_core::Ptr{Cvoid}, timing_level::Cint)::Cint - end +function setTimings(core::BraidCore, timing_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetTimings(core._braid_core::Ptr{Cvoid}, timing_level::Cint)::Cint + end end -function WriteConvHistory(core::BraidCore, filename::String) - GC.@preserve core begin - @ccall libbraid.braid_WriteConvHistory(core._braid_core::Ptr{Cvoid}, filename::Cstring)::Cint - end +function writeConvHistory(core::BraidCore, filename::String) + GC.@preserve core begin + @ccall libbraid.braid_WriteConvHistory(core._braid_core::Ptr{Cvoid}, filename::Cstring)::Cint + end end -function SetMaxLevels(core::BraidCore, max_levels::Integer) - @assert max_levels > 0 "Max. levels must be an integer greater than 0" - GC.@preserve core begin - @ccall libbraid.braid_SetMaxLevels(core._braid_core::Ptr{Cvoid}, max_levels::Cint)::Cint - end +function setMaxLevels(core::BraidCore, max_levels::Integer) + @assert max_levels > 0 "Max. levels must be an integer greater than 0" + GC.@preserve core begin + @ccall libbraid.braid_SetMaxLevels(core._braid_core::Ptr{Cvoid}, max_levels::Cint)::Cint + end end -function SetIncrMaxLevels(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_SetIncrMaxLevels(core._braid_core::Ptr{Cvoid})::Cint - end +function setIncrMaxLevels(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetIncrMaxLevels(core._braid_core::Ptr{Cvoid})::Cint + end end -function SetSkip(core::BraidCore, skip::Bool) - GC.@preserve core begin - @ccall libbraid.braid_SetSkip(core._braid_core::Ptr{Cvoid}, skip::Cint)::Cint - end +function setSkip(core::BraidCore, skip::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetSkip(core._braid_core::Ptr{Cvoid}, skip::Cint)::Cint + end end - -function SetRefine(core::BraidCore, refine::Bool) - GC.@preserve core begin - @ccall libbraid.braid_SetSkip(core._braid_core::Ptr{Cvoid}, refine::Cint)::Cint - end +function setRefine(core::BraidCore, refine::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetRefine(core._braid_core::Ptr{Cvoid}, refine::Cint)::Cint + end end -function SetMaxRefinements(core::BraidCore, max_refinements::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetMaxRefinements(core._braid_core::Ptr{Cvoid}, max_refinements::Cint)::Cint - end +function setMaxRefinements(core::BraidCore, max_refinements::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetMaxRefinements(core._braid_core::Ptr{Cvoid}, max_refinements::Cint)::Cint + end end - -function SetTPointsCutoff(core::BraidCore, tpoints_cutoff::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetTPointsCutoff(core._braid_core::Ptr{Cvoid}, tpoints_cutoff::Cint)::Cint - end +function setTPointsCutoff(core::BraidCore, tpoints_cutoff::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetTPointsCutoff(core._braid_core::Ptr{Cvoid}, tpoints_cutoff::Cint)::Cint + end end -function SetMinCoarse(core::BraidCore, min_coarse::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetMinCoarse(core._braid_core::Ptr{Cvoid}, min_coarse::Cint)::Cint - end +function setMinCoarse(core::BraidCore, min_coarse::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetMinCoarse(core._braid_core::Ptr{Cvoid}, min_coarse::Cint)::Cint + end end -function SetRelaxOnlyCG(core::BraidCore, relax_only_cg::Bool) - GC.@preserve core begin - @ccall libbraid.braid_SetRelaxOnlyCG(core._braid_core::Ptr{Cvoid}, relax_only_cg::Cint)::Cint - end +function setRelaxOnlyCG(core::BraidCore, relax_only_cg::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetRelaxOnlyCG(core._braid_core::Ptr{Cvoid}, relax_only_cg::Cint)::Cint + end end -function SetAbsTol(core::BraidCore, atol::Real) - GC.@preserve core begin - @ccall libbraid.braid_SetAbsTol(core._braid_core::Ptr{Cvoid}, atol::Cdouble)::Cint - end +function setAbsTol(core::BraidCore, atol::Real) + GC.@preserve core begin + @ccall libbraid.braid_SetAbsTol(core._braid_core::Ptr{Cvoid}, atol::Cdouble)::Cint + end end -function SetRelTol(core::BraidCore, rtol::Real) - GC.@preserve core begin - @ccall libbraid.braid_SetRelTol(core._braid_core::Ptr{Cvoid}, rtol::Cdouble)::Cint - end +function setRelTol(core::BraidCore, rtol::Real) + GC.@preserve core begin + @ccall libbraid.braid_SetRelTol(core._braid_core::Ptr{Cvoid}, rtol::Cdouble)::Cint + end end -function SetNRelax(core::BraidCore, level::Integer, nrelax::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetNRelax(core._braid_core::Ptr{Cvoid}, level::Cint, nrelax::Cint)::Cint - end +function setNRelax(core::BraidCore, level::Integer, nrelax::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetNRelax(core._braid_core::Ptr{Cvoid}, level::Cint, nrelax::Cint)::Cint + end end -function SetCRelaxWt(core::BraidCore, level::Integer, Cwt::Real) - GC.@preserve core begin - @ccall libbraid.braid_SetCRelaxWt(core._braid_core::Ptr{Cvoid}, level::Cint, Cwt::Cdouble)::Cint - end +function setCRelaxWt(core::BraidCore, level::Integer, Cwt::Real) + GC.@preserve core begin + @ccall libbraid.braid_SetCRelaxWt(core._braid_core::Ptr{Cvoid}, level::Cint, Cwt::Cdouble)::Cint + end end -function SetCFactor(core::BraidCore, level::Integer, cfactor::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetCFactor(core._braid_core::Ptr{Cvoid}, level::Cint, cfactor::Cint)::Cint - end +function setCFactor(core::BraidCore, level::Integer, cfactor::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetCFactor(core._braid_core::Ptr{Cvoid}, level::Cint, cfactor::Cint)::Cint + end end -function SetMaxIter(core::BraidCore, max_iter::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetMaxIter(core._braid_core::Ptr{Cvoid}, max_iter::Cint)::Cint - end +function setMaxIter(core::BraidCore, max_iter::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetMaxIter(core._braid_core::Ptr{Cvoid}, max_iter::Cint)::Cint + end end -function SetFMG(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_SetFMG(core._braid_core::Ptr{Cvoid})::Cint - end +function setFMG(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetFMG(core._braid_core::Ptr{Cvoid})::Cint + end end -function SetNFMG(core::BraidCore, k::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetNFMG(core._braid_core::Ptr{Cvoid}, k::Cint)::Cint - end +function setNFMG(core::BraidCore, k::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetNFMG(core._braid_core::Ptr{Cvoid}, k::Cint)::Cint + end end -function SetNFMGVcyc(core::BraidCore, nfmg_Vcyc::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetNFMGVcyc(core._braid_core::Ptr{Cvoid}, nfmg_Vcyc::Cint)::Cint - end +function setNFMGVcyc(core::BraidCore, nfmg_Vcyc::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetNFMGVcyc(core._braid_core::Ptr{Cvoid}, nfmg_Vcyc::Cint)::Cint + end end -function SetStorage(core::BraidCore, storage::Bool) - GC.@preserve core begin - @ccall libbraid.braid_SetStorage(core._braid_core::Ptr{Cvoid}, storage::Cint)::Cint - end +function setStorage(core::BraidCore, storage::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetStorage(core._braid_core::Ptr{Cvoid}, storage::Cint)::Cint + end end -function SetTemporalNorm(core::BraidCore, tnorm::Bool) - GC.@preserve core begin - @ccall libbraid.braid_SetTemporalNorm(core._braid_core::Ptr{Cvoid}, tnorm::Cint)::Cint - end +function setTemporalNorm(core::BraidCore, tnorm::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetTemporalNorm(core._braid_core::Ptr{Cvoid}, tnorm::Cint)::Cint + end end -function SetPeriodic(core::BraidCore, periodic::Bool) - GC.@preserve core begin - @ccall libbraid.braid_SetPeriodic(core._braid_core::Ptr{Cvoid}, periodic::Cint)::Cint - end +function setPeriodic(core::BraidCore, periodic::Bool) + GC.@preserve core begin + @ccall libbraid.braid_SetPeriodic(core._braid_core::Ptr{Cvoid}, periodic::Cint)::Cint + end end -function SetPrintLevel(core::BraidCore, print_level::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetPrintLevel(core._braid_core::Ptr{Cvoid}, print_level::Cint)::Cint - end +function setPrintLevel(core::BraidCore, print_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetPrintLevel(core._braid_core::Ptr{Cvoid}, print_level::Cint)::Cint + end end -function SetFileIOLevel(core::BraidCore, io_level::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetFileIOLevel(core._braid_core::Ptr{Cvoid}, io_level::Cint)::Cint - end +function setFileIOLevel(core::BraidCore, io_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetFileIOLevel(core._braid_core::Ptr{Cvoid}, io_level::Cint)::Cint + end end -function SetPrintFile(core::BraidCore, filename::String) - GC.@preserve core begin - @ccall libbraid.braid_SetPrintFile(core._braid_core::Ptr{Cvoid}, filename::Cstring)::Cint - end +function setPrintFile(core::BraidCore, filename::String) + GC.@preserve core begin + @ccall libbraid.braid_SetPrintFile(core._braid_core::Ptr{Cvoid}, filename::Cstring)::Cint + end end -function SetDefaultPrintFile(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_SetDefaultPrintFile(core._braid_core::Ptr{Cvoid})::Cint - end +function setDefaultPrintFile(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetDefaultPrintFile(core._braid_core::Ptr{Cvoid})::Cint + end end """ @@ -532,201 +501,208 @@ Set access level for XBraid. This controls how often the user's access function Default is 1. """ -function SetAccessLevel(core::BraidCore, access_level::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetAccessLevel(core._braid_core::Ptr{Cvoid}, access_level::Cint)::Cint - end +function setAccessLevel(core::BraidCore, access_level::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetAccessLevel(core._braid_core::Ptr{Cvoid}, access_level::Cint)::Cint + end end -function SetFinalFCRelax(core::BraidCore) - GC.@preserve core begin - @ccall libbraid.braid_SetFinalFCRelax(core._braid_core::Ptr{Cvoid})::Cint - end +function setFinalFCRelax(core::BraidCore) + GC.@preserve core begin + @ccall libbraid.braid_SetFinalFCRelax(core._braid_core::Ptr{Cvoid})::Cint + end end -function GetNumIter(core::BraidCore) - niter = Ref{Cint}(0) - GC.@preserve core begin - @ccall libbraid.braid_GetNumIter(core._braid_core::Ptr{Cvoid}, niter::Ref{Cint})::Cint - end - return niter[] +function getNumIter(core::BraidCore) + niter = Ref{Cint}(0) + GC.@preserve core begin + @ccall libbraid.braid_GetNumIter(core._braid_core::Ptr{Cvoid}, niter::Ref{Cint})::Cint + end + return niter[] end -function GetRNorms(core::BraidCore) - nrequest = Ref{Cint}() - nrequest[] = GetNumIter(core) - rnorms = zeros(nrequest[]) - GC.@preserve core begin - @ccall libbraid.braid_GetRNorms(core._braid_core::Ptr{Cvoid}, nrequest::Ref{Cint}, rnorms::Ref{Cdouble})::Cint - end - nrequest[] == 0 && return Float64[] - return rnorms[1:nrequest[]] +function getRNorms(core::BraidCore) + nrequest = Ref{Cint}() + nrequest[] = getNumIter(core) + rnorms = zeros(nrequest[]) + GC.@preserve core begin + @ccall libbraid.braid_GetRNorms(core._braid_core::Ptr{Cvoid}, nrequest::Ref{Cint}, rnorms::Ref{Cdouble})::Cint + end + nrequest[] == 0 && return Float64[] + return rnorms[1:nrequest[]] end -function GetNLevels(core::BraidCore) - nlevels = Ref(0) - GC.@preserve core begin - @ccall libbraid.braid_GetNLevels(core._braid_core::Ptr{Cvoid}, nlevels::Ref{Cint})::Cint - end - return nlevels[] +function getNLevels(core::BraidCore) + nlevels = Ref(0) + GC.@preserve core begin + @ccall libbraid.braid_GetNLevels(core._braid_core::Ptr{Cvoid}, nlevels::Ref{Cint})::Cint + end + return nlevels[] end -function SetSeqSoln(core::BraidCore, seq_soln::Bool) - GC.@preserve core begin - @ccall libbraid.braid_GetNLevels(core._braid_core::Ptr{Cvoid}, seq_soln::Cint)::Cint - end +function setSeqSoln(core::BraidCore, seq_soln::Bool) + GC.@preserve core begin + @ccall libbraid.braid_GetNLevels(core._braid_core::Ptr{Cvoid}, seq_soln::Cint)::Cint + end end -function SetRichardsonEstimation(core::BraidCore, est_error::Bool, richardson::Bool, local_order::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetRichardsonEstimation(core._braid_core::Ptr{Cvoid}, est_error::Cint, richardson::Cint, local_order::Cint)::Cint - end +function setRichardsonEstimation(core::BraidCore, est_error::Bool, richardson::Bool, local_order::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetRichardsonEstimation(core._braid_core::Ptr{Cvoid}, est_error::Cint, richardson::Cint, local_order::Cint)::Cint + end end -function SetDeltaCorrection(core::BraidCore, rank::Integer, basis_init::Function, inner_prod::Function) - core._braid_app.basis_init = basis_init - core._braid_app.inner_prod = inner_prod - # deltaPrecompile(core._braid_app) +function setSync(core::BraidCore, sync::Function) + core._braid_app.sync = sync - GC.@preserve core begin - @ccall libbraid.braid_SetDeltaCorrection(core._braid_core::Ptr{Cvoid}, rank::Cint, _c_init_basis::Ptr{Cvoid}, _c_inner_prod::Ptr{Cvoid})::Cint - end + GC.@preserve core begin + @ccall libbraid.braid_SetSync(core._braid_core::Ptr{Cvoid}, _c_sync::Ptr{Cvoid})::Cint + end end -function SetDeferDelta(core::BraidCore, level::Integer, iter::Integer) - GC.@preserve core begin - @ccall libbraid.braid_SetDeferDelta(core._braid_core::Ptr{Cvoid}, level::Cint, iter::Cint)::Cint - end +function setDeltaCorrection(core::BraidCore, rank::Integer, basis_init::Function; inner_prod::Function=default_inner_prod) + core._braid_app.basis_init = basis_init + core._braid_app.inner_prod = inner_prod + # deltaPrecompile(core._braid_app) + + GC.@preserve core begin + @ccall libbraid.braid_SetDeltaCorrection(core._braid_core::Ptr{Cvoid}, rank::Cint, _c_init_basis::Ptr{Cvoid}, _c_inner_prod::Ptr{Cvoid})::Cint + end +end + +function setDeferDelta(core::BraidCore, level::Integer, iter::Integer) + GC.@preserve core begin + @ccall libbraid.braid_SetDeferDelta(core._braid_core::Ptr{Cvoid}, level::Cint, iter::Cint)::Cint + end end -function SetLyapunovEstimation(core::BraidCore; relax::Bool = false, cglv::Bool = true, exponents::Bool = false) - GC.@preserve core begin - @ccall libbraid.braid_SetLyapunovEstimation(core._braid_core::Ptr{Cvoid}, relax::Cint, cglv::Cint, exponents::Cint)::Cint - end +function setLyapunovEstimation(core::BraidCore; relax::Bool=false, cglv::Bool=true, exponents::Bool=false) + GC.@preserve core begin + @ccall libbraid.braid_SetLyapunovEstimation(core._braid_core::Ptr{Cvoid}, relax::Cint, cglv::Cint, exponents::Cint)::Cint + end end -# Still missing spatial coarsening, sync, residual, and adjoint +# Still missing spatial coarsening, residual, and adjoint # braid_test function testInitAccess(app::BraidApp, t::Real, outputFile::Libc.FILE) - pass = @ccall libbraid.braid_TestInitAccess( - app::Ref{BraidApp}, - app.comm::MPI.MPI_Comm, - outputFile::Libc.FILE, - t::Cdouble, - _c_init::Ptr{Cvoid}, - _c_access::Ptr{Cvoid}, - _c_free::Ptr{Cvoid}, - )::Cint - return pass > 0 - - # println("Serialized size of user vector: $(app.bufsize)") - # println("Check output for objects not properly freed:") - # println(app.ref_ids) + pass = @ccall libbraid.braid_TestInitAccess( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + )::Cint + return pass > 0 + + # println("Serialized size of user vector: $(app.bufsize)") + # println("Check output for objects not properly freed:") + # println(app.ref_ids) end testInitAccess(app::BraidApp, t::Real, outputFile::IO) = testInitAccess(app, t, Libc.FILE(outputFile)) testInitAccess(app::BraidApp, t::Real) = testInitAccess(app, t, c_stdout) function testClone(app::BraidApp, t::Real, outputFile::Libc.FILE) - pass = @ccall libbraid.braid_TestClone( - app::Ref{BraidApp}, - app.comm::MPI.MPI_Comm, - outputFile::Libc.FILE, - t::Cdouble, - _c_init::Ptr{Cvoid}, - _c_access::Ptr{Cvoid}, - _c_free::Ptr{Cvoid}, - _c_clone::Ptr{Cvoid}, - )::Cint - return pass > 0 + pass = @ccall libbraid.braid_TestClone( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + )::Cint + return pass > 0 end testClone(app::BraidApp, t::Real, outputFile::IO) = testClone(app, t, Libc.FILE(outputFile)) testClone(app::BraidApp, t::Real) = testClone(app, t, c_stdout) function testSum(app::BraidApp, t::Real, outputFile::Libc.FILE) - pass = @ccall libbraid.braid_TestSum( - app::Ref{BraidApp}, - app.comm::MPI.MPI_Comm, - outputFile::Libc.FILE, - t::Cdouble, - _c_init::Ptr{Cvoid}, - _c_access::Ptr{Cvoid}, - _c_free::Ptr{Cvoid}, - _c_clone::Ptr{Cvoid}, - _c_sum::Ptr{Cvoid}, - )::Cint - return pass > 0 + pass = @ccall libbraid.braid_TestSum( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + )::Cint + return pass > 0 end testSum(app::BraidApp, t::Real, outputFile::IO) = testSum(app, t, Libc.FILE(outputFile)) testSum(app::BraidApp, t::Real) = testSum(app, t, c_stdout) function testSpatialNorm(app::BraidApp, t::Real, outputFile::Libc.FILE) - pass = @ccall libbraid.braid_TestSpatialNorm( - app::Ref{BraidApp}, - app.comm::MPI.MPI_Comm, - outputFile::Libc.FILE, - t::Cdouble, - _c_init::Ptr{Cvoid}, - _c_free::Ptr{Cvoid}, - _c_clone::Ptr{Cvoid}, - _c_sum::Ptr{Cvoid}, - _c_norm::Ptr{Cvoid}, - )::Cint - return pass > 0 + pass = @ccall libbraid.braid_TestSpatialNorm( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + _c_norm::Ptr{Cvoid}, + )::Cint + return pass > 0 end testSpatialNorm(app::BraidApp, t::Real, outputFile::IO) = testSpatialNorm(app, t, Libc.FILE(outputFile)) testSpatialNorm(app::BraidApp, t::Real) = testSpatialNorm(app, t, c_stdout) function testBuf(app::BraidApp, t::Real, outputFile::Libc.FILE) - pass = @ccall libbraid.braid_TestBuf( - app::Ref{BraidApp}, - app.comm::MPI.MPI_Comm, - outputFile::Libc.FILE, - t::Cdouble, - _c_init::Ptr{Cvoid}, - _c_free::Ptr{Cvoid}, - _c_sum::Ptr{Cvoid}, - _c_norm::Ptr{Cvoid}, - _c_bufsize::Ptr{Cvoid}, - _c_bufpack::Ptr{Cvoid}, - _c_bufunpack::Ptr{Cvoid}, - )::Cint - return pass > 0 - # print('\n') - # println("Check output for objects not freed:") - # println(app.ref_ids) + pass = @ccall libbraid.braid_TestBuf( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + _c_init::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + _c_norm::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, + _c_bufpack::Ptr{Cvoid}, + _c_bufunpack::Ptr{Cvoid}, + )::Cint + return pass > 0 + # print('\n') + # println("Check output for objects not freed:") + # println(app.ref_ids) end testBuf(app::BraidApp, t::Real, outputFile::IO) = testBuf(app, t, Libc.FILE(outputFile)) testBuf(app::BraidApp, t::Real) = testBuf(app, t, c_stdout) function testDelta(app::BraidApp, t::Real, dt::Real, rank::Integer, outputFile::Libc.FILE) - app.basis_init::Function - app.inner_prod::Function - pass = @ccall libbraid.braid_TestDelta( - app::Ref{BraidApp}, - app.comm::MPI.MPI_Comm, - outputFile::Libc.FILE, - t::Cdouble, - dt::Cdouble, - rank::Cint, - _c_init::Ptr{Cvoid}, - _c_init_basis::Ptr{Cvoid}, - _c_access::Ptr{Cvoid}, - _c_free::Ptr{Cvoid}, - _c_clone::Ptr{Cvoid}, - _c_sum::Ptr{Cvoid}, - _c_bufsize::Ptr{Cvoid}, - _c_bufpack::Ptr{Cvoid}, - _c_bufunpack::Ptr{Cvoid}, - _c_inner_prod::Ptr{Cvoid}, - _c_step::Ptr{Cvoid}, - )::Cint - return pass > 0 - # println("Check output for objects not freed:") - # println(app.ref_ids) + app.basis_init::Function + app.inner_prod::Function + pass = @ccall libbraid.braid_TestDelta( + app::Ref{BraidApp}, + app.comm::MPI.MPI_Comm, + outputFile::Libc.FILE, + t::Cdouble, + dt::Cdouble, + rank::Cint, + _c_init::Ptr{Cvoid}, + _c_init_basis::Ptr{Cvoid}, + _c_access::Ptr{Cvoid}, + _c_free::Ptr{Cvoid}, + _c_clone::Ptr{Cvoid}, + _c_sum::Ptr{Cvoid}, + _c_bufsize::Ptr{Cvoid}, + _c_bufpack::Ptr{Cvoid}, + _c_bufunpack::Ptr{Cvoid}, + _c_inner_prod::Ptr{Cvoid}, + _c_step::Ptr{Cvoid}, + )::Cint + return pass > 0 + # println("Check output for objects not freed:") + # println(app.ref_ids) end testDelta(app::BraidApp, t::Real, dt::Real, rank::Integer, outputFile::IO) = testDelta(app, t, dt, rank, Libc.FILE(outputFile)) testDelta(app::BraidApp, t::Real, dt::Real, rank::Integer) = testDelta(app, t, dt, rank, c_stdout) -# END MODULE XBraid -end +end # module XBraid diff --git a/braid/braid.jl/status.jl b/braid/braid.jl/status.jl index 76a974b2..71459e27 100644 --- a/braid/braid.jl/status.jl +++ b/braid/braid.jl/status.jl @@ -19,83 +19,159 @@ * ***********************************************************************EHEADER=# -# should these automatically qeuery the braid status for some commonly used values? + +module Status +export AccessStatus, StepStatus, SyncStatus +export CallerError, CallerFInterp, CallerFRestrict, CallerFRefine, CallerFAccess +export CallerFRefine_AfterInitHier, CallerDrive_TopCycle, CallerFCRelax +export CallerDrive_AfterInit, CallerBaseStep_diff, CallerComputeFullRNorm +export CallerFASResidual, CallerResidual, CallerInitGuess, CallerWarmup + +using ..XBraid.BraidUtils: libbraid, malloc_null_double_ptr +using ..XBraid: BraidVector + +# using MPI: MPI_Comm, Comm + +# calling functions +const CallerError = -1 +const CallerFInterp = 0 +const CallerFRestrict = 1 +const CallerFRefine = 2 +const CallerFAccess = 3 +const CallerFRefine_AfterInitHier = 4 +const CallerDrive_TopCycle = 5 +const CallerFCRelax = 6 +const CallerDrive_AfterInit = 7 +const CallerBaseStep_diff = 8 +const CallerComputeFullRNorm = 9 +const CallerFASResidual = 10 +const CallerResidual = 11 +const CallerInitGuess = 12 +const CallerWarmup = 13 + +# status structures struct StepStatus ptr::Ptr{Cvoid} end +StepStatus() = StepStatus(C_NULL) struct AccessStatus ptr::Ptr{Cvoid} end +AccessStatus() = AccessStatus(C_NULL) +struct SyncStatus + ptr::Ptr{Cvoid} +end +SyncStatus() = SyncStatus(C_NULL) + +# status get/set functions (not exported) # @ccall requires these type annotations to be known at compile time, so macros won't work here -function status_GetT(status::Union{StepStatus, AccessStatus}) +function getT(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return 0.0 t = Ref{Cdouble}() @ccall libbraid.braid_StatusGetT(status.ptr::Ptr{Cvoid}, t::Ref{Cdouble})::Cint return t[] end -function status_GetTStop(status::StepStatus) +function getTStop(status::StepStatus) + status.ptr == C_NULL && return 0.0 tstop = Ref{Cdouble}() @ccall libbraid.braid_StatusGetTStop(status.ptr::Ptr{Cvoid}, tstop::Ref{Cdouble})::Cint return tstop[] end -function status_GetTstartTstop(status::StepStatus) +function getTstartTstop(status::StepStatus) + status.ptr == C_NULL && return (0.0, 0.0) tstart, tstop = Ref{Cdouble}(), Ref{Cdouble}() @ccall libbraid.braid_StepStatusGetTstartTstop(status.ptr::Ptr{Cvoid}, tstart::Ref{Cdouble}, tstop::Ref{Cdouble})::Cint return tstart[], tstop[] end -function status_GetTIndex(status::Union{StepStatus, AccessStatus}) +function getTIndex(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return 0 ti = Ref{Cint}() @ccall libbraid.braid_StatusGetTIndex(status.ptr::Ptr{Cvoid}, ti::Ref{Cint})::Cint - return ti[] + # julia uses 1-based indexing + return ti[] + 1 end -function status_GetLevel(status::Union{StepStatus, AccessStatus}) +function getLevel(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return 0 level = Ref{Cint}() @ccall libbraid.braid_StatusGetLevel(status.ptr::Ptr{Cvoid}, level::Ref{Cint})::Cint return level[] end -function status_GetWrapperTest(status::Union{StepStatus, AccessStatus}) +function getCFactor(status::Union{StepStatus, AccessStatus}, level::Integer) + status.ptr == C_NULL && return 1 + cfactor = Ref{Cint}() + @ccall libbraid.braid_StatusGetCFactor(status.ptr::Ptr{Cvoid}, cfactor::Ref{Cint}, level::Cint)::Cint + return cfactor[] +end + +function getCFactor(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return 0 + getCFactor(status, getLevel(status)) +end + +function getWrapperTest(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return false wtest = Ref{Cint}() @ccall libbraid.braid_StatusGetWrapperTest(status.ptr::Ptr{Cvoid}, wtest::Ref{Cint})::Cint return (wtest[] > 0) end -function status_GetIter(status::Union{StepStatus, AccessStatus}) +function getIter(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return 0 iter = Ref{Cint}() @ccall libbraid.braid_StatusGetIter(status.ptr::Ptr{Cvoid}, iter::Ref{Cint})::Cint return iter[] end -function status_GetNLevels(status::Union{StepStatus, AccessStatus}) +function getNLevels(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return 0 nlevels = Ref{Cint}() @ccall libbraid.braid_StatusGetNLevels(status.ptr::Ptr{Cvoid}, nlevels::Ref{Cint})::Cint return nlevels[] end -function status_GetNTPoints(status::Union{StepStatus, AccessStatus}) +function setRFactor(status::StepStatus, rfactor::Real) + status.ptr == C_NULL && return nothing + @ccall libbraid.braid_StatusSetRFactor(status.ptr::Ptr{Cvoid}, rfactor::Cdouble)::Cint + return nothing +end + +function getNRefine(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return 0 + nrefine = Ref{Cint}() + @ccall libbraid.braid_StatusGetNRefine(status.ptr::Ptr{Cvoid}, nrefine::Ref{Cint})::Cint + return nrefine[] +end + +function getNTPoints(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return 0 ntpoints = Ref{Cint}() @ccall libbraid.braid_StatusGetNTPoints(status.ptr::Ptr{Cvoid}, ntpoints::Ref{Cint})::Cint return ntpoints[] end -function status_GetResidual(status::Union{StepStatus, AccessStatus}) +function getResidual(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return 0.0 res = Ref{Cdouble}() @ccall libbraid.braid_StatusGetResidual(status.ptr::Ptr{Cvoid}, res::Ref{Cdouble})::Cint return res[] end -function status_GetDone(status::Union{StepStatus, AccessStatus}) +function getDone(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return false done = Ref{Cint}() @ccall libbraid.braid_StatusGetDone(status.ptr::Ptr{Cvoid}, done::Ref{Cint})::Cint return (done[] > 0) end -function status_GetTILD(status::Union{StepStatus, AccessStatus}) +function getTILD(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return (0.0, 0, 0, false) time = Ref{Cdouble}() iter = Ref{Cint}() level = Ref{Cint}() @@ -104,41 +180,63 @@ function status_GetTILD(status::Union{StepStatus, AccessStatus}) return time[], iter[], level[], (done[] > 0) end -function status_GetCallingFunction(status::Union{StepStatus, AccessStatus}) +function getTIUL(status::Union{StepStatus, SyncStatus}, level::Integer) + status.ptr == C_NULL && return (0, 0) + iu = Ref{Cint}() + il = Ref{Cint}() + @ccall libbraid.braid_StatusGetTIUL(status.ptr::Ptr{Cvoid}, iu::Ref{Cint}, il::Ref{Cint}, level::Cint)::Cint + # julia uses 1-based indexing + return iu[] + 1, il[] + 1 +end + +function getCallingFunction(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return CallerError func = Ref{Cint}() @ccall libbraid.braid_StatusGetCallingFunction(status.ptr::Ptr{Cvoid}, func::Ref{Cint})::Cint return func[] end -function status_GetTol(status::StepStatus) +function getTol(status::StepStatus) + status.ptr == C_NULL && return 0.0 tol = Ref{Cdouble}() @ccall libbraid.braid_StatusGetTol(status.ptr::Ptr{Cvoid}, tol::Ref{Cdouble})::Cint return tol[] end -function status_GetRNorms(status::StepStatus) - iters = status_GetIter(status) +function getRNorms(status::StepStatus) + status.ptr == C_NULL && return zeros(0) + iters = getIter(status) nrequest = Ref{Cint}(iters) norms = zeros(Cdouble, iters) @ccall libbraid.braid_StatusGetRNorms(status.ptr::Ptr{Cvoid}, nrequest::Ref{Cint}, norms::Ref{Cdouble})::Cint return norms end -function status_GetProc(status::Union{StepStatus, AccessStatus}) +function getProc(status::Union{StepStatus, AccessStatus, SyncStatus}) + status.ptr == C_NULL && return 0 proc = Ref{Cint}() @ccall libbraid.braid_StatusGetProc(status.ptr::Ptr{Cvoid}, proc::Ref{Cint})::Cint return proc[] end -function status_GetDeltaRank(status::Union{StepStatus, AccessStatus}) +# function getTComm(status::SyncStatus) +# status.ptr == C_NULL && return MPI.Comm(MPI.COMM_NULL) +# tcomm = Ref{MPI.MPI_Comm}() +# @ccall libbraid.braid_StatusGetTComm(status.ptr::Ptr{Cvoid}, tcomm::Ref{MPI.MPI_Comm})::Cint +# return MPI.Comm(tcomm[]) +# end + +function getDeltaRank(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return 0 rank = Ref{Cint}() @ccall libbraid.braid_StatusGetDeltaRank(status.ptr::Ptr{Cvoid}, rank::Ref{Cint})::Cint return rank[] end # These are special cases -function status_GetLocalLyapExponents(status::AccessStatus) - rank = status_GetDeltaRank(status) +function getLocalLyapExponents(status::AccessStatus) + status.ptr == C_NULL && return zeros(0) + rank = getDeltaRank(status) rank < 1 && return [] exps = zeros(rank) @@ -147,11 +245,12 @@ function status_GetLocalLyapExponents(status::AccessStatus) return exps end -function status_GetBasisVectors(status::Union{StepStatus, AccessStatus}) - rank = status_GetDeltaRank(status) - Ψ = [] - rank < 1 && return Ψ +function getBasisVectors(status::Union{StepStatus, AccessStatus}) + status.ptr == C_NULL && return [] + rank = getDeltaRank(status) + rank < 1 && return [] + Ψ = [] for i in 1:rank pp = malloc_null_double_ptr(Cvoid) GC.@preserve pp begin @@ -168,6 +267,4 @@ function status_GetBasisVectors(status::Union{StepStatus, AccessStatus}) return Ψ end -function status_SetRFactor(status::StepStatus, rfactor::Real) - @ccall libbraid.braid_StatusSetRFactor(status.ptr::Ptr{Cvoid}, rfactor::Cdouble)::Cint -end +end # module Status \ No newline at end of file diff --git a/braid/braid_status.c b/braid/braid_status.c index 79504e26..d4dfaf05 100644 --- a/braid/braid_status.c +++ b/braid/braid_status.c @@ -114,6 +114,25 @@ braid_StatusGetLevel(braid_Status status, return _braid_error_flag; } +braid_Int +braid_StatusGetCFactor(braid_Status status, + braid_Int *cfactor_ptr, + braid_Int level + ) +{ + _braid_Grid **grids = _braid_StatusElt(status, grids); + braid_Int nlevels = _braid_StatusElt(status, nlevels); + + if (level < 0 || level >= nlevels) + { + *cfactor_ptr = -1; + return 1; + } + + *cfactor_ptr = _braid_GridElt(grids[level], cfactor); + return _braid_error_flag; +} + braid_Int braid_StatusGetNLevels(braid_Status status, braid_Int *nlevels_ptr @@ -251,15 +270,15 @@ braid_StatusGetBasisVec(braid_Status status, ) { braid_Basis ba = _braid_StatusElt(status, lvectors); + + *v_ptr = NULL; if ((ba != NULL) && (index < ba->rank)) { *v_ptr = ba->userVecs[index]; + return _braid_error_flag; } - else // gracefully return NULL - { - v_ptr = NULL; - } - return _braid_error_flag; + // else + return 1; } braid_Int @@ -721,6 +740,7 @@ ACCESSOR_FUNCTION_GET1(Access, CallingFunction, Int) ACCESSOR_FUNCTION_GET1(Access, SingleErrorEstAccess, Real) ACCESSOR_FUNCTION_GET1(Access, DeltaRank, Int) ACCESSOR_FUNCTION_GET2(Access, LocalLyapExponents, Real, Int) +ACCESSOR_FUNCTION_GET1_IN1(Access, CFactor, Int, Int) ACCESSOR_FUNCTION_GET1_IN1(Access, BasisVec, Vector, Int) /*-------------------------------------------------------------------------- @@ -852,6 +872,7 @@ ACCESSOR_FUNCTION_GET1(Step, Done, Int) ACCESSOR_FUNCTION_GET1(Step, SingleErrorEstStep, Real) ACCESSOR_FUNCTION_GET1(Step, CallingFunction, Int) ACCESSOR_FUNCTION_GET1(Step, DeltaRank, Int) +ACCESSOR_FUNCTION_GET1_IN1(Step, CFactor, Int, Int) ACCESSOR_FUNCTION_GET1_IN1(Step, BasisVec, Vector, Int) /*-------------------------------------------------------------------------- diff --git a/braid/braid_status.h b/braid/braid_status.h index d719d26c..d3bf0992 100644 --- a/braid/braid_status.h +++ b/braid/braid_status.h @@ -184,6 +184,14 @@ braid_Int braid_StatusGetLevel(braid_Status status, /**< structure containing current simulation info */ braid_Int *level_ptr /**< output, current level in XBraid */ ); +/** + * Return the coarsening factor on the given level from the Status structure. + **/ +braid_Int +braid_StatusGetCFactor(braid_Status status, /**< structure containing current simulation info */ + braid_Int *cfactor_ptr, /**< output, coarsening factor on current level */ + braid_Int level /**< level to get coarsening factor for */ + ); /** * Return the total number of XBraid levels from the Status structure. @@ -626,6 +634,7 @@ ACCESSOR_HEADER_GET1(Access, CallingFunction, Int) ACCESSOR_HEADER_GET1(Access, SingleErrorEstAccess, Real) ACCESSOR_HEADER_GET1(Access, DeltaRank, Int) ACCESSOR_HEADER_GET2(Access, LocalLyapExponents, Real, Int) +ACCESSOR_HEADER_GET1_IN1(Access, CFactor, Int, Int) ACCESSOR_HEADER_GET1_IN1(Access, BasisVec, Vector, Int) /*-------------------------------------------------------------------------- @@ -688,6 +697,7 @@ ACCESSOR_HEADER_GET1(Step, Done, Int) ACCESSOR_HEADER_GET1(Step, SingleErrorEstStep, Real) ACCESSOR_HEADER_GET1(Step, CallingFunction, Int) ACCESSOR_HEADER_GET1(Step, DeltaRank, Int) +ACCESSOR_HEADER_GET1_IN1(Step, CFactor, Int, Int) ACCESSOR_HEADER_GET1_IN1(Step, BasisVec, Vector, Int) /*-------------------------------------------------------------------------- @@ -754,6 +764,8 @@ ACCESSOR_HEADER_GET1(Objective, Tol, Real) #define braid_ASCaller_Residual 11 /** When CallingFunction equals 12, Braid is in InitGuess */ #define braid_ASCaller_InitGuess 12 +/** When CallingFunction equals 13, Braid is in Warmup */ +#define braid_ASCaller_Warmup 13 /** @}*/ diff --git a/braid/util.c b/braid/util.c index 3bc9b233..53430255 100644 --- a/braid/util.c +++ b/braid/util.c @@ -342,7 +342,7 @@ braid_Warmup(braid_App app, /* step */ braid_StepStatus sstatus = (braid_StepStatus)status; - _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, NULL, sstatus); + _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, braid_ASCaller_Warmup, NULL, sstatus); step(app, v, NULL, u, sstatus); /* coarsen, refine */ @@ -366,7 +366,7 @@ braid_Warmup(braid_App app, B->userVecs = _braid_TAlloc(braid_Vector, 1); init_basis(app, t, 0, &(B->userVecs[0])); innerprod(app, u, B->userVecs[0], &throwaway); - _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, 0, B, sstatus); + _braid_StepStatusInit(t, t+fdt, 0, 1e-16, 0, 0, 0, 2, braid_ASCaller_Warmup, B, sstatus); step(app, v, NULL, u, sstatus); myfree(app, B->userVecs[0]); diff --git a/drivers/julia/adaptive_theta/build_function_sundials.jl b/drivers/julia/adaptive_theta/build_function_sundials.jl new file mode 100644 index 00000000..a2529a3f --- /dev/null +++ b/drivers/julia/adaptive_theta/build_function_sundials.jl @@ -0,0 +1,101 @@ +using Symbolics, Dates + +# code generation for sundials (C) +struct SunTarget <: Symbolics.BuildTargets end + +function sunliterals(expr) + expr isa Real && return :(RCONST($(float(expr)))) + return expr +end + +function sunoperators(expr) + expr isa Expr || return expr + for e in expr.args + if e isa Expr + sunoperators(e) + end + end + for i in eachindex(expr.args) + if expr.args[i] isa Union{Float32, Float64, Rational} + # RCONST is a macro from sundials which constructs a real constant + expr.args[i] = :(RCONST($(float(expr.args[i])))) + end + end + # Introduce another factor 1 to prevent contraction of terms like "5 * t" to "5t" (not valid C code) + if expr.head==:call && expr.args[1]==:* && length(expr.args)==3 && isa(expr.args[2], Real) && isa(expr.args[3], Symbol) + push!(expr.args, 1) + # Power operator does not exist in C, replace by multiplication or "pow" + elseif expr.head==:call && expr.args[1]==:^ + @assert length(expr.args)==3 "Don't know how to handle ^ operation with <> 2 arguments" + x = expr.args[2] + n = expr.args[3] + empty!(expr.args) + # Replace by multiplication/division if + # x is a symbol and n is a small integer + # x is a more complex expression and n is ±1 + # n is exactly 0 + if (isa(n,Integer) && ((isa(x, Symbol) && abs(n) <= 3) || abs(n) <= 1)) || n==0 + if n >= 0 + append!(expr.args, [:*, fill(x, n)...]) + # fill up with factor 1 so this expr can still be a multiplication + while length(expr.args) < 3 + push!(expr.args, 1) + end + else # inverse of the above + if n==-1 + term = x + else + term = :( ($(x)) ^ ($(-n))) + coperators(term) + end + append!(expr.args, [:/, 1., term]) + end + #... otherwise use "pow" function + else + append!(expr.args, [:SUNpowerI, x, n]) + end + # replace bare real constants by RCONST + elseif expr.head==:call && (expr.args[1]==:* || expr.args[1]==:+ || expr.args[1]==:/) + for i in eachindex(expr.args) + if expr.args[i] isa Real + expr.args[i] = :(RCONST($(float(expr.args[i])))) + end + end + end + expr +end + +function Symbolics._build_function(target::SunTarget, ex::AbstractArray, args...; + conv = Symbolics.toexpr, + header = false, + fname = :diffeqf, + lhsname = :du, + rhsnames = [Symbol("RHS$i") for i in 1:length(args)]) + @info "Building function for Sundials" + fname = Symbol('_' * string(fname)) + ex = hcat([row for row in eachrow(ex)]...) + varnumbercache = Symbolics.buildvarnumbercache(args...) + equations = Vector{String}() + for col ∈ axes(ex, 2), row ∈ axes(ex, 1) + lhs = string(lhsname, "[", (col-1) * size(ex,1) + row-1, "]") + rhs = Symbolics.numbered_expr(ex[row, col].val, varnumbercache, args...; + lhsname = lhsname, + rhsnames = rhsnames, + offset = -1) |> sunoperators |> sunliterals |> string + push!(equations, string(lhs, " = ", rhs, ";")) + end + + argstrs = join(vcat("sunrealtype* $(lhsname)",[typeof(args[i])<:AbstractArray ? "const sunrealtype* $(rhsnames[i])" : "const sunrealtype $(rhsnames[i])" for i in 1:length(args)]),", ") + + head = """ + #include + #include + + """ + body = "void $fname($(argstrs...))\n{$([string("\n ", eqn) for eqn ∈ equations]...)\n}\n" + if header + return head*body + else + return body + end +end diff --git a/drivers/julia/adaptive_theta/butcher-tables.jl b/drivers/julia/adaptive_theta/butcher-tables.jl new file mode 100644 index 00000000..ddc723b1 --- /dev/null +++ b/drivers/julia/adaptive_theta/butcher-tables.jl @@ -0,0 +1,260 @@ +using DataStructures, PrettyTables +using Symbolics, NLsolve + +include("build_function_sundials.jl") + +struct ButcherTable + A::AbstractMatrix + b::AbstractVector + c::AbstractVector + s::Int +end +ButcherTable(A, b) = ButcherTable(A, b, A*ones(length(b)), length(b)) +ButcherTable(A, b, c) = ButcherTable(A, b, c, length(b)) +function ButcherTable(A::Vector{<:Real}, b::Vector{<:Real}, c::Vector{<:Real}, s::Integer) + return ButcherTable(reshape(A, s, s), b, c, s) +end +function Base.show(io::IO, bt::ButcherTable) + print(io, "ButcherTable: \n") + # pretty table + b = [1.0; bt.b] + data = [bt.c bt.A; b'] + pretty_table( + data; + show_header=false, + body_hlines=[bt.s], + vlines=[:begin, 1, :end] + ) +end + +forest = OrderedDict( + # 2nd order + :{τ} => 1, + # 3rd order + :{{τ}} => 2, + :{τ, τ} => 3, + # 4th order + :{{{τ}}} => 4, + :{{τ, τ}} => 5, + :{τ, {τ}} => 6, + :{τ, τ, τ} => 7, + # 5th order + :{{{{τ}}}} => 8, + :{{{τ, τ}}} => 9, + :{{τ, {τ}}} => 10, + :{{τ, τ, τ}} => 11, + :{τ, {{τ}}} => 12, + :{τ, {τ, τ}} => 13, + :{{τ}, {τ}} => 14, + :{τ, τ, {τ}} => 15, + :{τ, τ, τ, τ} => 16 +) + +elem_weights = [ + # 2nd order + (A, b, c) -> b' * c, + # 3rd order + (A, b, c) -> b' * A * c, + (A, b, c) -> b' * c.^2, + # 4th order + (A, b, c) -> b' * A * A * c, + (A, b, c) -> b' * A * c.^2, + (A, b, c) -> b' * (c .* (A*c)), + (A, b, c) -> b' * c.^3, + # 5th order + (A, b, c) -> b' * A * A * A * c, + (A, b, c) -> b' * A * A * c.^2, + (A, b, c) -> b' * A * (c .* (A*c)), + (A, b, c) -> b' * A * c.^3, + (A, b, c) -> b' * (c .* (A*A*c)), + (A, b, c) -> b' * (c .* (A*c.^2)), + (A, b, c) -> b' * (A * c).^2, + (A, b, c) -> b' * (c .* (c .* (A*c))), + (A, b, c) -> b' * c.^4 +] + +rhs_classical = [ + # 2nd order + 1/2, + # 3rd order + 1/6, 1/3, + # 4th order + 1/24, 1/12, 1/8, 1/4, + # 5th order + 1/120, 1/60, 1/40, 1/20, 1/30, 1/15, 1/20, 1/10, 1/5 +] +num_conditions = [0, 1, 3, 7, 16] + +ψ(B::ButcherTable, tree::Expr) = elem_weights[forest[tree]](B.A, B.b, B.c) + +@variables θ[1:15] + +function gen_lhs_func(B::ButcherTable, order::Integer) + num_conds = num_conditions[order] + fA, fb, fc = [eval(build_function(syms, collect(θ[1:num_conds]); checkbounds=true)[1]) for syms ∈ (B.A, B.b, B.c)] + conds_sym = [expand(ψ(B, t)) for t ∈ forest.keys[1:num_conds]] + conds_jac = Symbolics.jacobian(conds_sym, θ[1:num_conds]) + f = eval(build_function(conds_sym, collect(θ[1:num_conds]); checkbounds=true)[2]) + J = eval(build_function(conds_jac, collect(θ[1:num_conds]); checkbounds=true)[2]) + function fill(θ::AbstractVector) + @assert length(θ) == num_conds + ButcherTable(fA(θ), fb(θ), fc(θ), B.s) + end + return f, J, fill +end + +function solve_order_conditions(f!::Function, J!::Function, fill::Function, order::Integer, rhs::Vector{<:Real}; + guess=zeros(num_conditions[order]), method=:trust_region) + @assert length(rhs) == num_conditions[order] + function g!(G, θ) + f!(G, θ) + G .-= rhs + end + result = nlsolve(g!, J!, guess; method=method) + if !result.f_converged + @warn "Order conditions solver not converged" + @info result + result = nlsolve(g!, J!, guess; method=:newton) + end + fill(result.zero) +end + +function gen_c_lhs_func(B::ButcherTable, order::Integer, name::AbstractString) + rhsnames = [:th] + num_conds = num_conditions[order] + # fill butcher table + fA = build_function( + B.A, collect(θ[1:num_conds]); + target=SunTarget(), + fname=name * "_btable_A", + lhsname=:A, rhsnames=rhsnames + ) + fb = build_function( + B.b, collect(θ[1:num_conds]); + target=SunTarget(), + fname=name * "_btable_b", + lhsname=:b, rhsnames=rhsnames + ) + fc = build_function( + B.c, collect(θ[1:num_conds]); + target=SunTarget(), + fname=name * "_btable_c", + lhsname=:c, rhsnames=rhsnames + ) + # compute order conditions + conds_sym = [expand(ψ(B, t)) for t ∈ forest.keys[1:num_conds]] + conds_jac = Symbolics.jacobian(conds_sym, θ[1:num_conds]) + func = build_function( + conds_sym, collect(θ[1:num_conds]); + target=SunTarget(), header=true, + fname=name * "_lhs", + lhsname=:phi, rhsnames=rhsnames + ) + func_j = build_function( + conds_jac, collect(θ[1:num_conds]); + target=SunTarget(), + fname=name * "_lhs_jac", + lhsname=:phi_J, rhsnames=rhsnames + ) + open("c_funcs/$name.c", "w") do file + println(file, func) + println(file, func_j) + println(file, fA) + println(file, fb) + println(file, fc) + end +end + +beuler = ButcherTable([1.0], [1.0], [1.0]) +sdirk212 = ButcherTable([1. 0.; -1. 1.], [1 / 2, 1 / 2], [1., 0.]) +sdirk212_emb = [1., 0.] + +#sdirk33 +x = 0.4358665215 +sdirk33 = ButcherTable([x 0.0 0.0; (1-x)/2 x 0; (-3.0x^2/2 + 4.0x - 1/4) (3x^2/2 - 5.0x + 5/4) x], [(-3.0x^2/2 + 4.0x - 1/4), (3.0x^2/2 - 5.0x + 5/4), x]) + +#esdirk423 +diag = 1767732205903 / 4055673282236 +A3 = zeros(4, 4) +A3[2, 1] = diag +A3[2, 2] = diag +A3[3, 1] = 2746238789719 / 10658868560708 +A3[3, 2] = -640167445237 / 6845629431997 +A3[3, 3] = diag +A3[4, 1] = 1471266399579 / 7840856788654 +A3[4, 2] = -4482444167858 / 7529755066697 +A3[4, 3] = 11266239266428 / 11593286722821 +A3[4, 4] = diag +esdirk3 = ButcherTable(A3, A3[4, :], [0, 2diag, 3 / 5, 1]) +esdirk3_emb = [2756255671327/12835298489170, -10771552573575/22201958757719, 9247589265047/10645013368117, 2193209047091/5459859503100] + +#sdirk534 +A4 = [ + 1/4 0 0 0 0; + 1/2 1/4 0 0 0; + 17/50 -1/25 1/4 0 0; + 371/1360 -137/2720 15/544 1/4 0; + 25/24 -49/48 125/16 -85/12 1/4 +] +sdirk4 = ButcherTable(A4, A4[5, :]) + + +# θ methods + +# can approximate up to 2nd order +θesdirk2 = ButcherTable([0.0 0.0; 1-θ[1] θ[1]], [1-θ[1], θ[1]]) +θesdirk2_lhs!, θesdirk2_J!, θesdirk2_fill = gen_lhs_func(θesdirk2, 2) +θesdirk2_guess = [1/2] +gen_c_lhs_func(θesdirk2, 2, "theta_esdirk2") +#= + 0 │ 0 0 + 1 │ 1 - θ θ +──────┼─────────── + 1 │ 1 - θ θ +=# + +# can approximate up to 2nd order +θsdirk2 = ButcherTable([θ[1] 0.0; 1-θ[1] θ[1]], [1-θ[1], θ[1]]) +θsdirk2_lhs!, θsdirk2_J!, θsdirk2_fill = gen_lhs_func(θsdirk2, 2) +θsdirk2_guess = [1-√2/2] +gen_c_lhs_func(θsdirk2, 2, "theta_sdirk2") +#= + θ │ θ 0 + 1 │ 1 - θ θ +──────┼─────────── + 1 │ 1 - θ θ +=# + +# can approximate up to 3rd order +θesdirk3 = ButcherTable([0. 0. 0.; θ[2]-θ[1] θ[1] 0; θ[3] (1.0-θ[3]-θ[1]) θ[1]], [θ[3], 1.0-θ[3]-θ[1], θ[1]]) +θesdirk3_lhs!, θesdirk3_J!, θesdirk3_fill = gen_lhs_func(θesdirk3, 3) +gen_c_lhs_func(θesdirk3, 3, "theta_esdirk3") +#= + 0 │ 0 0 0 + θ₂ │ θ₂ - θ₁ θ₁ 0 + 1 │ θ₃ 1 - θ₃ - θ₁ θ₁ +──────┼────────────────────────── + 2 │ θ₃ 1 - θ₃ - θ₁ θ₁ +=# + +θsdirk3 = ButcherTable([θ[1] 0.0 0.0; θ[2]-θ[1] θ[1] 0; (1.0-θ[3]-θ[1]) θ[3] θ[1]], [1.0-θ[3]-θ[1], θ[3], θ[1]]) +θsdirk3_lhs!, θsdirk3_J!, θsdirk3_fill = gen_lhs_func(θsdirk3, 3) +θsdirk3_guess = [0.4358665215, 0.71793326075, -0.6443631706532353] +gen_c_lhs_func(θsdirk3, 3, "theta_sdirk3") +#= + θ₁ │ θ₁ 0 0 + θ₂ │ θ₂ - θ₁ θ₁ 0 + 1 │ 1 - θ₃ - θ₁ θ₃ θ₁ +──────┼────────────────────────── + 2 │ 1 - θ₃ - θ₁ θ₃ θ₁ +=# + +# can approximate up to 4th order +# θsdirk4 = ButcherTable([θ[1] 0 0 0 0; θ[2]-θ[1] θ[1] 0 0; θ[3] (1-θ[3]-θ[1]) θ[1] 0 0; θ[4] 0 (1-θ[4]-θ[3]-θ[1]) θ[1]], [θ[4], 0, 1-θ[4]-θ[3]-θ[1], θ[1]]) +# θsdirk4_lhs, θsdirk4_J = gen_lhs_func(θsdirk4, 4) +# gen_c_lhs_func(θsdirk4, 4, "theta_sdirk4") +# #= +# θ₁ │ θ₁ 0 0 0 +# θ₂ │ θ₂ - θ₁ θ₁ 0 0 +# θ₃ │ θ₃ 1 - θ₃ - θ₁ θ₁ 0 +# 1 │ θ₄ 0 θ₃ θ₁ diff --git a/drivers/julia/adaptive_theta/drive-adaptive-theta.jl b/drivers/julia/adaptive_theta/drive-adaptive-theta.jl new file mode 100644 index 00000000..7ab529a5 --- /dev/null +++ b/drivers/julia/adaptive_theta/drive-adaptive-theta.jl @@ -0,0 +1,265 @@ +using ToeplitzMatrices, OffsetArrays, LinearAlgebra, IterativeSolvers +using Plots +using MPI + +include("../../../braid/braid.jl/XBraid.jl") +include("butcher-tables.jl") +using .XBraid + +mutable struct AdvDifApp + # usual parameters for simulation + nx::Int + Δx::Float64 + A::Circulant + I::SymmetricToeplitz + useTheta::Bool + useRich::Bool + order::Int + cf::Int + max_levels::Int + solTf::Vector{Vector{Float64}} + residuals::Vector{Float64} + times::Vector{Float64} + spatial_tol::Float64 + + # parameters for adaptive theta + skip_downcycle::Bool + cgorder::Int + flag_refine_downcycle::Bool + fine_btable::ButcherTable + cg_btables::Vector{Vector{ButcherTable}} # butcher tables for each level/step +end + +fine_btables = [beuler, sdirk212, esdirk3, sdirk4] + +# default constructor +function AdvDifApp(nx::Integer, Δx::Real, ν::Real, α::Real, useTheta::Bool, useRich::Bool, order::Integer, cf::Integer, ml::Integer; spatial_tol=1e-3, cgorder=order+1, skip=true) + I = SymmetricToeplitz([1, zeros(nx-1)...]) + Δ = (1/Δx^2)Circulant([-2, 1, zeros(nx-3)..., 1]) + ∇ = (1/Δx)Circulant([0, -1/2, zeros(nx-3)..., 1/2]) + # ∇ = (1/Δx)Circulant([-1, 1, zeros(nx-2)...]) + A = ν*Δ - α*∇ + AdvDifApp(nx, Δx, A, I, useTheta, useRich, order, cf, ml, [], [], [], spatial_tol, skip, cgorder, false, fine_btables[order], []) +end + +mutable struct ThetaVector + u::Vector{Float64} + ψ::Vector{Float64} + t_prior::Float64 +end +function ThetaVector(u, order::Integer) + numconditions = [0, 1, 3, 7, 16] + order > 5 && return ThetaVector(u, zeros(numconditions[5]), 0.) + return ThetaVector(u, zeros(numconditions[order]), 0.) +end + +function dirk_step!(app::AdvDifApp, u::Vector{<:Real}, Δt::Real, btable::ButcherTable; embedding=nothing) + k = zeros(app.nx, btable.s) + for i in 1:btable.s + k[:, i] .= Δt * app.A * (u .+ sum(btable.A[i, j]*k[:, j] for j ∈ 1:i-1; init=zeros(app.nx))) + if btable.A[i, i] != 0 + k[:, i] .= (app.I - Δt * btable.A[i, i] * app.A) \ k[:, i] + end + end + u .= u + sum(btable.b[j] * k[:, j] for j ∈ 1:btable.s) + if embedding !== nothing + @assert length(embedding) == btable.s + err_est = sum(embedding[j] * k[:, j] for j ∈ 1:btable.s) - sum(btable.b[j] * k[:, j] for j ∈ 1:btable.s) + return norm(err_est) + end + return 0. +end + +function get_btable(app::AdvDifApp, status::XBraid.Status.StepStatus, ti::Integer) + level = XBraid.Status.getLevel(status) + if level == 0 || !app.useTheta + return app.fine_btable + end + iu, il = XBraid.Status.getTIUL(status, level) + i = ti - il + 1 + return app.cg_btables[level][i] +end + +#===================== +XBraid functions +=====================# + +function my_init(app, t) + t == 0.0 && return ThetaVector(sin.(range(0., 2π-app.Δx, app.nx)), app.cgorder) + u = randn(app.nx) + # u = zeros(app.nx) + return ThetaVector(u, app.cgorder) +end + +T = forest +function my_step!(app::AdvDifApp, status::XBraid.Status.StepStatus, u::ThetaVector, ustop::ThetaVector, tstart, tstop) + Δt = tstop - tstart + ti = XBraid.Status.getTIndex(status) + caller = XBraid.Status.getCallingFunction(status) + level = XBraid.Status.getLevel(status) + + # get Butcher table for this step + btable = get_btable(app, status, ti) + + # f-relax + if app.flag_refine_downcycle && caller ∈ [XBraid.CallerFRestrict, XBraid.CallerFASResidual] + cfactor = XBraid.Status.getCFactor(status) + # first step of f-interval + if XBraid.isCPoint(ti, cfactor) + u.ψ .= 0. + u.t_prior = tstart + end + Δt_prior = tstart - u.t_prior + ψ_old = deepcopy(u.ψ) + ψ_step = [ψ(btable, t) for t ∈ T.keys[1:length(u.ψ)]] + + # second order + u.ψ[T[:{τ}]] += Δt^2 * ψ_step[T[:{τ}]] + Δt * Δt_prior + + # third order + if app.cgorder >= 3 + u.ψ[T[:{{τ}}]] += Δt^3 * ψ_step[T[:{{τ}}]] + Δt * ψ_old[T[:{τ}]] + Δt^2 * Δt_prior * ψ_step[T[:{τ}]] + u.ψ[T[:{τ, τ}]] += Δt^3 * ψ_step[T[:{τ, τ}]] + 2Δt^2 * Δt_prior * ψ_step[T[:{τ}]] + Δt * Δt_prior^2 + end + + # last step of f-interval + if XBraid.isCPoint(ti+1, cfactor) + # normalize order conditions + Δt_c = tstop - u.t_prior + # 2nd order + u.ψ[T[:{τ}]] /= Δt_c^2 + f!, J!, fill, guess = θsdirk2_lhs!, θsdirk2_J!, θsdirk2_fill, θsdirk2_guess + # 3rd order + if app.cgorder >= 3 + u.ψ[T[:{{τ}}]] /= Δt_c^3 + u.ψ[T[:{τ, τ}]] /= Δt_c^3 + f!, J!, fill, guess = θsdirk3_lhs!, θsdirk3_J!, θsdirk3_fill, θsdirk3_guess + end + + # solve order conditions: + # @info """ + # Solving order conditions at t = $tstop + # ψ = $(u.ψ) + # """ + iu, il = XBraid.Status.getTIUL(status, level+1) + ic = XBraid.mapFineToCoarse(ti, cfactor) - il + 1 + app.cg_btables[level+1][ic] = solve_order_conditions(f!, J!, fill, app.cgorder, u.ψ; guess=guess) + end + end + + # turn off order conditions once upcycle starts + if app.flag_refine_downcycle && caller == XBraid.Status.CallerFInterp + @info "done computing coarse grid butcher tables" + app.flag_refine_downcycle = false + end + + # finally, take the step + embed = nothing + if level == 0 && app.order == 2 + embed = sdirk212_emb + end + # println("level: $level, ti: $ti") + # display(btable) + # println("emb: $embed") + err_est = dirk_step!(app, u.u, Δt, btable; embedding=embed) + if err_est >= app.spatial_tol && caller != XBraid.CallerWarmup + XBraid.Status.setRFactor(status, 2) + end +end + +function my_sync!(app::AdvDifApp, status::XBraid.SyncStatus) + app.useTheta || return + caller = XBraid.Status.getCallingFunction(status) + iter = XBraid.Status.getIter(status) + # println("sync! caller: $caller, iter: $(XBraid.Status.getIter(status))") + + if caller == XBraid.CallerDrive_AfterInit || caller == XBraid.Status.CallerFRefine_AfterInitHier + app.cg_btables = Vector{ButcherTable}[] + # initialize storage for coarse grid butcher tables + nlevels = XBraid.Status.getNLevels(status) + for l ∈ 1:nlevels-1 + iu, il = XBraid.Status.getTIUL(status, l) + ncpoints = iu - il + 1 + push!(app.cg_btables, [app.fine_btable for i ∈ 1:ncpoints]) + end + + # turn on computation of coarse grid butcher tables + app.flag_refine_downcycle = true + end + + # make sure we still compute these when we skip the first downcycle + if app.skip_downcycle && caller == XBraid.CallerDrive_TopCycle && iter == 0 + app.flag_refine_downcycle = true + end + app.flag_refine_downcycle && @info "Recomputing coarse grid butcher tables" + return +end + +function my_norm(app::AdvDifApp, u::ThetaVector) + LinearAlgebra.normInf(u.u) +end + +function my_sum!(app::AdvDifApp, α::Real, x::ThetaVector, β::Real, y::ThetaVector) + y.u .= α * x.u + β * y.u +end + +function my_access(app::AdvDifApp, status, u) + XBraid.Status.getWrapperTest(status) && return + t = XBraid.Status.getT(status) + ti = XBraid.Status.getTIndex(status) + ntime = XBraid.Status.getNTPoints(status) + done = XBraid.Status.getDone(status) + if ti == ntime + push!(app.solTf, deepcopy(u.u)) + end + if done + push!(app.times, t) + end +end + +function test(;ν=1., α=1., order=1) + MPI.Init() + comm = MPI.COMM_WORLD + + my_app = AdvDifApp(512, 2π/513, ν, α, false, false, order, 2, 2) + app = XBraid.BraidApp(my_app, comm, my_step!, my_init, my_access; sum=my_sum!, spatialnorm=my_norm) + + XBraid.testInitAccess(app, 0.) + XBraid.testSum(app, 0.) + XBraid.testSpatialNorm(app, 0.) + u = my_init(my_app, 0.) + println(my_norm(my_app, u)) + plot(u.u, label="u0", legend=:bottomright) + my_step!(my_app, XBraid.StepStatus(), u, u, 0., .05) + println(my_norm(my_app, u)) + plot!(u.u, label="u1") +end + +function main(;tstop=2π, ntime=512, nx=512, ν=1., α=1., useTheta=false, useRich=false, cf=2, ml=2, maxiter=10, order=1, skip=false, refine=false, maxrefine=2) + MPI.Init() + comm = MPI.COMM_WORLD + + Δx = 2π / nx + app = AdvDifApp(nx, Δx, ν, α, useTheta, useRich, order, cf, ml; skip=skip) + + core = XBraid.Init(comm, 0., tstop, ntime, my_step!, my_init, my_access; + app=app, sync=my_sync!, sum=my_sum!, spatialnorm=my_norm) + + XBraid.setPrintLevel(core, 2) + XBraid.setMaxLevels(core, ml) + XBraid.setCFactor(core, -1, app.cf) + XBraid.setAccessLevel(core, 1) + XBraid.setNRelax(core, -1, 1) + XBraid.setAbsTol(core, 1e-8) + XBraid.setMaxIter(core, maxiter) + XBraid.setSkip(core, skip) + XBraid.setRefine(core, refine) + XBraid.setMaxRefinements(core, maxrefine) + XBraid.setNRelax(core, 0, 0) + + XBraid.Drive(core) + + append!(app.residuals, XBraid.getRNorms(core)) + + return app +end \ No newline at end of file diff --git a/drivers/julia/drive-TaylorGreen.jl b/drivers/julia/drive-TaylorGreen.jl index aa0534ac..e1266e09 100644 --- a/drivers/julia/drive-TaylorGreen.jl +++ b/drivers/julia/drive-TaylorGreen.jl @@ -244,9 +244,9 @@ function my_sum!(app, a, x, b, y) end function my_access(app, status, u) - index = XBraid.status_GetTIndex(status) - lyap_exps = XBraid.status_GetLocalLyapExponents(status) - lyap_vecs = XBraid.status_GetBasisVectors(status) + index = XBraid.Status.getTIndex(status) + lyap_exps = XBraid.Status.getLocalLyapExponents(status) + lyap_vecs = XBraid.Status.getBasisVectors(status) if length(lyap_vecs) > 0 push!(app.lyapunov_vecs, deepcopy(reduce(hcat, lyap_vecs))) push!(app.lyapunov_exps, lyap_exps) diff --git a/drivers/julia/drive-advdiff-theta.jl b/drivers/julia/drive-advdiff-theta.jl index add1d651..d79b7b2c 100644 --- a/drivers/julia/drive-advdiff-theta.jl +++ b/drivers/julia/drive-advdiff-theta.jl @@ -64,7 +64,7 @@ function θSDIRK2!(app, u, Δt, m) return u end -function base_step!(app, u, Δt, level) +function base_step!(app, u, Δt, level; step_scaling=1) if app.order == 2 if app.useTheta throw("Theta method not implemented for order 2") @@ -73,7 +73,7 @@ function base_step!(app, u, Δt, level) end if app.useTheta && level > 0 - return θSDIRK2!(app, u, Δt, app.cf^level) + return θSDIRK2!(app, u, Δt, app.cf^level ÷ step_scaling) # return θ_euler!(app, u, Δt, app.cf^level) end @@ -82,7 +82,7 @@ end function my_step!(app, status, u, ustop, tstart, tstop) Δt = tstop - tstart - level = XBraid.status_GetLevel(status) + level = XBraid.Status.getLevel(status) if !app.useRich || level == 0 return base_step!(app, u, Δt, level) end @@ -93,17 +93,17 @@ function my_step!(app, status, u, ustop, tstart, tstop) end θ = 2^p * (m^p - 1) / (m^p * (2^p - 1)) u_sub = deepcopy(u) - base_step!(app, u_sub, Δt/2, level) - base_step!(app, u_sub, Δt/2, level) + base_step!(app, u_sub, Δt/2, level; step_scaling=2) + base_step!(app, u_sub, Δt/2, level; step_scaling=2) base_step!(app, u, Δt, level) @. u = θ*u_sub + (1-θ)*u return u end function my_access(app, status, u) - XBraid.status_GetWrapperTest(status) && return - ti = XBraid.status_GetTIndex(status) - ntime = XBraid.status_GetNTPoints(status) + XBraid.Status.getWrapperTest(status) && return + ti = XBraid.Status.getTIndex(status) + ntime = XBraid.Status.getNTPoints(status) if ti == ntime push!(app.solTf, deepcopy(u)) end diff --git a/drivers/julia/drive-burgers.jl b/drivers/julia/drive-burgers.jl index 8225e22b..d0c05d96 100644 --- a/drivers/julia/drive-burgers.jl +++ b/drivers/julia/drive-burgers.jl @@ -74,24 +74,7 @@ function semi_lagrangian!(burger::BurgerApp, y, v, Δt) sitp = scale(itp, range(0., lengthScale - Δx, nₓ)) extp = extrapolate(sitp, Periodic()) - # forward euler x_back .= burger.x .- Δt * v - - # backward euler - # max_newton = 1000 - # av_iters = 0. - # for i ∈ eachindex(x_back) - # for iter ∈ 1:max_newton - # p = extp(x_back[i]) - # p_prime = Interpolations.gradient(extp, x_back[i])[] - # x_back[i] -= (x_back[i] + Δt * p - burger.x[i]) / (1 + Δt * p_prime) - # if abs(x_back[i] + Δt * p - burger.x[i]) < 1e-10 - # # av_iters += iter/nₓ - # break - # end - # end - # end - # println(av_iters) # y_intp .= interp_periodic.(x_back, Ref(y)) y_intp .= extp.(x_back) @@ -111,7 +94,7 @@ function diffuse_beuler!(burger::BurgerApp, y, Δt::Float64; init_guess=nothing) return y end -function diffuse_fft!(burger::BurgerApp, y::AbstractArray, Δt::Real; init_guess=nothing) +function diffuse_fft!(burger::BurgerApp, y::AbstractArray, Δt::Real) P̂ = burger.P̂ κ = burger.κ # y .= real(P̂ \ (exp.(Δt*(κ.^2 - κ.^4)))) @@ -138,11 +121,11 @@ function fillDualArray!(y::Vector{ForwardDiff.Dual{T,V,P}}, vs, ps) where {T,V,P end # this enables ForwardDiff through the FFT where it normally doesn't work -function diffuse_fft!(burger::BurgerApp, y::Vector{ForwardDiff.Dual{T,V,P}}, Δt::Real; μ=1e-4) where {T, V, P} +function diffuse_fft!(burger::BurgerApp, y::Vector{ForwardDiff.Dual{T,V,P}}, Δt::Real) where {T, V, P} vs = ForwardDiff.value.(y) ps = extractPartials(y) - diffuse_fft!(burger, vs, Δt; μ=μ) - map(eachcol(ps)) do p diffuse_fft!(burger, p, Δt; μ=μ) end + diffuse_fft!(burger, vs, Δt) + map(eachcol(ps)) do p diffuse_fft!(burger, p, Δt) end fillDualArray!(y, vs, ps) return y end @@ -174,21 +157,21 @@ end function base_step!(burger::BurgerApp, u, Δt::Float64; init_guess=nothing) u_mid = deepcopy(u) - semi_lagrangian!(burger, u_mid, u, Δt/2) - semi_lagrangian!(burger, u, u_mid, Δt) # u(∇ u) + # semi_lagrangian!(burger, u_mid, u, Δt/2) + # semi_lagrangian!(burger, u, u_mid, Δt) # u(∇ u) # semi_lagrangian!(burger, u, sin.(burger.x), Δt) # ∇ u - # semi_lagrangian!(burger, u, u, Δt) # u(∇ u) + semi_lagrangian!(burger, u, u, Δt) # u(∇ u) if burger.μ > 0. diffuse_fft!(burger, u, Δt) # Δu end end function my_step!( - burger::BurgerApp, status::XBraid.StepStatus, + burger::BurgerApp, status::XBraid.Status.StepStatus, u, ustop, tstart::Float64, tstop::Float64 ) Δt = tstop - tstart - level = XBraid.status_GetLevel(status) + level = XBraid.Status.getLevel(status) if !burger.useTheta || level == 0 base_step!(burger, u, Δt; init_guess=ustop) else @@ -208,7 +191,7 @@ function my_step!( return u end -function my_step!(burger::BurgerApp, status::XBraid.StepStatus, u, ustop, tstart::Float64, tstop::Float64, Ψ) +function my_step!(burger::BurgerApp, status::XBraid.Status.StepStatus, u, ustop, tstart::Float64, tstop::Float64, Ψ) rank = length(Ψ) Ψ_new = reduce(hcat, Ψ) # perturb(r) = base_step!(burger, u + r' * Ψ, Δt) @@ -225,34 +208,27 @@ function my_sum!(burger, a, x, b, y) @. y = a*x + b*y end -function my_access(burger::BurgerApp, status::XBraid.AccessStatus, u) - XBraid.status_GetWrapperTest(status) && return - ti = XBraid.status_GetTIndex(status) +function my_access(burger::BurgerApp, status::XBraid.Status.AccessStatus, u) + XBraid.Status.getWrapperTest(status) && return + ti = XBraid.Status.getTIndex(status) push!(burger.solution, deepcopy(u)) push!(burger.times, ti) - if XBraid.status_GetDeltaRank(status) > 0 - Ψ = XBraid.status_GetBasisVectors(status) - λ = XBraid.status_GetLocalLyapExponents(status) + if XBraid.Status.getDeltaRank(status) > 0 + Ψ = XBraid.Status.getBasisVectors(status) + λ = XBraid.Status.getLocalLyapExponents(status) push!(burger.lyap_vecs, deepcopy(reduce(hcat, Ψ))) push!(burger.lyap_exps, deepcopy(λ)) end end -my_norm(burger, u) = LinearAlgebra.norm2(u) -my_innerprod(burger, u, v) = u' * v - # test user routines: function test() x_new = zeros(nₓ) y_new = zeros(nₓ) x = Array(range(0., lengthScale-Δx, nₓ)) κ = 2π/lengthScale .* fftfreq(nₓ, nₓ) - burger = BurgerApp(x, κ, 0., 4, false, x_new, y_new); - test_app = XBraid.BraidApp( - burger, comm, comm, - my_step!, my_init, - my_sum!, my_norm, my_access, - my_basis_init, my_innerprod) + burger = BurgerApp(x, κ, 1e-3, 4, false, x_new, y_new); + test_app = XBraid.BraidApp(burger, comm, my_step!, my_init, my_access; basis_init=my_basis_init) open("drive-burgers.test.out", "w") do file cfile = Libc.FILE(file) @@ -261,10 +237,10 @@ function test() XBraid.testDelta(test_app, 0.0, 0.1, 3, cfile) end # plot!(burger.lyap_vecs[1][1]) - plot(burger.solution[1]) + # plot(burger.solution[1]) end -function main(;tstop=π, ntime=128, deltaRank=0, useTheta=false, fmg=false, ml=1, cf=4, saveGif=false, maxiter=30, μ=0.) +function main(;tstop=π, ntime=nₓ, deltaRank=0, useTheta=false, fmg=false, ml=1, cf=4, saveGif=false, maxiter=30, μ=0.) x_new = zeros(nₓ) y_new = zeros(nₓ) x = Array(range(0., lengthScale-Δx, nₓ)) @@ -272,35 +248,31 @@ function main(;tstop=π, ntime=128, deltaRank=0, useTheta=false, fmg=false, ml=1 burger = BurgerApp(x, κ, μ, 4, useTheta, x_new, y_new); tstart = 0.0 - core = XBraid.Init( - comm, comm, tstart, tstop, ntime, - my_step!, my_init, my_sum!, my_norm, my_access; app=burger - ) + core = XBraid.Init(comm, tstart, tstop, ntime, my_step!, my_init, my_access; app=burger) if deltaRank > 0 - XBraid.SetDeltaCorrection(core, deltaRank, my_basis_init, my_innerprod) - XBraid.SetLyapunovEstimation(core; exponents=true) + XBraid.setDeltaCorrection(core, deltaRank, my_basis_init) + XBraid.setLyapunovEstimation(core; exponents=true) end # println("Wrapper test:") # @time test() - XBraid.SetMaxIter(core, maxiter) - XBraid.SetMaxLevels(core, ml) - XBraid.SetCFactor(core, -1, cf) - XBraid.SetAccessLevel(core, 1) - XBraid.SetNRelax(core, -1, 1) - XBraid.SetAbsTol(core, 1e-6) - XBraid.SetSkip(core, false) + XBraid.setMaxIter(core, maxiter) + XBraid.setMaxLevels(core, ml) + XBraid.setCFactor(core, -1, cf) + XBraid.setAccessLevel(core, 1) + XBraid.setNRelax(core, -1, 1) + XBraid.setAbsTol(core, 1e-6) + XBraid.setSkip(core, false) if fmg - XBraid.SetFMG(core) - XBraid.SetNFMG(core, 1) + XBraid.setFMG(core) + XBraid.setNFMG(core, 1) end - XBraid.SetTimings(core, 2) - XBraid.Warmup(core) - XBraid.Drive(core; warmup=false) - XBraid.PrintTimers(core) + XBraid.setTimings(core, 2) + XBraid.Drive(core) + XBraid.printTimers(core) if deltaRank > 0 exponents = sum(burger.lyap_exps) @@ -326,7 +298,7 @@ function main(;tstop=π, ntime=128, deltaRank=0, useTheta=false, fmg=false, ml=1 end end - return burger, XBraid.GetRNorms(core) + return burger, XBraid.getRNorms(core) end # test() diff --git a/drivers/julia/drive-kflow.jl b/drivers/julia/drive-kflow.jl index 902bee34..9c61f738 100644 --- a/drivers/julia/drive-kflow.jl +++ b/drivers/julia/drive-kflow.jl @@ -4,13 +4,11 @@ using MPI, Base.Threads using BenchmarkTools, Plots, LaTeXStrings using Statistics: mean using Random: seed! -using LinearAlgebra: norm +using LinearAlgebra: norm, normInf include("../../braid/braid.jl/XBraid.jl") using .XBraid -MPI.Init() -comm = MPI.COMM_WORLD Float = Float64 @@ -201,6 +199,7 @@ end function project_incompressible!(app::KFlowApp, u::AbstractArray{ForwardDiff.Dual{T, V, P}}, Δt::Real) where {T, V, P} vs = ForwardDiff.value.(u) ps = extractPartials(u) + # @info "size(ps)" project_incompressible!(app, vs, Δt) map(eachslice(ps, dims = 4)) do p project_incompressible!(app, p, Δt) @@ -258,30 +257,30 @@ end function my_init(app::KFlowApp, t::Float) seed!(1) - # u = zeros(Float, nₓ, nₓ, 2) - u = randn(Float, nₓ, nₓ, 2) - # TaylorGreen!(app, u) + u = zeros(Float, nₓ, nₓ, 2) + # u = randn(Float, nₓ, nₓ, 2) + TaylorGreen!(app, u) project_incompressible!(app, u, 1e-1app.ℜ) u[:, :, 1] .-= mean(u[:, :, 1]) u[:, :, 2] .-= mean(u[:, :, 2]) - u ./= my_norm(app, u) + u ./= my_norm(app, u)/2 return u end function my_basis_init(app::KFlowApp, t::Float, i::Integer) ψ = zeros(Float, nₓ, nₓ, 2) - fourierMode2DVec!(app, ψ, i + 1) + fourierMode2DVec!(app, ψ, i) return ψ end function my_step!( - app::KFlowApp, status::Ptr{Cvoid}, + app::KFlowApp, status::XBraid.StepStatus, u::AbstractArray, ustop::AbstractArray, tstart::Real, tstop::Real, ) Δt = tstop - tstart - level = XBraid.status_GetLevel(status) - iter = XBraid.status_GetIter(status) + level = XBraid.Status.getLevel(status) + iter = XBraid.Status.getIter(status) if !app.useTheta || level == 0 u = base_step!(app, u, Δt) else @@ -300,7 +299,7 @@ function my_step!( end function my_step!( - app::KFlowApp, status::Ptr{Cvoid}, + app::KFlowApp, status::XBraid.StepStatus, u::AbstractArray, ustop::AbstractArray, tstart::Real, tstop::Real, Ψ::Vector{Any} @@ -317,23 +316,19 @@ function my_step!( return end -function my_sum!(app, a, x, b, y) - @. y = a * x + b * y -end - -function my_access(app::KFlowApp, status, u) - XBraid.status_GetWrapperTest(status) && return - ntime = XBraid.status_GetNTPoints(status) - index = XBraid.status_GetTIndex(status) - level, done = XBraid.status_GetLevel(status), XBraid.status_GetDone(status) +function my_access(app::KFlowApp, status::XBraid.AccessStatus, u) + XBraid.Status.getWrapperTest(status) && return + ntime = XBraid.Status.getNTPoints(status) + index = XBraid.Status.getTIndex(status) + level, done = XBraid.Status.getLevel(status), XBraid.Status.getDone(status) if level == 0 && done && index % app.cf == 0 push!(app.solution, deepcopy(u)) push!(app.times, index) - rank = XBraid.status_GetDeltaRank(status) + rank = XBraid.Status.getDeltaRank(status) if rank > 0 - exps = XBraid.status_GetLocalLyapExponents(status) - vecs = XBraid.status_GetBasisVectors(status) + exps = XBraid.Status.getLocalLyapExponents(status) + vecs = XBraid.Status.getBasisVectors(status) push!(app.lyapunov_exps, exps) push!(app.lyapunov_vecs, deepcopy(vecs)) end @@ -345,21 +340,19 @@ function my_access(app::KFlowApp, status, u) end end -my_norm(app, u) = LinearAlgebra.normInf(u) -my_innerprod(app, u, v) = u ⋅ v +my_norm(app, u) = normInf(u) const lengthScale = 2π -const nₓ = 99 +const nₓ = 256 const Δx = lengthScale / nₓ +MPI.Init() +comm = MPI.COMM_WORLD + function test() my_app = KFlowApp(2, false, 3, 2, 1600) - test_app = XBraid.BraidApp( - my_app, comm, comm, - my_step!, my_init, - my_sum!, my_norm, my_access, - my_basis_init, my_innerprod) + test_app = XBraid.BraidApp(my_app, comm, my_step!, my_init, my_access; spatialnorm=my_norm, basis_init=my_basis_init) XBraid.testInitAccess(test_app, 0.0) XBraid.testClone(test_app, 0.0) @@ -371,31 +364,28 @@ function test() # heatmap(curl') end -function main(;tstop=20.0, ntime=512, deltaRank=0, ml=1, cf=2, maxiter=10, fcf=1, relaxLyap=false, savegif=false, useTheta=false, k=4, ℜ=1600, deferDelta=(1, 1)) +function main(;tstop=10.0, ntime=1024, deltaRank=0, ml=1, cf=2, maxiter=10, fcf=1, relaxLyap=false, savegif=false, expPlot=false, useTheta=false, k=4, ℜ=100, deferDelta=(1, 1)) my_app = KFlowApp(cf, useTheta, deltaRank, k, ℜ) tstart = 0.0 - core = XBraid.Init( - comm, comm, tstart, tstop, ntime, - my_step!, my_init, my_sum!, my_norm, my_access; app = my_app, - ) + core = XBraid.Init(comm, tstart, tstop, ntime, my_step!, my_init, my_access; spatialnorm=(app, u)->normInf(u), app=my_app) if deltaRank > 0 - XBraid.SetDeltaCorrection(core, deltaRank, my_basis_init, my_innerprod) - # XBraid.SetDeferDelta(core, deferDelta...) - XBraid.SetLyapunovEstimation(core; relax = relaxLyap, exponents = true) + XBraid.setDeltaCorrection(core, deltaRank, my_basis_init) + # XBraid.setDeferDelta(core, deferDelta...) + XBraid.setLyapunovEstimation(core; relax = relaxLyap, exponents = true) end - XBraid.SetMaxLevels(core, ml) - XBraid.SetMaxIter(core, maxiter) - XBraid.SetCFactor(core, -1, cf) - XBraid.SetNRelax(core, -1, fcf) - XBraid.SetAccessLevel(core, 2) - XBraid.SetAbsTol(core, 1e-6) - - XBraid.SetTimings(core, 2) + XBraid.setMaxLevels(core, ml) + XBraid.setMaxIter(core, maxiter) + XBraid.setCFactor(core, -1, cf) + XBraid.setNRelax(core, -1, fcf) + XBraid.setAccessLevel(core, 2) + XBraid.setAbsTol(core, 1e-6) + + XBraid.setTimings(core, 2) XBraid.Drive(core) - XBraid.PrintTimers(core) + XBraid.printTimers(core) p = sortperm(my_app.times) @@ -408,23 +398,22 @@ function main(;tstop=20.0, ntime=512, deltaRank=0, ml=1, cf=2, maxiter=10, fcf=1 end end - # if deltaRank > 0 - # movingaverage(g, n) = [i < n ? mean(g[begin:i]) : mean(g[i-n+1:i]) for i in eachindex(g)] - # println("Lyap Exps: ", sum(my_app.lyapunov_exps) ./ tstop) - # exps = reduce(hcat, my_app.lyapunov_exps[p])' - # nt = size(exps)[1] - # Δt = tstop / nt - # exps = movingaverage.([exps[:, i] ./ Δt for i ∈ 1:size(exps)[2]], length(exps[:, 1])) - # exps_plot = plot(exps; legend = false) - # savefig(exps_plot, "lyapunov_exps.png") - # end + if deltaRank > 0 && expPlot + movingaverage(g, n) = [i < n ? mean(g[begin:i]) : mean(g[i-n+1:i]) for i in eachindex(g)] + exps = reduce(hcat, my_app.lyapunov_exps[p])' + nt = size(exps)[1] + Δt = tstop / nt + exps = movingaverage.([exps[:, i] ./ Δt for i ∈ 1:size(exps)[2]], length(exps[:, 1])) + exps_plot = plot(exps; legend = false) + savefig(exps_plot, "lyapunov_exps.png") + end if savegif preprocess(app, u) = spectralInterpolation(∇X(app, u), 128)' heatmapArgs = Dict(:ticks => false, :colorbar => false, :aspect_ratio => :equal) - # anim = @animate for i in p[1:cf:end] count = 1 - for i in p + anim = @animate for i in p[1:cf:end] + # for i in p u = my_app.solution[i] plots = [heatmap(preprocess(my_app, u); heatmapArgs...)] for j in 1:min(8, deltaRank) @@ -432,12 +421,11 @@ function main(;tstop=20.0, ntime=512, deltaRank=0, ml=1, cf=2, maxiter=10, fcf=1 push!(plots, heatmap(preprocess(my_app, ψⱼ); heatmapArgs...)) end fig = plot(plots...; size = (600, 600)) - savefig(fig, "kflow_gif/beamer/kolmo_$(nₓ)_$(ntime)_ml$(ml)-$(count).png") + # savefig(fig, "kflow_gif/beamer/kolmo_$(nₓ)_$(ntime)_ml$(ml)-$(count).png") count += 1 end - # gif(anim, "kflow_gif/kolmo_$(nₓ)_$(ntime)_ml$(ml).gif", fps = 20) + gif(anim, "kflow_gif/kolmo_$(nₓ)_$(ntime)_ml$(ml).gif", fps = 20) end - return my_app, core end diff --git a/drivers/julia/matrix_stencils.ipynb b/drivers/julia/matrix_stencils.ipynb new file mode 100644 index 00000000..018bb250 --- /dev/null +++ b/drivers/julia/matrix_stencils.ipynb @@ -0,0 +1,312 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "using ToeplitzMatrices, LinearAlgebra, SparseArrays\n", + "using Plots" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "4×4 SparseMatrixCSC{Float64, Int64} with 10 stored entries:\n", + " -5.09348 0.000259382 ⋅ ⋅ \n", + " 5.09322 -5.09348 0.000259382 ⋅ \n", + " ⋅ 5.09322 -5.09348 0.000259382\n", + " ⋅ ⋅ 5.09322 -5.09348" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + " α = 1.\n", + " ν = 1e-5\n", + " nx = 32\n", + " Δx = 2π/nx\n", + " I = SymmetricToeplitz([1, zeros(nx-1)...])\n", + " Δ = (1/Δx^2)Circulant([-2, 1, zeros(nx-3)..., 1])\n", + " # ∇ = (1/Δx)Circulant([0, -1/2, zeros(nx-3)..., 1/2])\n", + " ∇ = (1/Δx)Circulant([1, -1, zeros(nx-2)...])\n", + " A = ν*Δ - α*∇;\n", + " A = sparse(A)\n", + " display(A[1:4, 1:4])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "m = 4\n", + "Δt = Δx\n", + "# forward euler\n", + "# Φ = I + Δt*A;\n", + "# backward euler\n", + "Φ = inv(I - Δt*A)\n", + "Φ_m = inv(I - m*Δt*A);" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# SDIRK2\n", + "a = √2/2\n", + "K1 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A))\n", + "K2 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A*(I + (2a-1)*K1)))\n", + "Φ_sdirk = I + 1/2*(K1 + K2);" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# θSDIRK2\n", + "function θSDIRK2(m)\n", + " a = √2√(m*(m-1))/2m\n", + " θ = (m^2 -3m - √2√(m*(m-1)))/(2m^2 - 4m)\n", + " K1 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A))\n", + " K2 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A*(I + (2a-1)*K1)))\n", + " Φ_θ = I + (θ)K1 + (1-θ)K2;\n", + "end\n", + "Φ_θ = θSDIRK2(m);" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "# θ extrapolation\n", + "p = 1\n", + "θ = 2^p * (m^p - 1) / (m^p * (2^p - 1))\n", + "Φ_θ_extrap = θ*inv(I - m/2*Δt*A)^2 + (1-θ)*inv(I - m*Δt*A);" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "# both?\n", + "p = 2\n", + "θ = 2^p * (m^p - 1) / (m^p * (2^p - 1))\n", + "Φ_both = θ*θSDIRK2(m÷2)^2 + (1-θ)*θSDIRK2(m);" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAGQCAIAAAD9V4nPAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdd2BN5/8H8M855+6be5N7sxNZVsQeCRIjSBDU3i1tFaWbVqm2ulR1/FqUamt8lVZRtKVUrdhbCCFBkL2Tm3H3OOf8/jhERMg6yZXk8/orPfec5zy3xtuzCZZlASGEEGqqSHtXACGEELInDEKEEEJNGgYhQgihJg2DECGEUJOGQYgQQqhJwyBECCHUpGEQIoQQatIwCBFCCDVpGIQIIYSaNAxChBBCTdrTFYQJCQlr1661dy3qm81ms3cV7MBqtdq7CnaA37rpsNlsTXADS4ZhaJq2dy2q7ekKwvj4+P3799u7FvWKZVmz2WzvWtiByWSydxXsAL9102GxWBiGsXct6htN0w3xX/ZPVxAihBBC9QyDECGEUJOGQYgQQqhJwyBECCHUpGEQIoQQatIwCBFCCDVpGIQIIYSaNAxChBBCTRoGIUI1tPDo4uf3vGamLfauCEKoVjAIEaqJ81mXTmdcSClOO5tx0d51QQjVisDeFUCoQdoU9wf3Q3TKiXDfMPtWBjUsZ86cWbZsmb1rUSdYlmVZliTrpIk1YMCA2bNn10XJGIQIVdu5zJi4vHilSKGz6s5mXjTZTBKBxN6VQg3G5cuXi4uLZ8yYYe+KNCQXLlzYv38/BiFCT4tf4rYAwJT240+knY3Liz+VcSHCr4+9K4UakpYtW44fP97etWhIhEJhYmJiHRWOY4QIVc+ZjIvx+becxI4jWkUN8OsDAEdSTti7UgihmsMgRKh6NsZtBYDn2o+TCiT9fMNIgjybEaO3GuxdL4RQDWEQIlQNpzMuJBTcUktVI1pGAYBaqurs3t7KWE+mnbV31RBCNYRBiFA1cM3BZ9uOlQjE3BWudzQ65aQ9q4UQqgUMQoSq6lT6+RsFiWqpanjLwaUX+/qECkjqYvblErPWjnVDCNUYBiFCVbXp2jYAmNJuXGlzEAAcxcqu7p1sDH0iHXtHEWqQMAgRqpKT6eduFCQ6S1XPlGkOcgb49QaAaJw7ihq+w4cP//bbbxV+lJ+f//777+/Zs6eeq1QPMAgRqhwLLNccfK7deDElKvdpH59QISm8lH210FRsj9ohxAODwfDzzz+///77H3300XfffZebm1v2U41GM3DgwJSUlBkzZuzcudNelawjGIQIVe5k2tmbBbedpapnWg569FMHkTzEswvDMsfTTtd/3RCqvbt377Zv337Pnj3u7u7Ozs7x8fFBQUEnT96bAlZcXDxkyJCRI0du3rz5yJEj77333u7du7mPsrKyduzYkZiY+Omnny5fvlyv1+fn53/zzTdLly7Nycmx3xeqHtxZBqFKsMD+ErcVAKa2n/Boc5AzwL/36Yzz0SknR7YaUr+1Q41BgRliC1iWrafXBTqBj5woe2XRokXt27ffvXv3p59+GhcXt27dOjc3t9mzZ1+7dg0A1q9f/8Ybb0yZMgUAgoKCjh079tFHH/Xo0cPd3f3WrVuvvfZa165dhw0b9tdffx09etRkMg0ZMuT69euDBw++fPkyQRAVV+JpgkGIUCWOp565XZjkIlUPbTHwcff08u4hpkRXc6/lGwpcZM71WT3UCET+a4stqK8YBHAQQt4UoYR6cOXixYsLFy4se8/o0aOXLl2q1WoVCsXbb79d9iMvL69169aV/qdOp9uyZYuTk9PgwYNbt2595syZnj170jTt4uJy9+7dFi1a1PG34QEGIUJPUjo6+HyHiY9rDgKATCjt4dXteNqZY2mnxwYOr8cKosZgUnPSRcLU2+taKgkx9dAVZ2fncoOCubm5EolEIql8N3k/Pz8nJycA8PDwAIBOnToBAEVRrq6u+fn5GIQINXjHUk/fLkxyk7kMbR755DsH+PU5nnYmOuUEBiGqrgWdyAWd7DljY8KECV988cWQIfc69rOysj744IPnn39eKBRW+ixF3QtVrhe07H/WX29v7WAQIvRYLLCb4rYBwJT244VUJX8jhHqHSAWS63k3c/S57nK3eqkgQvx46623GIYZMGAATdM2my0oKGjatGlLly61d73qCc4aReixjqaculOU7C53rbQ5CAASgTjUO4QF9mgqzh1FDQxBEG+//XZOTs60adPCwsLy8vKWLVtWlX7RxgGDEKGKMey90cGp7SdU2hzk4Mp61KCRJLlkyZKdO3dWpUeU06tXr1OnTnE/y+VyjUYjEt0bSr9w4UJwcHCdVJRvGIQIVexI6sm7RSnucteo5hFVfKSnV7BcKLtRkJihzarTuiFURyQSiVwur/r9AoFAqVRyPxMEoVKpSj9SKpUCQcMYfcMgRKgCDMv+du0PAHi+/QQhWdU/zEJK2KtZDwA4knqqDiuHEOIVBiFCFTiScuJuUYq73G1wlZuDHDyzHqEGB4MQofIYltl4bRsAvNBhYtWbg5wQzy5KkeJ2YVJKcVrd1A4hO5s2bdqaNWvsXQs+YRAiVN7hlOMpxWnucrdBAf2r+6yApHr79ACAo9g7ihops9lstVrtXQs+YRAi9BCGZTbF/QEA0zpMqm5zkMP1jh5KPs5zzRCqS+np6d98882CBQv27dvHXUlOTt64cWPpDatWrSooKCj3VH5+/vLly999990dO3ZwV7Ra7bfffpuamvrJJ5+sXbu2fipfSw1jSg9C9eZQ8vHUknQvB49BzavdHOR09eiokjimlqQnFaUEOPnxWz3UKNHFBbac1Hp7HaVyE7h6l70SHx8fFRX1yiuvtG/fftGiRQkJCW+//XZiYuKqVateeOEF7p4lS5ZEREQ4Oz/YSjczM7NXr15Tpkzp0qXL6tWrz58///XXX5eUlCxcuHDPnj3jxo1zc2sYO0tgECL0AMMyv97fWZQiqErvrxBFUH19wnYl7otOOTkdgxBVQf5PH1izkuvtdYRQ5PXFDkL4YO/cjz/+eO7cuXPnzgWAXr16de/evdxG2xX68ssvx48fv3jxYgAYNmyYt7f3J598AgBWq3X58uXcpqMNAgYhQg8cTDqaWpLh5eAxKKBfbcoZ4Nd7V+K+Q8nHpnd6jqeqocZM3jPKeP1c/byLIEmhV0DZFASA2NjYq1evbt68mfvPgoICjUZTaVGxsbHp6enR0dHcf1osluTkZEdHR6FQ2KFDB95rXncwCBG6h2GZ365vB4AXO0yqcXOQ09GtvYtUnanLvqW501rdAHbfR/blED7KIXyUPSvg4PDee+8NGvTg3GlHR0eBQGCz2Uqv6PX6R5+aM2fO1KlTS68olcrs7GyBQECSDWkCSkOqK0J1an/SkdSSDG+FZ2RAeC2LIgmir28Y4HZrqIF45plnNm7cKJfLVSqVSqUyGAwkSQYEBNy+fZtrGu7evVur1T761G+//SYSiUqfKj16omHBIEQIAIBhmc3XdwDAix0m17I5yOHmjkannGChYZxEg5qy999/39XVtXnz5oMHD+7QocPMmTMBwN/f/9lnn+3QoUOvXr327Nnz6MyX2bNnh4WFtWrVauDAgV27dh0xYoQ96s4D7BpFCADgeNqZtJIMX6V3pH9fXgps79rGTeaSo89LyL/V1iWQlzIRqiNSqXTjxo0lJSXJycnu7u7u7u7c9Z9//jklJcVms7Vo0eKbb75xcHAAgA0bNnAtP5Ikly9fvmTJkjt37jg7O3t7ewOAp6dnZmamHb9LDWCLECGA+32YI1sNJQl+/lAQQPT36w0AR1JO8lIgQnVNqVR27NixNAU5fn5+3Cnzjo6OXP6JxeKyu2nL5fKOHTtyKQgAJElyB9Y3IBiECIHJZj6XeYkAoq9vKI/FlvaOMg3knG6EmiYMQoTgVMZ5k83U3jXITebCY7FtnFt5KzzzjZprefE8FosQ4heOESIER1NOAkB/v168l9zPt9fm6zuiU052dGvHe+EI8cVisfzwww8HDx7U6XTcHJmoqCiWZSdOnAgAUqnUycmpW7duo0aNKj19cO7cudOmTevYsWN0dPRPP/0EAI6OjkFBQS+99BLXNZqZmTlnzpwtW7ZwHar79u3bunXrN998k56e/ssvv1y9elWpVI4bN27q1KkEQdjvqwNgixAhk818PusyAUQfHz77RTkD7g8T0izNe+EI8WX+/Plbt2595513li1bFhERcevWLQBgWXb79u1hYWFDhw719/ffsGFDYGDgpUuXuEd27tyZkZEBAElJSbGxsePHjw8PD9+zZ09ERATDMACg1Wq3b9/OsiwA7Nq1a+rUqc8//7ybm9vff//t4eHxySefTJ06dcGCBT///LP9vvc92CJETR3XL9rBtS2//aKclqrmfo4+KcVpV3Kud/XoyHv5CPFiz5493377bUREBAB069at7EeDBw8OCgoCgLlz586ZM+fFF1+8evVqucednZ3Hjx8PAOHh4b6+vikpKQEBAaWf/vbbb2+//fbevXt79OgBAJ999lnpR4mJif/888/s2bPr7JtVCQYhaurqrl+U0883bGPctuiUExiE6HFy9LkJBYn19jp/R19/R5+yV5o3b75mzRo/P78OHTo8YVH8q6++umLFirt37zZv3rzCG7KzsymKksvlpVeWL1++YsWKo0ePtm3b9tH74+PjH1dUfcIgRE2ayWaqu35RTqR/+Ma4bcfTzswJsfM/e9FTa8GRz5KK6+/0CSEp/HfCFhH1YLvRtWvXzpkzJzQ0VCQSRUVFLV26tMJ88vf3JwgiKyur3Kd3796dNWuWXq8/evTokiVLyi69f++999atW1dhCu7evfu///6Li4vj75vVEAYhatJOZVyou35Rjq+yWYCTX1JRyqXsK0GKVnX0FtSgjW0z/GJWbL29zlfZrGwKAoCfn99ff/1lNBovXrz4xRdfDBo06ObNm4/OYdFoNCzLcsvqy3J0dIyMjMzNzT137pyHh0fZj3777bdZs2b5+voOGDCg7PWjR4/OmDFj9+7dnp6e/H2zGsIgRE1aXfeLcgb49V5flBKdciKoPQYhqsDwloOHtxxs71qAVCrt06fP2rVrfXx80tPTfXx8yt1w9OhRJycnbsiwrNIxwtDQ0P79+0dGRpaurx83bhxFUaNHj/7rr79Ks/DUqVMTJ07ctm1bWFhYHX+nKsFZo6jpqod+UU6kfzgAnEg/a6WtdfoihGpm69atOp2O+/nQoUMODg7lGmp5eXkbNmx4/fXXP/nkE5FIVFEZAABdu3YdPHjwF198Ufbi+PHj161bN3r0aO7AprNnz44aNWrDhg39+9fw7Gve8dYivHz5cnZ2dvfu3cueX1zKZDJdvnxZr9d36NCh3P49CNlLPfSLcrwcPFqpmicW3r2cHxfhVNujLRDi3d9///3yyy/7+PiYzWaLxfL777+LRCJuFUTPnj1pmpbL5Z06dVq7du3o0aOfXNTHH38cHBz87rvvlr3ItRdHjx79999///DDD/n5+cOGDeM+6tKlS+mSDHvhJwhfeumlY8eOtWvX7oUXXvjrr7969Xqoo+nSpUsRERGtW7d2dnY+derU119/PWvWLF7ei1Bt1E+/KGeAf5/EwrvH089GtMQgRE+drVu3mkymrKwsiUTi4eHBjQ6SJMk+fnfA1NR7s3umT58+ffr00uvt2rUzGo3cz2UfHz9+PBeHT09DsBQPQXj+/Pl//vnn5s2barV65cqV8+fPP3XqVNkbvLy8YmNj/fz8AGDfvn2jR49+8cUXxWJx7V+NUI1x/aIkQYT71McoRYRf3zWXN53LiTHTFjH12J4lhOxFIpGUXfzXpPAwRvjXX38NGTJErVYDwJQpU86cOZOTk1P2Bg8PDy4FASAoKMhisZT+ewEhezmdccFkM7VzaeMiq6Azn3fuctcgl1ZGm+lcZkw9vA4hVHU8tAjT0tJK/x2hUqkcHBzS0tIeNxD43XffDR069HGHdBiNxvT09D/++KP0Sr9+/Vxc6nb8xr5YlmUYhuuLb1Ls/q2505HCfXrVWzV6egXH59+6lpvQ27tHVe5naZvmx4UgkTvP+KRua1bH7P5rbRfct65wF80n9DfaV3Fxsc1mq3Cex9OA+9uyuk+RZOXtPR6C0GKxlD2bSiQSmc3mCu9ct27drl27Tp8+/biiCgsL09PTt23bVnrF29v70TUrjQnLsiaT6QlbOTRWZrNZKBTa6+2l/aI93bqaTKb6eamclAFAiUlbxTdaLh0x370OBGHU6wiqAa90su+vtb2YTCaWZSv8o221Po2Th+fOnfvnn38KhcLu3btv2rSJ+1v9wIEDq1evTk9Pd3Jy6t69++LFiymK+uGHH44dOwYALi4uvr6+AwcOLN2Vbdu2bRkZGW+//Xbpht0A4OnpOXHixNKVEosWLerXrx+3nVt2dvb8+fOnTJkSHBy8cuXK48ePGwyGrl27fvjhh4+uL2QYpgZ/WiUSSaVZyMOfLg8Pj/z8fO5nq9VaVFRU4QLJ33777dNPP42Oji5dX/IoLy+vnj177tixo/a1aii4fxvKZDJ7V6S+0TRtx299JiWGmy/q49ys3l6qkjsBgJm1VOmLs6z25G7uBwltoRTKOq5dHbLvr7UdicXiCoPwCcsP7GXfvn3R0dE3btwQiUSDBw9et27d7Nmzz549O2HChB9//DE4ODgzM/Off/5hGIaiqAsXLhAEMXPmzJycnCtXrvTr1+/1119funQpAMTHxyckJMD9DbvXr1/v5eUVExPTv3//Y8eO9ezZEwCio6N9fX0jIiJSU1MHDhw4evToQYMGXbhwISsr67333lOpVF999dWwYcMenUpKUVQd/UbiIQhDQ0OXLl3KsixBECdOnPDw8PD19S13z44dO+bPn3/w4MFWrXBBMbK/+pwvWkoukgGA3mqoys3GKyet2Sncz3SJhnJqzAMEyO42bNgwbdo0qVQKADNnzly2bNns2bMPHTrUv3//yZMnA0CrVq3Cwx9MePb394+MjASA5557bsKECT179hwxYkRoaPn1uKGhoUFBQVFRUceOHdu7dy8XhJybN28OGjTotddemz9/PgCEhISEhIRwH61cudLT0zMnJ6fe1trxEIRjxoxZtGjR7Nmz+/Tp89lnn73zzjtcm3rChAkBAQFfffXVxYsXJ0+ePGTIkF9//ZV7ZM6cOeW24UGo3tTzfNFSciEXhPrKb2XZkkNbAYAQCFmbldFq6rpuyL6MeWZ9Zj110QOARC1y8JGWvXLr1q0hQ4bcvXsXABQKBXcMU2Bg4Hfffbd58+ZBgwa5uro+rrTg4ODQ0NC9e/c+GoQcmqYzMjK4CZWcK1eufPzxxx9//HGFS+liY2PVanV9zg7hIQjFYvHJkydXrlx55MiRzz77bNKkSdz1iRMnqlQqAHB0dPz888/LPtIEh8TQ06N0HX39zBctJRfKAUBvrXzKtCnhgjX9DqVUiVt1McRE08UYhI1cwv9SDTkVT62oCwRFhC5tSwoeTOQpKCj48MMPJRIJANhstqKiIpqmx40bl5KS8vHHH0+ZMqVDhw7cSbwVFujv75+dnf3o9blz5yqVyri4uICAgLJrDdesWRMWFjZz5sxHH8nPz3/llVf+7//+rz5jgp8ReA8PjyVLlpS7OHbsWO6HVq1aLViwgJcXIVR7dukXBQC5UAoAekvlLcKSg9sAwGHAeNagBQC6BIOwkWsW4aq5rq231zn4SMqmIAC4urouWbJkyJAhAHDt2rV+/fpxOTRv3rx58+ZlZGRs37591qxZzZo1Gzhw4KMF5uXlVXi+RFRUlKenp0AgyMrK4vpdOZ988snWrVunT5++fv36sjNZCgsLBw0aNHny5Mclbh1pwFPREKoBe/WLAoBMWKUxQvPtq5ak66Rc6RA6RH/hEADQ2sL6qB+yH7dgJ7fgiheV1Y/AwMAbN25wQZiQkBAYGFj2U29v7zlz5vz777+nTp16NAhzc3PPnj1bYfOOO9R3woQJoaGhy5Yt44YDAcDV1fXw4cMRERFls7C4uDgqKqpfv37ltiqtB7jpNmpaTt1bRx9Uz/2iACAXygggjDYj88RlZCUHtgCAQ/hoQiyllGoAYEowCFHdmj59+rp163Q6ndVqXbVqFdeNeeTIkcuXL3M3JCcnX716tdy5ExqNJjo6evjw4e3atRs1atTjCicI4vPPP1+6dGlRUVHpRS4LY2JiZsyYwTCMXq8fMWJEp06dvv3227r5ik+CQYiaFnv1iwIASZBiSsSwrMn22GkRltSb5luXSYnMofdwAKAUKsCuUVT3IiMjx4wZ06pVq4CAgMDAwBdeeAEANBrN2LFjnZ2dAwMDO3XqNHXq1AkTJgCATCZbu3atWq0ODAx8//33R48efejQIa4rVSgUli4OUalUpeN8kZGRISEhv/zyCwAoFApui01XV9eDBw9euXJlwYIFJ0+ejIuL27Fjh7Ozs1qtVqvV9XlgL3aNoibEZDOdz7pkl35RjlwoM9FmvVUvE0orvEF7YAsAyHsPJ2UOAEAq1YBBiOrF4sWLP/jgA4ZhStfqjR07duzYsVqtVqvVenh4lA7mrV69evXq1RUW8uGHH3I/kCSp0Tz0+/bAgQPcD//991/pRXd395iYe5sOlru/PmEQoibkVMYFk81c//NFS8mEsgJTod5qrHAqujU71Xj9HCEUOfQdyV253zWqAZaFijbrQohH3KzRchQKhUKhqP/K1CfsGkVNCL/9orSJsZTYqvWITCCBxy8l1B74HVhWHjqEyz8AIIQiUurA0jbGqKtlbRFCj4NBiJoKfvtFjbnmS18nxiy9ZcyzVP0pmeCxE0dt+VmG2BMEJVD0G1v2OqnEYUJUT3JyctLT0+1dCzvAIERNBdcvyst8UV2G6eqqJHORlTYzt7aks0xVDxO4t7mMpYIg1B7aCgwtC4mk1G5lr+PEUVQPWJadNWtW7969Bw8ePGrUKIulGv+8AwCdTrdhw4Y6qls9wCBETQVf/aIlSYZrPyRZdTZVkELsJNQmGzKO5FfxWZlAChW1COmifMOFw0CSiojx5T7CiaOoHvzzzz8XL168fv16XFyc2Wxes2ZNtR4vKCh466236qhu9QAny6Amga9+0eLb+vj1KbSZce3i2PrZZsV39dd+Sk75L1cVpJB7VTDRoJzHBaE2ejtL22Rd+wlcy5/NQuHEUVT3Nm7c+OKLL3IrH6ZPn/7VV1+9/vrrAJCXl7d8+fJbt24FBQXNmzdPqVQeO3YsLi6O+zQjI+PHH39cuHDh6tWrzWbze++9BwDz5s27cuWKTqfLysr677//PvroI5FItHHjxjt37ri5ub3yyiudOnUCgMuXL8fGxorF4j///NPf33/hwoV2PAcRgxA1CbzMF9Vc197YmMrYWLdgp1aTvAmScGrl4NlLnXVSc+v39M5zWxBUJRM7uc1lDA9vN8roivRn/wOCUAyc9OgjZOnEUdR4FecnaLJi6u11CnVLF++eZa/cvn27X79+3EoGm812+/ZtACguLg4ODn711VfnzJnz999/R0VFnTp1Kjg4+JVXXlEqlVOmTHn++ecHDBggl8tDQkIoiuLOo5DJZNzu0y+//PKrr77q6up64cKFzp07jxkz5vr165GRkbGxsd7e3tevX3/nnXfGjRs3Z86cbdu2RUZGxsTEVOUQ3bqAQYiahNr3i+bGFCVuyWAZ1quPc/NRnnA/8gKe8Si6qddnmlIP5PoNqeTUmApnjWqP/sVazNIOoUJP/0cfudcixF3WGrVze2ZqC+/U2+sIUjDytduUQFx6RaPRrF69euPGjQBgtVqLi4tpml6zZk2fPn24naJ79eoVFBQUExMTHBy8devWyMjII0eOkCS5cOFCAAgJCREIBFwQctq3b1+6U5qPj4/FYsnOzhaJRCEhIfv373/ppZcAQCqVrl69WiAQhIWFtWzZ8ujRowMGDKi3/wllYRCixs9oM53LjKlNv2jWKc2dPzOBhWYDXP2feSjtSBHZerL31VVJ6Yfz1W2VCr+KV8pzHt1ulDHp9af2AoAiYkKFj1A4a7QJaBs2Pzf1RL29TqluXTYFAcDNzW3x4sVDhw4FgLi4uAEDBlAUlZCQcOTIkeDgYO6ekpKSzMxMAOjYseNzzz33/fff37lz53FtuLJ7cO/cuXPevHktWrRQq9WJiYlZWVnc9TZt2nBn9pEk2a5du9u3b2MQIlRXTmdcMNOWGveLpkfnJe/JAQICRnp4h1dwRprCX+bdzyU9Ou/WlvQu77QghY/t3nl0jFB3bBdj1EkCu4r8gyp8BDeXaQqatR7RrPUIO1YgKCgoISGBC8Lr169ze4q6uLiMGzduxYoV5W5OTEzcvHlzeHj4l19++dNPPwEAQRDswzvocgnHmTt37h9//MGdyjt+/PjSO8tuJVNQUGDHMUKcNYoav5r3i7KQ9E928p4cgiRaTfCuMAU5vlFuci+JMdecvDfnCeXJH24RshaT7sQuAKhwdJDDzRrF5ROoTs2cOXPNmjVFRUVms/n777+fMWMGAIwfP37z5s3Xrl3j7rl69arBYDCbzZMmTfr4449379595MiR33//HQBcXFxMJlNpU68cq9VqMpkA4MqVK3v37i29fuXKlSNHjgDAmTNn4uPjw8PD6/prPg62CFEjV+N+UZZh7+zIzD5bSFBE4BQfl07KJ9xMCojWk5vFLr+TeaLAub3SsaW8wtvutQjvryPUndrL6IpF/m3ELTs+tmSZghAIGaOOtVoIoahaXwGhKgoPD3/xxRcDAwMJgpg0adLUqVMBICQkZPXq1aNHjwYAg8Hg6+u7f//+pUuXtm7d+rXXXgOAzZs3T5o0KTw83Nvb+7PPPuvTp49Go7l48aJUKi17+uA333wzduxYNzc3d3f3CRMmlH7Ut2/fb775ZtasWVqt9pdffqnPI+nLwSBEjVzN+kVZmr25OT0/tpgUkUEv+qraOFT6iNxb4jvQNeW/3MStGV3mtaQkFXS3lO0aZW1W3dE/AUA56LknlUsQpEJFF+bSJRqBs0fVvwJC1bJw4cIFCxYwDINglwQAACAASURBVFO2V3PChAkTJkzQ6/UikUgoFALA4sWLSz8NDg7m5pdyj3MTZwCg9NxBzpQpUyZPnqzX65XKh/41KZfL9+7dq9VqHRwcCLtupYtdo6iRq0G/KGNjb2xKy48tFkip9rP8q5KCnGaRrgo/qUljSdpdcR/R/SDUA4D+3H66uEDo3VwSFPzkYu9tLoMTR1EdI0mybAqWksvlXArWGEVR5VKwlEKhsG8KArYIUeNWg35R2swkbEgtuqUTOgjaz/KXe1e+TL4UQRKtJze7/O2d7LOF6vZKddvye/bfnzVqBIbRHeGag5MrPVYCJ46ixqdHjx52nB1TDrYIUWN2Ov28mba0r3K/qM1IX/spueiWTqQUdHileinIkbqJ/Ya6AcDtPzJserrcpxKBmCRIk82kjTlky88UuPtIO/autMz7E0exRYgaj1atWg0ZMsTetbgHgxA1ZkdSTwFAP98q9YtatLa4VUnaFINELer4RnOZZ7VTkOPd18WxpdxSYrvzZ2a5jwgguCN5c49sBwBlxISqnDJ4f+JoQc3qgxB6MgxC1GgZbabz9/pFQyu92aK1Xf3+rj7LJPeSdHyrucS5FvMzCWg1yZsSk3mXi/Nji8t9yK2g0BakUyo3abf+VSmPwhYhQnUJgxA1WtXqF03alWUqsCj8pB1eCxApajt2LlGLAkZ6AsDtnZkW7UOH93JBaKBAETmBoKr0IlxTj1CdwiBEjVbV+0VLkgx5l4tJIRk41UcgpXh5u0dPlSpIYdPTt7dllL0utTIAYFYo5N0HVbEonDWKUJ3CWaOocap6vyjLsHd2ZgILPhEuEjWfK9ZbTfS+9HWiJl6bc67QvYeKuygu1IAI2M69qr46HmeNNjJyuXz9+vVbtmyxd0UaEqvVOmJEXe1Ch0GIGieuX7SjW7tK+0WzTmn0mSaJs8h7gCu/dRApBS3GeN38Le3uriyn1g5ildCWFC/WloAzRbdsW/nz95EKFRAErS0ChgE7nVODePT888/X3d/p9mW1WhmGEYvFld9afXJ5xRs21R4GIWqc7veLVrJ80Kqzpf6XCwDNR3mSAv5X9bp2dSy4VpIfW5y4Nb397ADzsZ0yhgAAA9gqfbYUQQlImYLRl9D6Ym4GKWrQCIJQqRrnr2OdBmHdwX9dokaoxKw9m3GBJMjwygYIk/fm2Iy0U6CDul35xe98aTHWS6QQFCXq0/5JsN25KgMKKjqk/snuDRPixFGE6gAGIWqEDiYfszK27p5dXaTqJ9ymSzPmnC8kKKLFGM+6q4xQTrWc4A0AqcfNVnBVNmsDNQ1CHCZEqC5gEKJG6L+7hwEgqvkTD/lkgZsj06y/i9S12j05eelnDm4K/3dtF01WTKU3q9spnAOMLCPIF0xRB4YAgKGaQUjiOfUI1RkcI0SNTVJRyi3NHaVI0atZ9yfcln2uUJtqFDsJm0W6AgDNws4kpshSefmEpUB+Y7E4YwcACwDHd4ztOvA73zZjnvAIYzIo8lYXw0tmxluVXALcdqPVwU0cZbBFiFAdwCBEjc2/dw4BQIR/XxH12PUJNgOdsjcbAAJGeFAikmFh2jH619vMk0smgB1g3TbFvFTMFloJ8Z/C15wgf7Bl04V9r2sLbrUNm08QFXexlPy7EUqy1K43cgtCxTkykILeoq/Wl8KuUYTqDgYhalRolj6Uchwq6xdN2Zdj1dPK5nKXTo4MC9NP0L/eZhRCmNSCfNzMUbnpbpvkhc6mUwBQqAiN91taaGuxPYOlFe2eKfzwxvkVRXnXug/9USgqP+nGmpWsO7kHSFI9oF/udjNZIgBptccISQUuJUSormAQokblTMZFjbHQ39GnjXOrx91jyDJlnykkSKLFWE+WgNdO0b/cYmQC2D1I0M+zghykbaabF1bdjP+eoa0SuVv73h/6tR0PAEYbtN9p+592cnAXv2bXZ2UnHT62bVTYyI0yZbMHD7Ns0c4fgKEdwkdJAv0AbkExAe41mCzjDDhrFKG6gZNlUKPCTZMZ0iLysXewcOfPLJZhvfo4yzwlr5+mf0pgZALYM7jiFMxOOnRgY9+Es98ytM03aNzA549yKQgAUgGs7kUBwIKksKBR/yrUrYrz46N/H5yfcbb0ccOFQ+bbcZRCpYyaInYSEhTB6FgRI6x+EGKLEKG6gkGIGo8Ss/ZsxkWKoAb693vcPbkxRcV39EKFoNlg1zdP06vjGakAdg8S9H8kBU267Av/vXHq76mGkjRH13b9J/0TErVSJHloHfTgZsTYAFJrhYXxfv0m/uPm29ds1JzYOTElfjsAMCZD8Z4NAOA4YgYpdSBIQuREAQtOFpXeUqNZoxiECNUB7BpFjcf+pCNWxhbmHeIsrXjbDtrMJO/JBoCA4R4fXoVV8YyIhO0Rggivh1KQZZnka7/HHf/MatFSAmlQz7dbd5tNkBX/YVkZSh3KYHYkMS+0Ug4b8/vVY5/cvrzu4v43i3Kv+mo86RKNKKCdLPjegKVIJTAX2NQm5zuy29X6aqRERogkrMXEmo2EWFqtZxFCT4YtQtR4/Hc3GgCimkc87obU/bmWEpvSX7acUXx9lRGRsDNSMMznoRQsyo07smXYpUPvWi1az+YDB714IjDk9celIAB4yuCjLhQAvH6aNtBUp36Lu0Z+Q5LC25fXXbq5ghawqnGvlZ6+K1YLAcDN6mJlrFbaWq1vh72jCNURDELUSNwtSrldeFcpUoR5h1R4gyHHnHmiAAjYH+i+9CrXFqSe8X2QglZzyZWji6J/H1KYEyt18Oj5zLqwkZtkCu9KX/1mO7KLM5GiY7+8QgNAQIcpvUb9KiRERZKSG/45VsWDVRwitQAAXK1uAKCr9sRR7B1FqE5gEKJGYu+dAwAQGRAupIQV3pC0K4ul2bSWqvlpIiEJ2yKoEX4Pfv8X5ycc2Nj39uV1BEEGhrw+eNpp71bDqvhqAQk/96ZIAr6+wiQUsQDgkGNpm95cQkt11pwjW4eV7j4jUgkAwNnsDNXfXAa3G0WojmAQosbAxtCHko4DwJDH9IvmXykuvKGziqhXSGeKgE3h1KgyKWjS557+e6pJn+PsFRLx3IH2vT+gBNUbhwtxJaYHkhYGZp+kaZOheM8GiU3cu9dyN9++Jn3u8R1jU2/8CfeD0MnkBAA6a3XX1GPXKEJ1AoMQNQZnMs4XmYsDnPxaq1s8+iljZZL/yQGAFc4uOgG1qR81qcWD3/m0zXzmn5cM2gxnr5C+43YoXdrUrA5fhlBuUjiezV78/Rdujoxjz+G9x/zesssM2ma+sO+1K0cXiVUUEKAwKUggDNXeZQ23G0WoTmAQosZg393DADD0Mc3BtEN5Jo3llkTyr8rxl3Dq2RZlf9uzF/e/qcmKkTv6hY7YQD5+V7ZKqcXwdXcq0JTiHrcXSFI17lUgCIKgOvVb3HnAUoIU3L687urxN4UKmmQohUVZ7c1lsEWIUN3AIEQNXqGp+FxmDEVQkf7hj35qKrCkROezBKzwdFsXLpjS8qHf89dPf5V+a7dQrOw16lextJKz7Cv1fEtiVcFPApa+0GK40PtB27RFpxd7j94skjjmJh80qrYDgNrsXLPtRnHfbYR4h0GIGryDSUdsDB3qHayuaPng4U2ZJM3uc3R8daDD860e+g2fmrDjxrnvCVLQc9gahfqxW7JVnfHi4TaF1woEqpeEk87ksmU/cvPtGzz4ewAwUucBwNmsrvYBFDhrFKG6gUGIGrz7ywcr2GV7yyGtY5pOT5EBw9xntXnod3t+xvmYg+8AsJ36LXbzq6ApWV2MyVD8z/8AILb7tCLSYfZJ2vbwaRauPr1IUmi0JTCkXm1W66s5Web+5jI4RogQzzAIUcN2U3P7TlGyUqzo6RVc7qMNN2g4lAUAxSFuM7s9NPinL049u2c6Q1tadX25RacXealJyb8buTkyE0YPbK4grmrY1QkPJaFAKFe6tGdZm1UcrzKrq73dqIMjkCSjL2ZpGy8VRghxMAhRw8btsj04oH+55YObEpnjf+f5WCwmJ/H4sQ8N/lkt2tO7XjAb8j38B3To+xEv1Sg9a0k17lWpkFgRSgHAoot0puGhDlK1V08AsIhjnc3O1Z01CiRJOTgCyzK6Yl7qjBDiYBCiBszK2A4nn4BH+kUTithFR83P5WsAIORZL4J6sH0My9jO/jOjpOCG0jmw+9AfCYLioR6lZy31vjdH5hlfYrQ/WWKFuWcfahSqPLsDgFUcq7KoqruOEB70jhbwUGeE0H0YhKgBO5V+rthc0tzJr6WqeelFKwMvHKNnZORKGMals6NjS3nZR2KPfpibelwid+s1erNQrOSlGoaLh++dtTRkaunFFaGkgxD+uMv8m/agUajy6EZSIqvwtpRhrYbq7TUKuLkMQnUDgxA1YFy/6LAWg8peXBRD08n6/iVaUkQGjPAo+1FizM93r2ykBJLQERuqsoloVZTOkXEcMZ2UOpRe95ETH3elAOCtM7SJvneRpCRqjy5AMBbxVbK44q3gngAnjiJUFzAIUUNVaCo6n3lZQFKRAX1LL57MZpddpefk5AKAT6Sr2OlB2GQnR8edWAxAdBu0TO3Rla9qlM6RkQWXX84/px3ZSU3cLmGXxtKlF119egGARRwrKKn2IWgUThxFqA5gEKKG6r+70TRLh3p3dxI7cleKLTDlKD2yoCjAZJY4i7z7uZTeXFJw8/ze2SxLt+/1nk/gKL7qYM1K1p96sI9MuU9LN+P+8gpzo+heB+m9IJRckeirfawgt7kMrqlHiF8YhKihOpB0BB7eZfu103RJsXVmfgEANB/lSQruJZNJn3Pyr2etFq1f2wmB3d/krQYsW7TzB5a2lc6ReVQPN2Jaa9LCwOxTNJeEzp7BJCm2Ce7Kq7d6AuDBdqMYhAjxCYMQNUgJBbfuFqWoJI49vLpxV3YmMZtvM6/m5UtttFNrB3U7BXedtpnO7H7JqM108e7eNfIbHutQ4RyZR33dnXKVwLEsdusdBgBISqRUdQGCdbRms8A+4cFH4QEUCNUFDELUIHG7yQwM6C8gKQDI0LMvn6Rbm0yDC4sJimgxxvP+jWzMgbma7EtyR9+ew/9Xmz21y6GLC4p2rQUAx+EvlZ0j8yi1GL7sTgHA3LN0sZUAAHef3gAgJ1LMNnO1XkrirFGE6gAGIWp4rIwtOuUEAAwO6A8ADAvPH6MLTfBZYS7Bgne4s9RNzN157eTStJt/C0WKsJGbar+n9gMMrdn0JaMrlrTpJguJrPT2aa3Jfp5EjhEWxwkAwKNVHwAghPFaYzX33VbcP4mJrV5TEiH0BBiEqOE5mXa2xKwNVLdsqQoAgGXXmOhMdqKxxLvQKFQImkW6crelxP9x88JKkhT2HL5e6RzIYwWK9/1qvhNHKVSqZ+c9OkfmUQTAyjBKSML629TFfFbt1YVgxTZBqiYjtVrvJURiUiJjrRammgmKEHoCDELU8HCnDw5uPgAArmrYDy7ScoZ5NS8PAAKGewgkFADkpZ+5dGgeAHSOWOrm24fHt5sTY7WH/wCCUE9dwA3aVUV7FfFWO5JmYeV1hiSFDNMaCDb/ztnqvv3+5jI4TIgQbzAIUQOTb9RcyLosJAWR/n3NNEw9SptpWM5oCL1N4S9z6+YEAEZt5rk9Mxna2rrbKwHtn+Px7XRJoebXr4BhlEOfF7fuXK1nX25DAsCeVMbGgE3QAgB0uTHVrcC9zWVw4ihC/MEgRA3MgaQjDMv0atbDUax87wJ9VcP2Fdva3tUAAS1GeQIBDG09u/dls7HA3b9/+z4f8vluhtH8+hVdUihu1VkZMbG6T7dyJFopWI0ZTuWwZllzADBpY6tbCE4cRYh3GISogeHmiw5uPuBQBrviGiMgYWlJLmNjPXqqHXylAHDl2CJNVoxM4R0StYog+PwdXrxvkzkxllKo1FMXAFmTkod60wCwO5WxqjxJRm6jM4y67GqVQCqdATeXQYhXGISoIYnPv5lSnKaSOLV27jr9BM0CrHIzMHe0AinlN8QNANJu/Hn3ykZKIO45fL1Yqubx1TUbGixnmDcDALtSWEJFCi3tACA//XS1SqAUuLkMQjzDIEQNCTdNJqr5gNdPQ6qO7esKwXE5AOAb5SZ0EBTnJ8QcnAcAnfovUbl34vG9tRkaLKu7M+MuhTslbKGMFJo7A0BeWjWDELcbRYhvGISowbDQFm75oIkK33aXcRTBKqrQmGeReYg9w9Q2i+7c3pdpm9E3aCy/E2RqOTRYFknAUB8SAG4ZRYw1EAByUk9WrwQcI0SIbxiEqME4kXZWZ9EHOLX6+HIzAFjdkdGdzAeAFmO8CAouHpij1dx2dG3H7z5qwMfQYFkj/QgAuF4kLaIcSdbBUJJiKEmv+uM4axQh3mEQogaD6xdNNvcvscIYf7Lr1Vzacu/o3ZsXVmUk7hVJHEOHr6cE1T7V4Ql4GRosa5A3KRPAba1UIyoSmjoCQF51hgkpXEeIEN8wCFHDkG8oiMm+QhKC05owbzmx3Necf6WYFJIBwz3y0k5dP/01QZAhUT/IHf14fClfQ4NlSQUQ4UXaWGmBWCMydwKAvLRTVX+clCkISsAYdKzVwkt9EEIYhKhh+O9uNMMyeXR3GyjWhpH5uzOBBZ9IV1akOb/vNZaxtekxxyOg/NG4tcKyfA0NljPSj7Cx8kJxgcjcGQBy06ozTEgQpEIF3I6jCCE+YBCihuG/pCMAkGXrP7cD2TmtUJ9pkjiLPMMdz+192aTPcfPtE9TzbX7fWPIvn0ODZQ33JVmQasQagTWAAkejNlNfnFL1xyk8gwIhXmEQogYgJvtKWkm6hVV5Krp8GgQp+3MBIGCkx7WTn+RnnJcpvLsP/YkgKB7faE6MLTm8jcehwbLcpNDBWa4Ra4AlRNZq947i5jII8QuDEDUA313YAgA5zNDNA4TZ+3NsetqptYNBePjOlQ0kJeo5fB2/a+dp7f2hwSG8DQ2WM9TXQSvQWimrQN8RqjtMiOfUI8QrDEL0tItOvZJeEm8FxeyOz7QwmnPOFRIk4RZefOngPADo3H+Jyp3XrHowNNhJGcnn0GBZo/xlLAGFwkKRkRsmrFaLkOsaxSBEiB8YhOhp98W5LQAglIyc11Ge9HcWy7AevSWx59+wWQ2+bcYEdJjC7+tK/t1kvsUNDb7H79BgWe1UJAtSjaRAQPuKxK4mfY6u8G4Vn+V2WcPNZRDiCwYheqp9dv6y1RJvA8XaiGfyLxUV39ELlVQusVSrSXR0adt14P/x+7o6HRosRyyQFYg0wBIOsm5QnUYhHkmIEL/4CcKLFy9OmjRpyJAhq1evZln20RsOHDjw+eefz5o1KyEhgZc3oqbgTgm769Y2AOjlM6qFVJK8JxsAhJ33Zt7dKxQreV87Xw9Dg2WpxDKNWAMAMugKAHnpVQ1CyhFnjSLEJx6CMCsrKzIyMiwsbN68eStWrPjxxx8fveeLL77Iysratm1bRkZG7d+ImgIbA88fuuRAxJOk4pPQZ9IO5lpKbJTPjeSUFQBE8KDlcid/Pt/H0JqNX9AlhZLArsqBk/gs+TGcpbJCsQYAbFpu9+1TABX8I/JR97tGC+q0egg1HTwE4f/+97/w8PA333wzIiLiq6++Wr58+aP3HD169IcfflAoFLV/HWoiPo+l9frtADCl3RgoIDOOFTCUpkCymGVsbXq85dVyCJ8vY9nCbd+bb8dRjs7qqfOBIPgs/DEcRDKNuAAAjPnuUgcPsyG/pCCxKg+SSjUQBKMrhop6XxBC1cVDEF66dCksLIz7OSwsLDExUavV1r5Y1JRdzGdXX73kSMZLhYpJrYfc2pzGMFZDwJdmU66bb5+2ofP4fV3xrrX6c/sJkcT5pUWkgxO/hT+OXCjTiIoYYEU6q6t3GADkVW2LGYISkDIFS9sYfXEd1xGhJkFQ+yJycnJUqnvTCtRqNQBkZ2fXrPGXkpJy+PDhrl27cv9JUdSSJUtKU7ZRYlnWaDRWOLDauOn1euIxDS8DTUw+LPSidgLApNbPZOwr0GWYDO5r9ebLEgfPduH/p9cbeayJKfoP09E/CUogm/S2xbmZRafjsfByyn5rEQhp0lYkZNRWgpF0BfgzK+mYe8sJVSmHkDuCvkSbnUF58PBHuK494de6ETMYDFarlaL43Ofh6We1WhmGsVqt9q7IAzKZjKxs+jcPf4ocHByMxnt/MRkMBgCocRdos2bNunbt+vXXX5deadu2rVTK54SIpw3LsiRJyuVye1ekvrEs6+DgUOFHc07QBYYYL1G8o1g5zCHq1qlMk/yATvQnSYlCh69Xu/jyWA3dyX9M0X8ASaqnzJd26ctjyRUq+62dZI4AUCI3q4tkSZYwIUBh9gW5XEYQlffTGJ1c6Nw0sc0kecz/w6fKE36tGzGSJMVicdMMQrFYbO+KVA8PQejr65uUlMT9nJSUJJVKXV1da1YURVEqlapbt261rxVqoHalMOtvMl3EOwHg2dbjUrfnWURXS5yXAQtdIr5Ue3Th8V2GC4eLdq4GglBNeLMeUrAcuVAGAIzSCEWy+AzXUKWPoSStJD/B0bVdpc/enziKKygQ4gEPY4STJk3asWNHQUEBAKxZs2b8+PHcP4K2b99+6dKl2pePmo5cI8w6SauoywoywVGs7JzQVadJLXb5jGVtrbu94t9uMo/vMsad0Wz5DljWccQMec8oHkuuIplQBgCUox4AinMtqnvDhFU6m/D+xFEMQoR4wEMQ9u/ff/jw4W3btu3UqVN0dPTixYu568uXL4+Oji69R61WZ2Zmjh49Wq1Wx8fH1/69qJFhAV46bssxQkfZTgCY5jg192JWkdsimijyCIho3+cDHt9lvhWr2bQUGFoZNUXRfyyPJVedg1AGACZ5CQC4mS2ZsjAAyK3aasL7243iUkKEeMBD1yhBED/++ONHH31UWFjYpk2b0mHJI0eOlP68a9cumqZLH1EqlbV/L2pkfohn9qax/pLLrC3Bg/LwPO2Vp37PJkh2cm3XY9jPPB4uYUm5kb/+U9ZqcegzQhnF8w5tVce1CDWSQgDwtlgPWUIjAPLTz7AsXemXxXPqEeIRb1POPD09PT09y14RiUSlP2PyoSdLKGIXnKcBoIfyzywtTM9/qZD83iK5JJG7h43cJBDyNpnImpWc//Mi1myUhUQ6jXmFr2JrwEEkB4B8UR4AeFusf2T5j3Dy1xclF+VeV7l3fPKzeCQhQjzCvUaR/VkZeOEYbbDBZN/YLG18WEkYm7/P4LCHoiShI/4nVXjx9SJbfmb+jwsZg1baIUw9eW79LJx/HJlQCgCFUCSQUlKGYQw2xrkXVG2vNRwjRIhHGITI/j6KoS/ksc0VhAOzTWlVRmS56JTrAIhug5epPbry9Ra6KD9/9UK6pFDcuov6hYVA2nleOzdr1GA1SpxFAOBttsaRPaFqZxOSjtg1ihBvMAiRnZ3MZr+5yghIWNQhNrEg8dn03iWOXwDBtO+90CdwFF9vYXTFeT++b9PkiPzbuEz/iBAI+Sq5xuRCOQDoLHqJiwgAvKzWbdpQAMjPOMcwlaxHJiVyQiRmzUbWzOfeAgg1TRiEyJ6KLTD1GE2z8EFn8lza1r65nSSS1Sxp9Gk1PjDkDb7ewpgM+T9/aMtJFXr6u7z8OSF+KrZokAulAGCwGrgWYUvGclnvJnZsYbPoinLjKn38Xu8oThxFqNYwCJE9vX6aTtay3VyISPfLmjRNV8sJhspzdOoWPIS3gwZZq6Vg7ceWtESBi5fLK0tJ2dOyxYlEIKEIykxbhGoBAAQLbACQ68CtJqxC7yhOHEWIJxiEyG7+TGZ+u83IBfB7f2pL3LbxBRqb6IZI4N1n0kaSElX+fBWwtK1gw2LznTjKycXl1aV1fdZudXHDhKQTAwD+NisAnLCFQtWCECeOIsQXDEJkH5kGmHmCBoBlPakCXUzH63cJ0TmSlfcZ+6tY6szPOxim8LdvTPEXSAdHl1e+EKjd+SmWP9zEUavCCgAyvUVCwS5DTwAiP+M8Q1ue/CxOHEWILxiEyA4YFmafF2nMMMKPnNmGPHNwpQt5mmAFncNWO3kF8fMOli3cvtJw+RgpkbvOXiJ053Orbr5wSwktMhMpIKxaW5Q7WwgutEMgbTNqsmOf/Cx2jSLEFwxCZAf/F8cczSHdpbC2N3U0ZpOfJhoAvN3eDeg5iJ8XMEzRztX6M/sIkdhl1mfCZi35KZZv3OYyOptBrBYBC6OdbABwSxQGVVhNeL9rFIMQodrCIET17b909v0LNAGwoa9AbknNP74YCJvEMjpkEj/TRFmzMX/9Z7qT/xACofNLH4kCKj/MwV4c7i0lvDdxNExsIwk4YK7SMCGF240ixBMMQlSvbhWzk6NtNAsL29si3XWHtownQCcyhfScvIQU8LDPC11ckLdqvun6WVKmcJn9haTNU32k170WoeVeEEp1lhBX4jLRAwhSk3WRtpme8Cw38Qe7RhGqPQxCVH9KrDD6IF1kgTH+5LtBxjO7ptuMqUJLCzLgdWdfHuZzWjOTcpfN4VZKuM1ZJm7ZofZl1qn7m8sYpC4iADAVWEb6kTrCqUTShraZNdlPOsWMxFmjCPEEgxDVE4aFZ6Nt8UVsJzWxqR916+ySvPQTJO1sNr82YAwPh+KaEi7mff8OXZQnCmjnNmeZwK1Z7cusa1wQ6u53jZryLSN8CQC4AJWvJqQcnIAkaV0RMPQTbkMIVQqDENWT+efpvWmssxj+HEilXVqRFv87wYgdNIsEUU5SkaSWheuO/Z2/9iPGZJB3H+j6+lekgyMvda5rXBAaS4OwwNpORbRyJKoShECSlNwRWJbWFtVLZRFqtDAIUX349TbzbRwjJGFHpMB6Y9X1U18SQDpqFpzwSR/SdUCtimaYop2ri/76CVhWGTVFNfltguLtcLG6JnvQIhQCASaNhWXY4b5EPNWdBUqTfYm2PWkrUVxBgRAvD+gydQAAIABJREFUMAhRnYvJZ2edpAFgZRjllfnTtZNfEEAqNO8kiV069m8tEYhrXDJrNuav/1R3YjchEKqnvKuMmmLfk5Wqy0F0b4yQFJIipZClWXORdYQvaSCUGcJ2DG0tyLzwhMfvraDAiaMI1Q4GIapbWQYYeZA22uCNdmR/3U9xxz8jgFJo3jHbuh8JPDy81eAal0wXF+SunGe6fo6UK11eXSrrVruWpT3cP4DCAAClvaO9PQgXCcQQXO/o6Sc8jhNHEeIFBiGqQyYaRh20ZejZPh7EK4Kf404sJoBSFLwDlj6/tNwwpvMwcU33FLWk3sz99nVr+h2Bq7fbnGXi5u35rXn94A6g0Fv1ACBxFgKAqcBCETDUh7wmqHw14f2uUWwRIlQrGISoDs04QZ/PY/0VxAqXn+NP3ktBsWnAr81/dfJQjGw1tGbFGq+eyls1nzti1+3tFQJXb36rXW+4MUK91QgA0nstQgsAjPAlEqjuDAg1ObE2i+5xj3PbjeLmMgjVEgYhqitfXmE232YchLDR86e7Zz4ngFJq5kmMkVsDtuU6Zb/b7TVBjc6I1x37u2DD56zFLO852GXWYlL6tByrVAPcXqMGqwEAuON5TfkWABjcjARKfpvqyDK2/Mzzj3ucwskyCPEBgxDVif3p7IcXaZKATZ4/5cUsIQjKsehdiT7yoN+B607X3unxqpfco9qFMnTR9lVFf/0EAMqoKapJcxvQBNEKcadP6O51jT5oEToIYYAXcY2qZBEFzhpFiBcYhIh/N4vZidE2moUfXH5irn9BEJRTyQKxNuKa/5VjzseGNI8Y6N+vumUyBl3ejx/oTu0hhCL11AXKqCl1UPH6JhfcmzUKpUGYf+/0pRF+5DVBT3hiEOKsUYR4gUGIeFZohuEH6GILfCJb7nr3C4Kg1Mb3RUX9C/3ytzn/4a3wfDP45eqWac24k7tirjkxllKqXd/8P1nXfnVQcTsQUkIhKbQxtJm2CB0ElIS0mWibgQaA4b7ELSrYCqKi3GtmY8VtPpw1ihAvMAgRn2gWnjtqSyxm36SWtc/5jiAoV9v7gry+hA+zXL2SJMkPw97h+gOriqFLDm7J/e4tW06a0CvA7e0VIp/WdVZ9OyhdSggPNwq9ZERHV1m8oCfL0rkpRyt8lhBJCLGUtVoY42Mn1CCEKoVBiPj0zjl6Xxo7jVnWt2gZQVDu5IdEZl+xu2Cl9yobaX258wttXaoRY7aCrLxV80v2bmQZWh421G3OMsrJte4qbxf3D6DQw/2Jo8aCe72jI/3Iy1Q/AMhOjn7c4xRuvY1QrWEQIt787xaz4hrzrPW7YfplBEF5yT9mk3uLHIW72v2VZc3u7tl1QtDIqpbFsvrT/+Z89Yr57nVK7eb66lLVhDeJWm9J+hRyEHITR43w8HwZABjhR1wSDACAnOQjLFvxzto4cRSh2mvYk+7Q0+N0DvvqKXqi5bsx5uUEQfm6fmaK6U5JyJQBt06knlFJHBeGvkVAlfY/ozW5mi3fmRNjAUDauY9qwlukrAGvkXiyhyaOujwUhO1VhNSpRbbR38OYXJhzRe3R9dHHMQgRqj1sESIeZBrY8YfpUcbvxpuXEwTV0m+JKaY7QREOYwQ/p28ggJjf8021tEonDhpjT+T836vmxFjSwcl5+kfOL37QiFMQyi0lfHjiKACM8CUuC/oBQHZSxb2jJJ5Tj1CtYRCi2io0wzP76b6F/8elYJugL0tOdgMC/Ma5fZW1zMrYJgaNCvMOqbQcWltYsP7Tgl+WMAadtFMfj4VrpB3C6qH+9lV6AAU80jUKACNKhwmTDlf4ODdxFDeXQag2sGsU1UquEQbvs7bL+mq05QeSFLbvtixvdxtg2YDhHr/Qm9JKMlqrW8zoPLXScoyxJwq3r2T0JaTUwXH4S/KwGu6+1uCUHlIPAGKVkKAIc7GVsbGkgACA3u5EhizMYpQU5V41G/LEsvJzhbhd1rBrFKHawBYhqrlsIwzdUxSZMWu05QeCFHYK+75gX1vGxnqEqa/5XT2QdFQikCzqNU9IPunfW4xRV/jH9wW/LGH0JZI23dwX/NR0UhBKD6m3GACAIAmxkxBYMGvuNQoFJAzylV0ThLEsk5185NHHcXMZhGoPgxDVUIqOHf1X4pTMkT1t+yiRsvvA9QX/BdmMtLqdQjqIWHFxDQDMDZntq3zSjtimGzE5X87Sn/6XEImdRs92mfU55eRSX9/gqVC2RQj358sYyw4T+hFPWERBOToDLp9AqHawaxTVxM1i9o2/Dr1S9IacLZY5tQiNWp/0O2UuNCn8pC2f83rr2EK91dDPt1dU88efEWi1FP35o+7EbmBZUUBb9XPzBC5e9fgNnhayeycx3Q/CR4YJo5qRb4sjwPxRdvJRlrERDzevsWsUodrDIETVdr2Q+WrHypd1XxPAuPoP6h6xIvG3Qn2GXuombjfTf238pvj8W14OHvN7vvG4EkzxF7R/fM8U5RFCkeOwFx3CRzesk+V5dH8d4b0glD4ycdRBCO2b+WbeaO5lvluQdcnFu3vZx0m5kqAEjFHH2qyEQFiPFUeo8cAgRNVzMUv7959vjrT8xwLRotvrbdrPS/g5TZ9lEikF7V72u1x89Y8bf1ME9WGvt7lOv3IsabeKd683J14BAJFPa9WUeUJ333r/Ek+RsrNG4ZGlhJwRvuS52wO8LHdzkqPLBSEQBKlwoovymZJCSu1WT5VGqHHBIETVcDTxTty+aV3oRCupCB26Ui3qc2VFklVrk7qK2s7wM8mNS48uZ1h2Zufn2rm0KfesrSC7ZO8vhsvHgGVJmUIcPsZ54ASo0ZGEjUnZvUahoq5RABjhR/xM9XsG1mUmHW7X671yJVAKNV2UT2s1GIQI1QwGIaqqAzEHc0687sWWFEuCxk38nyVVFbcumbEy/9/encdHVd77A/8+Z5l9spOZrASCAcIii5RFpBgQBStYRa3rVa9t/dV6rfdel171drG1rYJ9WatWa6+tWpW6ASLKjoqIEHZlDUkgyWSyzGT2MzNneX5/nGQIWYBMJpnD5Pt++fJ1cnJm5jkZks885zzP98kYbRlzRxFrZH699RmX0DYpd/wt5dd1fqAS8vs3vRv4fCUVo4TlzJdenbbw9qBEMQWhe48wFoQUYnV48k3EaJ8ZrjFDy7dCwGm0nLGUIw4cRaifMAjR+aArN/xZ+ub3JlCc2Yt/eNMfnVsDp9bXAQX7zKzS6/IIS/51eOX2hp3p+rQnZv83Q9pHI1NZCm5b4/v0TUUIACHGSZelX/PvXLYdAMDvT+YJaYblzFGjrJ7hLZwYkKI+UZd++p7fouGGg3Wzpkkbmmo3l4y/pfMzsOlYdxuhfsEgROcgRQMfvv9TxrmOAltd9PP/uvanVe84WvZ4CENGXGvPvywbAKo9J/+6/00C5OEZ9+cYswAAKBX2b/N+9DfJ5QQAfdnkjMX38IWlyT0XDeq8+oTKkKMTA5LQGu0chEuGk//ecfk0aUNj9yBUB45ilTWE4oVBiM7G33bik/fuYgPH/SSzsfyF/5px2cE/1wROCayBGXN7UeZYKwC0hlyPf/5UVI5eN/p7swunA0Dk2F7PqlfFhhMAwNuHp19zt2Hc9CSfiVaZeRMBIkgCBaoWJTdm6/y1obArml5qjh02IYu0ps+DMDhrP1cUkWFOZySDy/Mi1D8YhKhXjdXrv1x7Pyv6atly8ZK//cdw+/7nTkTaREO2rvye4SabHgBO+er/e/MvmoIto7NG/b/Jd4rOk97Vfwsf2gkAbMawtAU3m2dcBQzWbegVQxgDpxekcEgU1HG2PY6XAYA5IwvqWsqKxGMux65hhaersLYvQOHFIEQoThiEqEf06K4Xvvnyd4QqX3KLS7777L+BvP/5aiWqpI0wjb2rmLdwAHDEdfyRLb/2RLzlOWW/nXxf8P2/BHd8CorCmCzWeTda5lxLeF2yT+QCYOZNghQOiaH2IOxpBgUALB7OvLa7oih6zFmzuXsQKn4MQoTihEGIuooI7spP73fWbpaB+6fh8SUVP17Q4DrycRNQsM/ILL0+n7AEAHY79z/++VMhUZiZO/FBf57w9H9QSSS8zjJ3sXX+D1J77aTEMvEmENxBUVArandfjEk1x04eM14O0b+crNo04bLHY/tZK44aRahfMAhRZ7T2m7cPfvGbaLjNR7L/ZHrhfypmX7zTUVvpIQwZfnVuYUX76gfra7b+YcdzkiLPVXLu/nSvKFcCIaZLKtIX3Ymz2fqqx6mEQrceIUtg1IjvCPut4DkS8jeYrO1FXJm0TCBE8XuA0iFboAeh/sAgRO28rYf3bnrE5dgFAPu5Of9nevrl2QVFn55qrg6yemb0bUVZ46zqke8eWvXivr8pFK5sUm6vczAMZ5w2z1pxA59XkswTuGB1TCVsHziqs3KsjpGCsiTInPGMqZbXlOh2fXPpdOlTZ83mkRPbF7ciHM8YLUrIrwR9jCV9kBuPUArAIEQgS8LRXS8c3fUnRRZ9TO4/dD//xnL9uxdL1nerfe6oPoMf++/DLQUGAJADnlc3LXsrcIBQuLleWhywmiuutFy2eKgtGZFYHeVGhfavCeizdCFnOOwWLQVnBOFVhczfdJdPlz49VX06CAGATctUQn7Z34ZBiFAcMAiHusbq9fu2PBby1QNhNvC3vKH/n0sL03fkCS3v1IXDirXEVH5XMW/lpOZ677bVz9d/uiUbOAo/caVdOXOpeeZCotMn+wwueOoCFJ2nEhpzdCFnONwaVT9/xFh5MBfMo8eJ69Tnihxl2PaxSExaFjhPyV4XdsoRigMG4dAlBJz7tz7RcHwNADj4cc/rnqrjJz85gVzX0OJ4wwUUci/JGHVDvlizv/Wdld4jO/80ktuXTfSU+d/SG2ffejPejkqULgtQwOkZFJHuB88vzTtZPbZEPtTasCO3eI66Ux0vo+CceoTigkE4FFFFOrH/tW+//IMkBmUu7U3uwY/5O8dlcdtyAvIap8MnEYYUL8jMsu5rXvZrqelUkIPlo/VHzTSNM/9+3i+6F9RG/aHeIwzGLo3GZlB0GzgKAEuGk4e4ihL5UH315tNBmI4DRxGKHwbhkNPasHPvpkd8riMAUGWa/wzzVBtj/3mReGN1vW9bAADMdsaWtx+2vNcWDgKAz5b7h1JSK7bZzbnPVPzq7CvOoziY29fm7VRlrb1HKHY/ON9EhKzLofHPtVWbpsz9pbqTaV+eF3uECMUDg3AIiYa9h3csO7Hv/yhVZHPJH5knd8B3R5voh3o3u7HVJ1GWl7MM282O1YqDAoC+dIJrxmW/qF/ZHGodkV78TMUvh5lwUEzidfQIu18a7aFHCACTR08LODMs/qqgp9acUQKxOfXYI0QoLhiEQwQ9eei9g5//KiK4CMsfzLz3qfCDIugfNoWuO+4QvbIC1AJ7MiOr2UiQMacZL55tmbXouEF5dOuvvRHfpNzxT819vMeFdlH/WXRm6BKEWTxhSMQjUpmq5Qs6WzKc+wc7e5a0prF2y6hJdwEAq5YbxeIyCMUFgzD1eVq+3bf55+oEQcid85j4myORkunUu6z5KLj0IoCO1mfTDw18o37sZPO0+YYJMwnLbW/Y+atNz4SlyOzC6f87+yE9i8XSBoqp/dLo6SAkLNGlc5E2MewWjcO6/uQnZpE6SwV41hw9tkkNQgbLjSLUDxiEqayl/qtjlS84azYDUL3Ztsv2i983Xznft/eP3m+sgREU9AwVMsiGnFEBy3duME68lOiN6gPXVW9++uvnJUVeOHLeQzN+yhJcQXcAdR81CgDGHF2kTQy7ot2DEABKRl1OK5lQ45eyFGY5Q0e5UbxHiFA8MAhTEKWKo+qTY5Uvup17AIDjTcbSO16tXzB1/9e7vL8JKQslchEFmmY5UTybSZt1X+dZ2G1hz6v73/y4agMF+m8TfnD3xFt6fx2UGOb2JQnPCEJDtg6OB3u7TXhVqa1y7/hS+UBL/XZ7SQVjtBBep4RDNBomOkOPD0EI9QaDMKUoslh3dOXRXc/73ccBQG/Iys2d3+LIy92472n5uIu51gfzgYApPVK6ND993ITOjxUV6f2jH71+cEVQDPEMd/8lP1xy0cIkncfQYm4fLBPsvLO30tuqy+zkDd3lpcKBo0c320sqAIC1ZkruJtnXxuXkDXyTEUopGIQpQoz6T377zrHKF4WAEwCMuuxccUROlcQdPZoHJ7ykooGpoMBxBlJ8lT1vdhZhzhiCsdu5/0+Vr9R66wBgqv3i+y/54Yj04uScydDTfR4hxKYSunsOQp4Ba1EFHHvOWbMR4Deg3iZ0N8k+NwYhQn2FQXjBCwebqw+8XrXnFTHqBwCzZLZ5snOCGQTECGQd5a/WwzhO5oCAfXrm8KvtvPmMG351voYX9vztq4ZKAChKK7hvyr/PLLgkOWcyVBl5A0OYsBSWqRy7HXv2HiEAzB4zxXs8J104GWirtmSOVAeO4qqECMUBg/ACFnBXHfvyuVPVK2VFAgBrxJTntWWG01q5zA3WeVSZfFHYYJYpAFiHG0dem28dbjzj4dHgP799790jq0RFsujMt45besOYJTyD/yQGGwFi4o2BaDAkClZd+zqOp5fnpQA9FbNbVMz9Lzf7UnHlieObLv7OSFynHqG44V+9C4/kcjorV9Qcf7dFrKZACZBMIS0/UNiUNuOlzKkhZcLlPigLRgCAMJA51po/Jzuj7IxlchVKN9RseWnva21hL0PIghGX/2TK3ZkGXLggacy8KRANBsVQLAg5A8uZWSkoRwOSztrD76mFh2huBTSsPHps48Xf+SGjLs+LA0cR6jsMwgsAlUSxvipaezhQvafRua2ZqQvoQwDAUiZXKcnMu/Y9/vI3XGXz3KFrXJ40OQIAvJWzTcvMm52lz+C7PNu+poPP7361qq0GACbZJtw/9Z5RmSMG/6RQZ+ZuxWUAwJCtCwSFsCvaYxACwLiyy5UGFlp3SGJILTeKxWUQigMGoUbJXle09nC05lDk5BGh4YhL1+Iye/36ADVSAOCJcXjBApj44LOOUUcO+L/X5n3VX89QCgCWQqN9ZmbutEyG63pBrTnU+td9b6yv2QIAuaacey6+7cqRFYN/aqi79iCMnjFw1JitC5wSwq3RtJKea/osvij79c8vLpP3NJzcZsNyowjFC4NQMxRZbK6PVn8bqflWrDsuOk/JRHabfW6Tx2v3U0IBgCFcrn168cQfuIctemYPH1jtucF94oeRKAAQjmSPSy+Ym20d3sMfzbAUefvQB28dej8qRw2c/gdjr7t13PU6LBajGT0PHD1rxVEAKDATZ1pFWduevd9uWjTlHsAFKBCKCwZh8iiK1OqINlQLNYdC9VXRumNUjAKAQhSvIdCWG3AZ2xSQAIAQNjtvSmHZNZml161vyfzrHsF+2HWnz29QFADgM3UFs7Ps0zM5Uw/1X/zRwOaTX7x+cEWr4CZAFoy4/MeT/y3HmDXI54rOztLjVMLeF2OKKRxZAbuXBes3snMfBhw1ilBcMAgHjxL0iQ3VYmON6KgVHTWis1ZNvvbvEgjk69qswZZotayEAYAQJjtvWmHZNRkjr13vzFq1O8h+4p/mPf5DUQQASsB0kaVkTnZWubX7qEJRFnc4KtfXbP2qoVJURAAozym7f+qPynPKBu+E0Xkzd6u7DR09QqH3HiEAXDFu4u49wzKiDn+kGQiRA15QFGCYAW0tQikGg3CgUFmSnCdFR63YWCM6akRHTffLVlyWjdiLPFbBTeqaWyslUYAwEMIMK5xZULY4reR7G6utW/b4Lav8kwMt1yuK+ijJwNqmZJTMyTLm6ru+KNCDzYfW12zdeupLfzQAAAxhLsmbdM2oK79bPIv0OAwfaUBvg2UAQGiKhF1Rdbu7SdnsCsPcS4R3Kw9vKTenKwGPHPCoUykQQucJgzAxaDQstTik1gapxSE6asTGWqm5nspS52OI3sjnj+DzSog9z8/7PZE6V9PuNucWJaiuv0qy86cVli22Fl/92TeWnZ8HbCvaRoSdRerzEwhlG/MnWIZfbLUUGbvUhQGAk966zSe3ra/Z4gg41T0l6UVzi2cvLJ1nN+cO+Pmj/lEXoAideY9Qn85bCo2BemHfH0+Mub0oY7Sl58cWVsDxd+urN01ItykBj+JrwyBEqE8wCPusI/McUkuD1KpuOGSvq+txDMPlFvL5I/j8kXxeCc3O8oZqnI4drfVrPXu+oVRWjyKEzbBNLh692GBbtPObtKOb/MVOT77iygcAgCjLCPmmUVPSSi62dp8IAQAuoW3LyW1bT2072HJY3TPMlDOnaObC0nkXZY4csJ8BSrAey40CgQn3jTj2dr3rgO/bv54cvii3sGJY98fOGD+37Thn8u6i1psAQPa5eSgdlFYjlCIwCM9GCQdlV1NH2jX0mnkAhNdx2XncsAJuWD5nK+LzR/D2kqjoczv3NDp2NR/8u6f5G0rbr20ShsvMnZid/53MYdOCkanffgOuNYLd67OBzwYAAK1mHSm1XjwtbfgYU/d1WQEgIke/ati1rnrz1449MpUBwKIzzyr4zlUjK6bYJ+Il0AuOmVfvEQpd9rN6Zuy/FZ9a33xqfXPtmqZgY+SiG/MZ/oxbgHOLM57lJ48SdzWzgSxcnhehvsMgBCpGZU+L7GmVPS2Su1n2tsptLXJbs+xpUcKh7scTjudy1Mwr4HLyuZx8blg+mzEMCAEAwe9odXzdeuxvrVt2+FzHYo9iOX2mbXK2fQbPTmpqGdtWB6HDYV8oSmibHQAAIgxTl2WyjrHMmpE2O7+HG0IylWs8J79tPbqv6ZvtDTvDUgQAeIafVfidBSPmzsy/hGd76DKiC4JZXZv3zHmE7QgUX5lrzjcce6u+ZbdHaIqMvbu48+UBnoFITgU07joZbcjCGRQI9d1QCUIaEWSfW/Z7ZG+r7GmV25rltpb2/Ou9KhXRG7nMXG6YmnbtscdmtmceAIgRn7f1iK/uU+/eQ77Woz7X4WjYG3s4x5syc6dazFMjwkSP8yLhoOLZEQUABnzZAAAQJeSkSe/NNuSNT5s/wzLP2nWwn1toO+Q6dqj16LctR464q8JSuL1hQCbmjlswYu7c4ktjRbnQhaujR9jDBy9V9oS0ix8oPfR/JwP1wr5nT4y9syhtpDn23TGj50HjH0LiMSAlCs6pR6iPUiQIaTQi+92Kr00OeGWfS/F7FL9H9rnlgEfxtcl+N41GenssYTk2YxibkcNm5aobXMYwNsvGpucwpjMyRpYifvcx7+HPfK4j3tbDPtdRwe/o8mw6faY1baKOTIp4xstNI5hqqv5ts0IYAMIMc8Kgb0nXE5uxqNQ4dbThu1kkLITM5vY/aqIiHXdXH2o9esh19NuWI85gc+cnL7Tml+eUleeMnlUwzYZDYFJIj6NGuzDZ9ZMeLD36Rl3bkcDBl2pHfj8vb1b7oJirxo//1+e2TKUpxAtG7BEi1EcXcBBG6457V70ie92yz00jXW+udEF0etaaxaRlsmnZbEYO15F5bGYua82M9fDOeP6wN9TybdBT63Md8bYe8bUeDnhqY4NcVCxrNOpLWVKqhEsgWMx7i/hoJgAoADwAD1RgmON6vSPNwOQZ80oME0YZbs9lTJ1+6pRSV7htv+fbg82HD7YcOuY+EZFPzxszcoZRmSPKskZNzC2/OHc81sVOVeqo0bMHIQBwRrb8nuG1Hzc1bGk98Z4jWC+UXp9PWJLGkyZrRab3bY/Rn4E9QoT66EIOwupvIlUH1W3C6xhrBpuWxVoymLQs1prJWNLZ9GzGksFaM9i0LKI39vI0NBxsDvrqBF9DyF8f8jWE/PVBb13IXy9FA10OJYTTcSOIOJIVhuuE4VykhJHtBM64null2Rq9vtpkkHP12SXGCaWGJbmkwEwAQJTFxmDjwWZnY6DJEXA6Ak2NgabGgLPznz+GkBHpxWNzysbljBmXM3p4ejHTU0ijFGNRJ9RHzxGEAEAYMuIau6XAcHyFw7mjLdQUGXNnsc7K2UbOg71ve4w+rLuNUgGlihBUwkGq/j8cUsJBRQjRcFARgvrSCYbyaQl8tcQEoaIoR48eNZvNxcW9Lmve0NDg8XjGjBnDsj1UAouD5bIl+tKJhNcx6dmMoeeqxB3NEyPBpojgjoRahYAz5DsV6hR7itxz5Q4CJka2sWIeHy3mxBGcOJwVh5OOn5hISCPPOYw6B8+3GXmSoTPm6HPs/PAsdnY2udHgaQ41OQLOxkDTG06nGnsuwaVQ2v2FrDrLuJwx5Tmjx+WMHptTpl4lQ0PK+Vwa7WzYlAxjrv7wa6d8NaF9z54ov6t4/qTLdu7lA/pQ1NMykC1FqA+oJNJoRBH8NBqhYoSGQ0pEoNEIjQhKOESjYSpGFCFIw0FFCLUHnvplTwMVY4RvvrJrLQibmpoWLFggy7LH47nsssvefPPNLlFHKb333ntXrlyZl5cnCML69euHDx/e/9cFhuELSxVZjAquSGttJNQaCbVGBJcaeGF/S0RwR0KuaMQlRr1nexolgxVtjJLLyrmsaGMlG6PYWNHG0PYbhB6OO8VzDhPv4HVBM89lgTErkpbhyzb40nn/eMYvyX5P2OuJ+Fr9vupW31uBpkhP4coSNt8yLM9iy7PY8i32fIs9z2LLM9s4mY3dI0RDk47V8QwnKmJUjp5nMXRLoXHSg6VH/lHnPRE88OfqUTcW1BunDRe2t7GtheEgY8B/USgeihAACjQSoopMxaj6H0hRJSyAIishP6UKDYfUhKPRMJUlKgSpIitCEGRJiQhKNAxilEZCSjgEHfWw4sAYLcRgYgwmxmgmBhNjMBOjmTGYGaNZP3pyAk8ZEhKETz31VHl5+dtvvx0MBqdOnfrhhx8uXbq08wFbtmz5+OOPDx8+nJWVdd999z3xxBOvv/56/1+3/vCG3Rv+Q5I95z6UMgzNYKQ0RkmGNAL5AAAawElEQVRnlBxWtrGSjZVzWSmXkexBRu9miY+nXk726GW/WQrrhKghoOhbZUNANvlYzsMSn6L4w6LPF/GJiggegLO+bJreGou6fIstz2LPs9hs5mEs6dobppSGQufbD0ApzMSbvBFfUBTOf1UQ3sKNv7ekemVj45fuY2/VF5ROA9juMfoVXxsG4YWLRgQqywAAiqR0jH6gQqh9IrIkUrF96J8aWgCghPwAQBWFRkIA0J5e6lMpcseRVL3eCAA0HARKlWgYJIlKUSpGQFHO3gmLD2E5ojcwBjPR6QlvIEYzozcQXq8mHOH1RKdnjBaiNzJGMzGYmfbAMzHGQR0Mn4AgXLFixdtvvw0AZrP5lltuWbFiRZcgVPdkZWUBwI9+9KOZM2e+9tpr/b9AumvHKUXyAmEZOYNR0oicwcqZREljaAYrpck0PcJYRDAGwRxh9AITEbhohA2FdCGB9wmcJ6ir87OeIBeUGanX16AA3WZ2GThDuj4tw5CWoU9P11vT9Wlpemu6Pi3TkKFu55py8PIm6iszb/JGfCEx1KchUYQlpdfnmwuM1R84bKfGu+zgNQQkn4vLLRy4pp5F7E/wGTuFoNK9LgRVlHBP8ya7PZb2dEOh49uU9vIklFIq9PKtaJhKYtedYpRKXVuuCEE489XVPlCnI7qeBY2GqSQBgCzLDMOAJELH01JFpuHTw/oUIQBAOx4V6d6kpGAMZmAI0RkIy1PODLwBWJ7wOkavB4ZjDCaZ6hmdiXAc4XWE0wPHAWukwBKDCQijsJwMEgAvEV0YJCCKHFXkqJ9SJSwrUsQLMqVUVpSgFKFSIEppAMClRH2SLCtKAOSooghUCVMaoVSgiggQBJAVxQ9EBhIEKlES1ukuv+a+txJ41v0Nwkgk0tzcPHJkezWvkSNHrlmzpssxJ0+eXLRoUewAQRCam5vz8vK6P5soii0tLRs3blS/ZBhm+vTpvV02PFSSHW67L0JIRB8OM5EIGw6zkTAXCrPuCBtWoPdfnk6sOoue1ek5vYU3GzmDjtOZOKOZN+k5vYHVW3RmA2fI0KepyZeuT0vXp+EyfmggqB+eAuK5s6E7+4xMk11/5O+sR7aJbNPHHzwEtP//Snv+DaKUS8STDzhCCCgJ/jzKUKbL4Li+P0WnDoC50yeebn/kYj99AgxDz7zZRACoQplzdOAYyhLa8fmDyJTpYWg9oYSATEm40z51W6BMBEgUFAAFQAQapJQ5/Y+TMiJ0ehQlIQrxXwU9t9hPnQAAhAIJ7rz2NwiDwSCl1GAwqF+aTKZAoOtgy2AwGDvAaDQCQPdjVI2NjYcPH37qqadie37+85/PmDGjx4OvG1P+PnfCAIQhJo5hDawBAMy8iRCiY3Q6licd85QNnJ4jHEMYE29kCWvkDCbOqGf1+r5GmgyRUCQCvU5JjAOlVBAEpR9X0i9Qvf0bSG1nOWv1H3Cr15XP2+J4ZpINo35oq399ErDrgD8YfxPP+UID99Sal6zfUvnchyTsUQlAeQAeABSqB2ApUEr1QBgAkKmREgqUKMSggKwAo4AOAETCKkAk0EmMIoNOJCASohA2wihRwlKiFzigQEIMK7KSDFxx4bhFfv95NsdkMp3zAmR/gzAzM1On07ndbpvNBgAul8tut3c5xmaztbW1z21yu90A0P0YVXFx8Zw5c957773zeemxVuvjef8Vf9O1gVLKskN0sIzVak12E5Kgt7NOM1gAQOFo/D8WK2y5OMtcc+no9Ikmrn8/W0Io0/MfBwISKOEev9UbWZYTNVb8/FFKiZLgD1uUAXreHwQURWEYEt8nB4YAd7oPRGn3XihhCXuO/q7Cnu5IsayRcKeP1zHAqt9iAHiOYXuaXcYaoKNqo5klQAjDn75vRxgd4fQAwDEwzGhhdWZCGACQJElRFJ2u5z6GjuH1XNfF42I4hjNyhrOf1ADpbxASQiZPnrx9+/axY8cCwPbt26dMmdLlmClTpmzfvl3d3r59e1lZ2dD8C4jQWaiXRkPnPYOiR7WhJl9G9KHr7s82ZiaoXQng9/uH4K98KBTS6/WD/wkguURRVBRFr+817bQpAYNlHnjggYceeigvL8/hcHzwwQe7d+8GAI/HM3HixM8//7ykpOTuu+9+5plnnnvuufLy8kceeeSBBx7o/4silGJM7VMJz1Ej6SyCYsgX8Rs4fZYxI3HtQij1JSAIb775ZkmSXnrpJbPZvHbt2lGjRgEAz/Pf+9731Ct+drt906ZNy5cvX79+/X/+53/+6Ec/6v+LIpRiLP3uEdb7HQCQb7HjOlwI9UliKsvcfvvtt99+e+c9ZrP5xRdfjH05ZcqUf/7znwl5LYRSkqmPxWW6c/idAFBg7WE8NkLoLPo3FBghlCB9rbLWXUPACQAFFgxChPoGgxAhTTDr+t8jbASAfGvPQ7IRQr3BIERIEzp6hPFMqFc1+BsBe4QI9R0GIUKa0LFIffyjRtVLo9gjRKivMAgR0gSzujZvNM4eYVSOugQXS1ibeVhC24VQ6sMgREgT+jlq1BFoUii1W3K7r3CCEDo7DEKENMHCmwEgFO+lUUcAbxAiFCcMQoQ0IdYjpOe3cEoXDTiJEKF4YRAipAkcw+pZnUzliBTP8iZqjxBHyiAUBwxChLTCrIt/4Gg9zp1AKF4YhAhpRX+mEqr11bBHiFAcMAgR0oq4q6wpVGkKNhMg+RYMQoT6DIMQIa1oD8Jon4OwKdgiKlK2KUvP9rwgKkLoLDAIEdKKuHuEjoATAArxBiFCccEgREgr4g7C9pUI8QYhQnHBIERIK+IuLtO+EiH2CBGKCwYhQlph0cU5ahTLbSPUHxiECGmF2iOMo8qauhIhlpVBKD4YhAhpRdz3CBuDTQCAcycQig8GIUJaEV8QuoW2kCik6axWnWVg2oVQisMgREgr4ptHqN4gxOuiCMUNgxAhrehYpL6PQejHctsI9QsGIUJa0b5IfR9HjeJKhAj1EwYhQloR3+oTDVhuG6H+wSBESCviW31C7REW4j1ChOKFQYiQVhg5I0OIIIYV2odF6tWVCPPx0ihC8cIgREgrGEIMnIECFaTzvToaFEO+iN/A6bOMGQPaNoRSGAYhQhrS14Gj7UNGLXYCZACbhVBKwyBESEPaB45Gz/c2YQMWV0Oo3zAIEdKQjh7h+V4abS+3jcXVEOoHDEKENKSvA0ex3DZC/YdBiJCGmHV9KzfaXl8Nh4wi1A8YhAhpSF/rbmN9NYT6D4MQIQ3pUxBG5ahLcLGEzTUNG+B2IZTKMAgR0hATbwSA0PkFoSPQpFBqt+RyDDvA7UIolWEQIqQh6qjRwPmtxITlthFKCAxChDREvTR6nj1Ctdw2DhlFqJ8wCBHSkD7dI1R7hDhSBqF+wiBESEP6FIRquW28NIpQP2EQIqQhfZpH6MCVCBFKBAxChDTk/HuEClWags0ESJ7ZNvDtQiiVYRAipCHnv/pEU7BFVKRsU5aB0w98uxBKZRiECGnI+a8+4WgvrobXRRHqLwxChDTEwBlYwkbkqKhIZz8SF2BCKFEwCBHSFvU2oXCulZgcWG4boQTBIERIW9Qqa4FzrcRUj+W2EUoQDEKEtOU8i8s4cBIhQgmCQYiQtph157VIfWOwCbBHiFAiYBAipC3tA0fPemnULbSFRCFNZ7XqLIPVLoRSFgYhQtpyPlMJ1YXpsTuIUEJgECKkLe3FZc66EpMD504glDgYhAhpy/lUWWvAuRMIJQ4GIULacj6jRhtw7gRCiYNBiJC2mHgTAATOGoQda9NjECKUABiECGmLOmr0XD1CdW36/EFqE0IpDYMQIW3pmEfYaxAGxZA34jNw+ixjxiC2C6GUhUGIkLacc9Ro+w1Ci50AGbxmIZS6MAgR0pZzjhrFdScQSiwMQoS05dxBqM6mx5EyCCUIBiFC2mLWnSMIcTY9QomFQYiQtpg5NQh7rTWKs+kRSiwMQoS0hWd5nuElRY7I0R4PcOBseoQSCoMQIc2x6HotLhOVo62CiyVsrmnYoLcLodSEQYiQ5rQXl4n2cHW0MdCkUGq35HIMO+jtQig1YRAipDkd5UZ7WJu3IYAL0yOUYBiECGmOub3caA89QrW4Gt4gRCiBMAgR0pyzLECB5bYRSjgMQoQ0p6PcaE+XRrHcNkKJhkGIkOaoC1D0OJUQVyJEKOEwCBHSHDPf8wIUClWags0ESJ7Zlox2IZSaMAgR0pzeRo02h1pFRco2ZRk4fTLahVBqwiBESHNMvBF6mkfYvu4EjpRBKKEwCBHSHAtvhp5GjXbcIMRJhAglEgYhQprTXlmmWxA62sttY48QoUTCIERIc9RRo731CHEBJoQSi0vIs5w4ceKll15yuVxLliy59tprux9w8ODBysrK5ubmm266qaSkJCEvilCq6phH2EsQYn01hBIqAT1Ct9s9a9YsjuMqKiruu+++t956q/sx119//erVq5988smqqqr+vyJCqa23Reobg02AkwgRSrQE9Aj/8Y9/TJw48fe//z0AsCz79NNP33LLLV2OOXbsGAAUFRX1/+UQSnk9BqFbaAuJQprOatVZktQuhFJTAnqEX3311dy5c9XtuXPn7t+/PxjsdXFthNA5mXkTARISQxRobKe6MD12BxFKuPPqEUqS5Pf7u+9PT09nGMbpdM6bN0/dM2zYMABobGwcNWpUHK1xOBw7duy4/vrrY3sefPDBKVOmxPFUFwpKaSgUIoQkuyGDTRAElh1yK+qd/1nrOX1YCrt9bUbOoO6pdZ0EALspNxTqoRi3lg3N9zoUCsmyPNROXBRFRVFkWU52Q04zGAwMc44u33kF4ddff93jEJht27aNHj1ar9dHo1F1TyQSAQCj0djHprbLysoqKiq66aab2hvHcWPHjjUYDPE92wWBUqooSmqfY49EUcSzPgsLbwpLYZmRY8c3R1wAUJRWcMH93Ibme60oil6vH2pByLKseuLJbshp50xBOM8gvPTSS1taWnr7bmFhYV1dnbpdV1fH87zNFmchRIPBUFBQcOONN8b38AsRpZRhmPN5q1IMnvXZmXgTCO6QHI4d3xhoAoCCtLwL7uc2lN/roXbi6vlecGedgOZed911H3zwgXq55s0331y8eDHHcQCwdetWdYwMQqivLLquSxI2BByAs+kRGgAJCMJFixaNGzfukksuWbRo0WuvvfarX/1K3f/YY4+tXr1a3V6yZElpaanT6bzjjjtKS0uPHj3a/9dFKIWZui1S37E2PU4iRCjBEjB9gmXZlStX7tmzp62tbcaMGRZL+9juf/3rX2azWd1+8cUX1duHqsLCwv6/LkIprMsCFEEx5I34DJw+25iZ1HYhlIISU1mGEDJ16tQuOwsKCnrcRgidkxqEsQUo1JoyeRY7gSE3wBihgXaB3dJEaIjo6BG23yNUy20X4nVRhAYABiFCWtSxSH37pdF6dQEmHCmD0ADAIERIi7osQOHAdScQGjAYhAhpUZdRo+311bBHiNAAwCBESIssOnWR+vZLo9gjRGjgYBAipEUm3ggdPUJRFlsFF0vYXNOwZLcLoRSEQYiQFll4tUcYAgBHwKlQajMP45ihVbgSocGBQYiQFrXfI4yGoOMGYaE1P8ltQihFYRAipEWdR42qs+lxJUKEBggGIUJa1DGPUL002ghYbhuhAYNBiJAWGXkDQ5iwFJGpjOW2ERpQGIQIaREBYuQMFGhIFNT6ajh3AqEBgkGIkEapUwn90YAz0ESA5JnjXO8aIXR2GIQIaZQ6cLTGc0pUpGxTloHTJ7tFCKUmDEKENMrCmwDgmPsE4EgZhAYSBiFCGmXqFIQ4UgahgYNBiJBGqT3C423VgD1ChAYSBiFCGmXWmQGgJdQK2CNEaCBhECKkUSbOGNsutGAQIjRQMAgR0ij1HqEK66shNHAwCBHSKIuuPQitOotVZ0luYxBKYRiECGlUrEeINWUQGlAYhAhplCUWhDhkFKGBhEGIkEZ16hHiSoQIDSAMQoQ0Sl2kHnCkDEIDDIMQIY0y8e3TJ/DSKEIDCoMQIY1SV58AnE2P0ADjkt0AhFDPLDoLz3B6Tp9tzEx2WxBKZRiECGkUz3DL5z3JMzwBkuy2IJTKMAgR0q6Lc8cluwkIpT68R4gQQmhIwyBECCE0pGEQJlltbe0777yT7FYMtmAw+MILLyS7FUmwbNkySmmyWzHY/vrXv7rd7mS3YrCtWrXq0KFDyW7FYPv66683b96c7Fb0GQZhkh04cODDDz9MdisGm9PpfOWVV5LdiiR4+umnI5FIslsx2N54442amppkt2Kwffzxx5WVlcluxWD74osvNm3alOxW9BkGIUIIoSENgxAhhNCQhkGIEEJoSCOaunX/3HPPPf7447m5ucluyOARBMHn89lstmQ3ZFBJkuR0OgsLC5PdkMF28uTJ4uJiQobWBHmHw5GTk6PT6ZLdkEHV0tJiNBotlqG1orLX65VlOSsrK9kNOe2WW2558sknz36MtoJQUZTjx4/zPJ/shgweSqkoikPtbwQARCIRvV6f7FYMNjzroUMURZZlGWZoXXWTZZlSynEaKtWSl5dnNBrPfoy2ghAhhBAaZEPr0wpCCCHUBQYhQgihIQ2DECGE0JCGQYgQQmhI09DYnqHG6XSuXr069mVFRcWoUaOS2J4BVVVVtWfPHo/Hc9ttt5lMptj+HTt2vP/++1ar9a677ioqKkpiCwdCS0vL7t27T506NWfOnDFjxqg7a2pqNmzYEDvm6quvLigoSFIDB8Thw4c/+eSThoaGkpKSO+64Iz09Xd0fiUReffXVEydOTJky5dZbb02xOSStra0fffTRoUOH0tPTly5dGnu7V69e7XQ61e2srKylS5cmr42J53A4Vq1aVVVVZTKZ5s2bN3fu3Ni31q1bt27dOrvdfs8992hqNkWPsEeYNFVVVY899lh1h0AgkOwWDZTq6uoZM2b85S9/+fGPf+zxeGL7N23adNVVV9ntdrfbPW3atJaWliQ2ciBceeWVv/zlLx977LEvvvgitnPv3r2//e1vY++7IAhJbOFAWLhwYXV1dXFx8YYNGyZPntzW1qbuv+GGGz788MOysrJly5Y9/PDDyW1kwv3sZz9bu3at3W53uVxTp0797LPP1P3PPPPM+vXr1fe6vr4+uY1MuMrKyoMHDxYXF3Mcd8MNN/z5z39W9//973+/++67R4wYceDAgdmzZ0ej0eS289woSpIvvviivLw82a0YDOrUIq/XCwANDQ2x/VdcccXy5cvV7WuuueZ3v/tdcto3YNQTv+yyy1555ZXYzvfff3/u3LnJa9SAEwRB3ZBledSoUW+99Ral9MCBAxaLxe/3U0qPHTtmMpna2tqS2cpEi501pfT++++/9dZb1e3Zs2evWrUqSY0aVC+++OKsWbMopYqilJWVffDBB+r2xIkT33777WS37hywR5hMPp9v+fLlL7/8cm1tbbLbMoB6nFNMKf38888XLFigfnnFFVfEPkSnjN4mUzc1NS1btuzVV19tbGwc5CYNAoPBENsOh8NqaZXPPvts5syZ6vZFF11ks9l27dqVtCYOgB7PWvXJJ58sX7587dq1NHUnbYui+PXXX0+YMAEAGhsbjx07Nn/+fAAghMyfP1/7v9oYhEmj1+unT5/e1tb22WefTZgwYd26dclu0aByu92RSGTYsGHql7m5uSmZCt1ZLJZJkyZ5PJ61a9eOHTt2586dyW7RQPnDH/6QnZ191VVXAYDT6Yy91wCQm5vrcDiS17QBVFlZ+c477/zsZz9TvywvLzcYDM3Nzffff/8111yjKEpym5dwVVVVpaWlGRkZR44cWb58OQA0NjYajUar1aoeYLPZtP9e42CZpJk2bdp7772nbi9btuyRRx658sork9ukwaQW0pMkSf1SkqQhUmduwYIFsX7www8//MQTT6TkZ6B33333+eef37Jli/pGcxwny3Lsu6laVvD48ePf//73X3755dhgmZdfflndePTRR8vKytavX69+MkgZI0aMqKysrK+vf+SRR+6999433niD53lJkiil6ngoURS1X2APe4SaMGvWrOrq6mS3YlClpaVZLJaGhgb1y4aGhry8vOQ2afCl6vv+3nvvPfDAAxs3bhw9erS6p6CgIPZeU0odDkd+fn7yGjggqqur58+f/+tf//rmm2/u/t3MzMzy8vLUW6CYZdnMzMwJEyb89re/XbFiBaU0Pz9fFMXY2LcL4lcbgzBpOg8X/Oijj8aPH5/ExiTF4sWL3333XQCQZfmDDz5YvHhxsls0GDq/72vWrEm9933lypU/+clP1qxZU15eHtu5cOHC3bt319XVAcC2bdtkWZ4xY0by2ph4p06duuKKKx599NG77rortlMUxdg1j7q6un379o0bNy5JDRwQoVAotl1ZWVlUVEQIycnJmTVrlnq5KxQKffzxx9r/1cai20lz7733Hjx4cOTIkSdOnKitrV21atW0adOS3aiBcumll4ZCoX379k2cOJHn+Z07dzIMc+jQoblz51ZUVDgcjkgksnXr1nMWib+wPPLII5s2bTp69Gh2dnZOTs6zzz47Z86cpUuXulyuwsLCQ4cOud3udevWlZWVJbulCaMoitFozM7OjnX4fvrTn955550A8Oijj65YseLyyy9fu3btb37zm3vuuSeZDU20JUuWbNy4cezYseqXkyZNevXVV6urq2fPnj1z5kyO4zZs2HDzzTe/8MILyW1nYt12222nTp0qKSlxOBx79+598803Fy5cCABbt25dunTp1VdffeDAgeLi4pUrV2p82igGYdIEAoGvv/66ubk5Nzd3xowZZrM52S0aQHv37u08TGDKlCnqL4bb7d64cWNaWlpFRUXq3TSqrq6OzaIDAHVMgcfj2blzp8vlysvLmzlzpvZvn/QJpXTPnj2d9xQUFNjtdnV7165dVVVVkydPjt1CSxnHjx/3+XyxLy0Wi3pZ+PDhw4cPH1ZnEaTSJx5VOBzeuXNnQ0NDdnb29OnTY8UTAKChoeGLL76w2Wzf/e53tb8WFQYhQgihIU3rQY0QQggNKAxChBBCQxoGIUIIoSENgxAhhNCQhkGIEEJoSMMgRAghNKRhECKEEBrSMAgRQggNaRiECCGEhjQMQoQQQkMaBiFCCKEh7f8DczHS7XmfYrkAAAAASUVORK5CYII=", + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/html": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot((Φ^m)[nx ÷ 2, :], label=\"Φ^m\", linewidth=2)\n", + "plot!(Φ_m[nx ÷ 2, :], label=\"euler\", linewidth=2)\n", + "plot!(Φ_sdirk[nx ÷ 2, :], label=\"SDIRK2\", linewidth=2)\n", + "plot!(Φ_θ[nx ÷ 2, :], label=\"θSDIRK2\", linewidth=2)\n", + "plot!(Φ_θ_extrap[nx ÷ 2, :], label=\"θextrap\", linewidth=2)\n", + "# plot!(Φ_both[nx ÷ 2, :], label=\"θboth\", linewidth=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "32×32 Matrix{Float64}:\n", + " 0.0651948 0.00451346 0.00572298 … -0.021391 0.0172369\n", + " 0.0172369 0.0651948 0.00451346 -0.0390968 -0.021391\n", + " -0.021391 0.0172369 0.0651948 -0.0375144 -0.0390968\n", + " -0.0390968 -0.021391 0.0172369 -0.0227642 -0.0375144\n", + " -0.0375144 -0.0390968 -0.021391 -0.00132928 -0.0227642\n", + " -0.0227642 -0.0375144 -0.0390968 … 0.0215937 -0.00132928\n", + " -0.00132928 -0.0227642 -0.0375144 0.0425005 0.0215937\n", + " 0.0215937 -0.00132928 -0.0227642 0.0594241 0.0425005\n", + " 0.0425005 0.0215937 -0.00132928 0.07158 0.0594241\n", + " 0.0594241 0.0425005 0.0215937 0.0789898 0.07158\n", + " ⋮ ⋱ ⋮ \n", + " 0.0246593 0.0294486 0.0348455 0.0168616 0.0204731\n", + " 0.0204731 0.0246593 0.0294486 0.0137817 0.0168616\n", + " 0.0168616 0.0204731 0.0246593 … 0.0111819 0.0137817\n", + " 0.0137817 0.0168616 0.0204731 0.00900795 0.0111819\n", + " 0.0111819 0.0137817 0.0168616 0.00720551 0.00900795\n", + " 0.00900795 0.0111819 0.0137817 0.00572298 0.00720551\n", + " 0.00720551 0.00900795 0.0111819 0.00451346 0.00572298\n", + " 0.00572298 0.00720551 0.00900795 … 0.0651948 0.00451346\n", + " 0.00451346 0.00572298 0.00720551 0.0172369 0.0651948" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "Φ_both" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.9.0", + "language": "julia", + "name": "julia-1.9" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.9.0" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/julia/ex-01.jl b/examples/julia/ex-01.jl index 808fb227..a34fcc16 100644 --- a/examples/julia/ex-01.jl +++ b/examples/julia/ex-01.jl @@ -48,9 +48,9 @@ called by the solver to give access to solution values. =# function my_access(app, status, u) - XBraid.status_GetWrapperTest(status) && return - t = XBraid.status_GetT(status) - ti = XBraid.status_GetTIndex(status) + XBraid.Status.getWrapperTest(status) && throw("Test error") + t = XBraid.Status.getT(status) + ti = XBraid.Status.getTIndex(status) print("t: $(t[]),\tu: $(u[])\n") end @@ -79,10 +79,10 @@ tstart = 0.0 tstop = tstart + ntime / 2.0; core = XBraid.Init(comm, tstart, tstop, ntime, my_step!, my_init, my_access) -XBraid.SetPrintLevel(core, 2) -XBraid.SetMaxLevels(core, 2) -XBraid.SetAbsTol(core, 1.e-6) -XBraid.SetCFactor(core, -1, 2) +XBraid.setPrintLevel(core, 2) +XBraid.setMaxLevels(core, 2) +XBraid.setAbsTol(core, 1.e-6) +XBraid.setCFactor(core, -1, 2) XBraid.Drive(core) From 35172ad1f6bba15d5cb3aa6a23218fbc8b693256 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 31 May 2023 11:28:23 -0600 Subject: [PATCH 05/12] renaming status.jl to Status.jl --- braid/braid.jl/{status.jl => Status.jl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename braid/braid.jl/{status.jl => Status.jl} (100%) diff --git a/braid/braid.jl/status.jl b/braid/braid.jl/Status.jl similarity index 100% rename from braid/braid.jl/status.jl rename to braid/braid.jl/Status.jl From 83130d533050988a752b35fb001d044f0a157d46 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 31 May 2023 17:21:48 -0600 Subject: [PATCH 06/12] minor changes to build_function_sundials.jl --- .../adaptive_theta/build_function_sundials.jl | 2 +- .../adaptive_theta/drive-adaptive-theta.jl | 10 +- drivers/julia/matrix_stencils.ipynb | 312 ------------------ 3 files changed, 7 insertions(+), 317 deletions(-) delete mode 100644 drivers/julia/matrix_stencils.ipynb diff --git a/drivers/julia/adaptive_theta/build_function_sundials.jl b/drivers/julia/adaptive_theta/build_function_sundials.jl index a2529a3f..869fdc89 100644 --- a/drivers/julia/adaptive_theta/build_function_sundials.jl +++ b/drivers/julia/adaptive_theta/build_function_sundials.jl @@ -71,7 +71,7 @@ function Symbolics._build_function(target::SunTarget, ex::AbstractArray, args... fname = :diffeqf, lhsname = :du, rhsnames = [Symbol("RHS$i") for i in 1:length(args)]) - @info "Building function for Sundials" + @info "Building function _$fname for target $target" fname = Symbol('_' * string(fname)) ex = hcat([row for row in eachrow(ex)]...) varnumbercache = Symbolics.buildvarnumbercache(args...) diff --git a/drivers/julia/adaptive_theta/drive-adaptive-theta.jl b/drivers/julia/adaptive_theta/drive-adaptive-theta.jl index 7ab529a5..2e39d52a 100644 --- a/drivers/julia/adaptive_theta/drive-adaptive-theta.jl +++ b/drivers/julia/adaptive_theta/drive-adaptive-theta.jl @@ -1,4 +1,4 @@ -using ToeplitzMatrices, OffsetArrays, LinearAlgebra, IterativeSolvers +using ToeplitzMatrices, LinearAlgebra, IterativeSolvers using Plots using MPI @@ -235,12 +235,12 @@ function test(;ν=1., α=1., order=1) plot!(u.u, label="u1") end -function main(;tstop=2π, ntime=512, nx=512, ν=1., α=1., useTheta=false, useRich=false, cf=2, ml=2, maxiter=10, order=1, skip=false, refine=false, maxrefine=2) +function main(;tstop=2π, ntime=512, nx=512, ν=1., α=1., useTheta=false, useRich=false, cf=4, ml=2, maxiter=10, order=1, skip=false, refine=false, maxrefine=2, finefcf=true, tol=1e-3) MPI.Init() comm = MPI.COMM_WORLD Δx = 2π / nx - app = AdvDifApp(nx, Δx, ν, α, useTheta, useRich, order, cf, ml; skip=skip) + app = AdvDifApp(nx, Δx, ν, α, useTheta, useRich, order, cf, ml; skip=skip, spatial_tol=tol) core = XBraid.Init(comm, 0., tstop, ntime, my_step!, my_init, my_access; app=app, sync=my_sync!, sum=my_sum!, spatialnorm=my_norm) @@ -255,7 +255,9 @@ function main(;tstop=2π, ntime=512, nx=512, ν=1., α=1., useTheta=false, useRi XBraid.setSkip(core, skip) XBraid.setRefine(core, refine) XBraid.setMaxRefinements(core, maxrefine) - XBraid.setNRelax(core, 0, 0) + if !finefcf + XBraid.setNRelax(core, 0, 0) + end XBraid.Drive(core) diff --git a/drivers/julia/matrix_stencils.ipynb b/drivers/julia/matrix_stencils.ipynb deleted file mode 100644 index 018bb250..00000000 --- a/drivers/julia/matrix_stencils.ipynb +++ /dev/null @@ -1,312 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "using ToeplitzMatrices, LinearAlgebra, SparseArrays\n", - "using Plots" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "4×4 SparseMatrixCSC{Float64, Int64} with 10 stored entries:\n", - " -5.09348 0.000259382 ⋅ ⋅ \n", - " 5.09322 -5.09348 0.000259382 ⋅ \n", - " ⋅ 5.09322 -5.09348 0.000259382\n", - " ⋅ ⋅ 5.09322 -5.09348" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - " α = 1.\n", - " ν = 1e-5\n", - " nx = 32\n", - " Δx = 2π/nx\n", - " I = SymmetricToeplitz([1, zeros(nx-1)...])\n", - " Δ = (1/Δx^2)Circulant([-2, 1, zeros(nx-3)..., 1])\n", - " # ∇ = (1/Δx)Circulant([0, -1/2, zeros(nx-3)..., 1/2])\n", - " ∇ = (1/Δx)Circulant([1, -1, zeros(nx-2)...])\n", - " A = ν*Δ - α*∇;\n", - " A = sparse(A)\n", - " display(A[1:4, 1:4])" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "m = 4\n", - "Δt = Δx\n", - "# forward euler\n", - "# Φ = I + Δt*A;\n", - "# backward euler\n", - "Φ = inv(I - Δt*A)\n", - "Φ_m = inv(I - m*Δt*A);" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "# SDIRK2\n", - "a = √2/2\n", - "K1 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A))\n", - "K2 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A*(I + (2a-1)*K1)))\n", - "Φ_sdirk = I + 1/2*(K1 + K2);" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "# θSDIRK2\n", - "function θSDIRK2(m)\n", - " a = √2√(m*(m-1))/2m\n", - " θ = (m^2 -3m - √2√(m*(m-1)))/(2m^2 - 4m)\n", - " K1 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A))\n", - " K2 = m*Δt*((I - (1-a) * m*Δt * A) \\ (A*(I + (2a-1)*K1)))\n", - " Φ_θ = I + (θ)K1 + (1-θ)K2;\n", - "end\n", - "Φ_θ = θSDIRK2(m);" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "# θ extrapolation\n", - "p = 1\n", - "θ = 2^p * (m^p - 1) / (m^p * (2^p - 1))\n", - "Φ_θ_extrap = θ*inv(I - m/2*Δt*A)^2 + (1-θ)*inv(I - m*Δt*A);" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "# both?\n", - "p = 2\n", - "θ = 2^p * (m^p - 1) / (m^p * (2^p - 1))\n", - "Φ_both = θ*θSDIRK2(m÷2)^2 + (1-θ)*θSDIRK2(m);" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAGQCAIAAAD9V4nPAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdd2BN5/8H8M855+6be5N7sxNZVsQeCRIjSBDU3i1tFaWbVqm2ulR1/FqUamt8lVZRtKVUrdhbCCFBkL2Tm3H3OOf8/jhERMg6yZXk8/orPfec5zy3xtuzCZZlASGEEGqqSHtXACGEELInDEKEEEJNGgYhQgihJg2DECGEUJOGQYgQQqhJwyBECCHUpGEQIoQQatIwCBFCCDVpGIQIIYSaNAxChBBCTdrTFYQJCQlr1661dy3qm81ms3cV7MBqtdq7CnaA37rpsNlsTXADS4ZhaJq2dy2q7ekKwvj4+P3799u7FvWKZVmz2WzvWtiByWSydxXsAL9102GxWBiGsXct6htN0w3xX/ZPVxAihBBC9QyDECGEUJOGQYgQQqhJwyBECCHUpGEQIoQQatIwCBFCCDVpGIQIIYSaNAxChBBCTRoGIUI1tPDo4uf3vGamLfauCEKoVjAIEaqJ81mXTmdcSClOO5tx0d51QQjVisDeFUCoQdoU9wf3Q3TKiXDfMPtWBjUsZ86cWbZsmb1rUSdYlmVZliTrpIk1YMCA2bNn10XJGIQIVdu5zJi4vHilSKGz6s5mXjTZTBKBxN6VQg3G5cuXi4uLZ8yYYe+KNCQXLlzYv38/BiFCT4tf4rYAwJT240+knY3Liz+VcSHCr4+9K4UakpYtW44fP97etWhIhEJhYmJiHRWOY4QIVc+ZjIvx+becxI4jWkUN8OsDAEdSTti7UgihmsMgRKh6NsZtBYDn2o+TCiT9fMNIgjybEaO3GuxdL4RQDWEQIlQNpzMuJBTcUktVI1pGAYBaqurs3t7KWE+mnbV31RBCNYRBiFA1cM3BZ9uOlQjE3BWudzQ65aQ9q4UQqgUMQoSq6lT6+RsFiWqpanjLwaUX+/qECkjqYvblErPWjnVDCNUYBiFCVbXp2jYAmNJuXGlzEAAcxcqu7p1sDH0iHXtHEWqQMAgRqpKT6eduFCQ6S1XPlGkOcgb49QaAaJw7ihq+w4cP//bbbxV+lJ+f//777+/Zs6eeq1QPMAgRqhwLLNccfK7deDElKvdpH59QISm8lH210FRsj9ohxAODwfDzzz+///77H3300XfffZebm1v2U41GM3DgwJSUlBkzZuzcudNelawjGIQIVe5k2tmbBbedpapnWg569FMHkTzEswvDMsfTTtd/3RCqvbt377Zv337Pnj3u7u7Ozs7x8fFBQUEnT96bAlZcXDxkyJCRI0du3rz5yJEj77333u7du7mPsrKyduzYkZiY+Omnny5fvlyv1+fn53/zzTdLly7Nycmx3xeqHtxZBqFKsMD+ErcVAKa2n/Boc5AzwL/36Yzz0SknR7YaUr+1Q41BgRliC1iWrafXBTqBj5woe2XRokXt27ffvXv3p59+GhcXt27dOjc3t9mzZ1+7dg0A1q9f/8Ybb0yZMgUAgoKCjh079tFHH/Xo0cPd3f3WrVuvvfZa165dhw0b9tdffx09etRkMg0ZMuT69euDBw++fPkyQRAVV+JpgkGIUCWOp565XZjkIlUPbTHwcff08u4hpkRXc6/lGwpcZM71WT3UCET+a4stqK8YBHAQQt4UoYR6cOXixYsLFy4se8/o0aOXLl2q1WoVCsXbb79d9iMvL69169aV/qdOp9uyZYuTk9PgwYNbt2595syZnj170jTt4uJy9+7dFi1a1PG34QEGIUJPUjo6+HyHiY9rDgKATCjt4dXteNqZY2mnxwYOr8cKosZgUnPSRcLU2+taKgkx9dAVZ2fncoOCubm5EolEIql8N3k/Pz8nJycA8PDwAIBOnToBAEVRrq6u+fn5GIQINXjHUk/fLkxyk7kMbR755DsH+PU5nnYmOuUEBiGqrgWdyAWd7DljY8KECV988cWQIfc69rOysj744IPnn39eKBRW+ixF3QtVrhe07H/WX29v7WAQIvRYLLCb4rYBwJT244VUJX8jhHqHSAWS63k3c/S57nK3eqkgQvx46623GIYZMGAATdM2my0oKGjatGlLly61d73qCc4aReixjqaculOU7C53rbQ5CAASgTjUO4QF9mgqzh1FDQxBEG+//XZOTs60adPCwsLy8vKWLVtWlX7RxgGDEKGKMey90cGp7SdU2hzk4Mp61KCRJLlkyZKdO3dWpUeU06tXr1OnTnE/y+VyjUYjEt0bSr9w4UJwcHCdVJRvGIQIVexI6sm7RSnucteo5hFVfKSnV7BcKLtRkJihzarTuiFURyQSiVwur/r9AoFAqVRyPxMEoVKpSj9SKpUCQcMYfcMgRKgCDMv+du0PAHi+/QQhWdU/zEJK2KtZDwA4knqqDiuHEOIVBiFCFTiScuJuUYq73G1wlZuDHDyzHqEGB4MQofIYltl4bRsAvNBhYtWbg5wQzy5KkeJ2YVJKcVrd1A4hO5s2bdqaNWvsXQs+YRAiVN7hlOMpxWnucrdBAf2r+6yApHr79ACAo9g7ihops9lstVrtXQs+YRAi9BCGZTbF/QEA0zpMqm5zkMP1jh5KPs5zzRCqS+np6d98882CBQv27dvHXUlOTt64cWPpDatWrSooKCj3VH5+/vLly999990dO3ZwV7Ra7bfffpuamvrJJ5+sXbu2fipfSw1jSg9C9eZQ8vHUknQvB49BzavdHOR09eiokjimlqQnFaUEOPnxWz3UKNHFBbac1Hp7HaVyE7h6l70SHx8fFRX1yiuvtG/fftGiRQkJCW+//XZiYuKqVateeOEF7p4lS5ZEREQ4Oz/YSjczM7NXr15Tpkzp0qXL6tWrz58///XXX5eUlCxcuHDPnj3jxo1zc2sYO0tgECL0AMMyv97fWZQiqErvrxBFUH19wnYl7otOOTkdgxBVQf5PH1izkuvtdYRQ5PXFDkL4YO/cjz/+eO7cuXPnzgWAXr16de/evdxG2xX68ssvx48fv3jxYgAYNmyYt7f3J598AgBWq3X58uXcpqMNAgYhQg8cTDqaWpLh5eAxKKBfbcoZ4Nd7V+K+Q8nHpnd6jqeqocZM3jPKeP1c/byLIEmhV0DZFASA2NjYq1evbt68mfvPgoICjUZTaVGxsbHp6enR0dHcf1osluTkZEdHR6FQ2KFDB95rXncwCBG6h2GZ365vB4AXO0yqcXOQ09GtvYtUnanLvqW501rdAHbfR/blED7KIXyUPSvg4PDee+8NGvTg3GlHR0eBQGCz2Uqv6PX6R5+aM2fO1KlTS68olcrs7GyBQECSDWkCSkOqK0J1an/SkdSSDG+FZ2RAeC2LIgmir28Y4HZrqIF45plnNm7cKJfLVSqVSqUyGAwkSQYEBNy+fZtrGu7evVur1T761G+//SYSiUqfKj16omHBIEQIAIBhmc3XdwDAix0m17I5yOHmjkannGChYZxEg5qy999/39XVtXnz5oMHD+7QocPMmTMBwN/f/9lnn+3QoUOvXr327Nnz6MyX2bNnh4WFtWrVauDAgV27dh0xYoQ96s4D7BpFCADgeNqZtJIMX6V3pH9fXgps79rGTeaSo89LyL/V1iWQlzIRqiNSqXTjxo0lJSXJycnu7u7u7u7c9Z9//jklJcVms7Vo0eKbb75xcHAAgA0bNnAtP5Ikly9fvmTJkjt37jg7O3t7ewOAp6dnZmamHb9LDWCLECGA+32YI1sNJQl+/lAQQPT36w0AR1JO8lIgQnVNqVR27NixNAU5fn5+3Cnzjo6OXP6JxeKyu2nL5fKOHTtyKQgAJElyB9Y3IBiECIHJZj6XeYkAoq9vKI/FlvaOMg3knG6EmiYMQoTgVMZ5k83U3jXITebCY7FtnFt5KzzzjZprefE8FosQ4heOESIER1NOAkB/v168l9zPt9fm6zuiU052dGvHe+EI8cVisfzwww8HDx7U6XTcHJmoqCiWZSdOnAgAUqnUycmpW7duo0aNKj19cO7cudOmTevYsWN0dPRPP/0EAI6OjkFBQS+99BLXNZqZmTlnzpwtW7ZwHar79u3bunXrN998k56e/ssvv1y9elWpVI4bN27q1KkEQdjvqwNgixAhk818PusyAUQfHz77RTkD7g8T0izNe+EI8WX+/Plbt2595513li1bFhERcevWLQBgWXb79u1hYWFDhw719/ffsGFDYGDgpUuXuEd27tyZkZEBAElJSbGxsePHjw8PD9+zZ09ERATDMACg1Wq3b9/OsiwA7Nq1a+rUqc8//7ybm9vff//t4eHxySefTJ06dcGCBT///LP9vvc92CJETR3XL9rBtS2//aKclqrmfo4+KcVpV3Kud/XoyHv5CPFiz5493377bUREBAB069at7EeDBw8OCgoCgLlz586ZM+fFF1+8evVqucednZ3Hjx8PAOHh4b6+vikpKQEBAaWf/vbbb2+//fbevXt79OgBAJ999lnpR4mJif/888/s2bPr7JtVCQYhaurqrl+U0883bGPctuiUExiE6HFy9LkJBYn19jp/R19/R5+yV5o3b75mzRo/P78OHTo8YVH8q6++umLFirt37zZv3rzCG7KzsymKksvlpVeWL1++YsWKo0ePtm3b9tH74+PjH1dUfcIgRE2ayWaqu35RTqR/+Ma4bcfTzswJsfM/e9FTa8GRz5KK6+/0CSEp/HfCFhH1YLvRtWvXzpkzJzQ0VCQSRUVFLV26tMJ88vf3JwgiKyur3Kd3796dNWuWXq8/evTokiVLyi69f++999atW1dhCu7evfu///6Li4vj75vVEAYhatJOZVyou35Rjq+yWYCTX1JRyqXsK0GKVnX0FtSgjW0z/GJWbL29zlfZrGwKAoCfn99ff/1lNBovXrz4xRdfDBo06ObNm4/OYdFoNCzLcsvqy3J0dIyMjMzNzT137pyHh0fZj3777bdZs2b5+voOGDCg7PWjR4/OmDFj9+7dnp6e/H2zGsIgRE1aXfeLcgb49V5flBKdciKoPQYhqsDwloOHtxxs71qAVCrt06fP2rVrfXx80tPTfXx8yt1w9OhRJycnbsiwrNIxwtDQ0P79+0dGRpaurx83bhxFUaNHj/7rr79Ks/DUqVMTJ07ctm1bWFhYHX+nKsFZo6jpqod+UU6kfzgAnEg/a6WtdfoihGpm69atOp2O+/nQoUMODg7lGmp5eXkbNmx4/fXXP/nkE5FIVFEZAABdu3YdPHjwF198Ufbi+PHj161bN3r0aO7AprNnz44aNWrDhg39+9fw7Gve8dYivHz5cnZ2dvfu3cueX1zKZDJdvnxZr9d36NCh3P49CNlLPfSLcrwcPFqpmicW3r2cHxfhVNujLRDi3d9///3yyy/7+PiYzWaLxfL777+LRCJuFUTPnj1pmpbL5Z06dVq7du3o0aOfXNTHH38cHBz87rvvlr3ItRdHjx79999///DDD/n5+cOGDeM+6tKlS+mSDHvhJwhfeumlY8eOtWvX7oUXXvjrr7969Xqoo+nSpUsRERGtW7d2dnY+derU119/PWvWLF7ei1Bt1E+/KGeAf5/EwrvH089GtMQgRE+drVu3mkymrKwsiUTi4eHBjQ6SJMk+fnfA1NR7s3umT58+ffr00uvt2rUzGo3cz2UfHz9+PBeHT09DsBQPQXj+/Pl//vnn5s2barV65cqV8+fPP3XqVNkbvLy8YmNj/fz8AGDfvn2jR49+8cUXxWJx7V+NUI1x/aIkQYT71McoRYRf3zWXN53LiTHTFjH12J4lhOxFIpGUXfzXpPAwRvjXX38NGTJErVYDwJQpU86cOZOTk1P2Bg8PDy4FASAoKMhisZT+ewEhezmdccFkM7VzaeMiq6Azn3fuctcgl1ZGm+lcZkw9vA4hVHU8tAjT0tJK/x2hUqkcHBzS0tIeNxD43XffDR069HGHdBiNxvT09D/++KP0Sr9+/Vxc6nb8xr5YlmUYhuuLb1Ls/q2505HCfXrVWzV6egXH59+6lpvQ27tHVe5naZvmx4UgkTvP+KRua1bH7P5rbRfct65wF80n9DfaV3Fxsc1mq3Cex9OA+9uyuk+RZOXtPR6C0GKxlD2bSiQSmc3mCu9ct27drl27Tp8+/biiCgsL09PTt23bVnrF29v70TUrjQnLsiaT6QlbOTRWZrNZKBTa6+2l/aI93bqaTKb6eamclAFAiUlbxTdaLh0x370OBGHU6wiqAa90su+vtb2YTCaWZSv8o221Po2Th+fOnfvnn38KhcLu3btv2rSJ+1v9wIEDq1evTk9Pd3Jy6t69++LFiymK+uGHH44dOwYALi4uvr6+AwcOLN2Vbdu2bRkZGW+//Xbpht0A4OnpOXHixNKVEosWLerXrx+3nVt2dvb8+fOnTJkSHBy8cuXK48ePGwyGrl27fvjhh4+uL2QYpgZ/WiUSSaVZyMOfLg8Pj/z8fO5nq9VaVFRU4QLJ33777dNPP42Oji5dX/IoLy+vnj177tixo/a1aii4fxvKZDJ7V6S+0TRtx299JiWGmy/q49ys3l6qkjsBgJm1VOmLs6z25G7uBwltoRTKOq5dHbLvr7UdicXiCoPwCcsP7GXfvn3R0dE3btwQiUSDBw9et27d7Nmzz549O2HChB9//DE4ODgzM/Off/5hGIaiqAsXLhAEMXPmzJycnCtXrvTr1+/1119funQpAMTHxyckJMD9DbvXr1/v5eUVExPTv3//Y8eO9ezZEwCio6N9fX0jIiJSU1MHDhw4evToQYMGXbhwISsr67333lOpVF999dWwYcMenUpKUVQd/UbiIQhDQ0OXLl3KsixBECdOnPDw8PD19S13z44dO+bPn3/w4MFWrXBBMbK/+pwvWkoukgGA3mqoys3GKyet2Sncz3SJhnJqzAMEyO42bNgwbdo0qVQKADNnzly2bNns2bMPHTrUv3//yZMnA0CrVq3Cwx9MePb394+MjASA5557bsKECT179hwxYkRoaPn1uKGhoUFBQVFRUceOHdu7dy8XhJybN28OGjTotddemz9/PgCEhISEhIRwH61cudLT0zMnJ6fe1trxEIRjxoxZtGjR7Nmz+/Tp89lnn73zzjtcm3rChAkBAQFfffXVxYsXJ0+ePGTIkF9//ZV7ZM6cOeW24UGo3tTzfNFSciEXhPrKb2XZkkNbAYAQCFmbldFq6rpuyL6MeWZ9Zj110QOARC1y8JGWvXLr1q0hQ4bcvXsXABQKBXcMU2Bg4Hfffbd58+ZBgwa5uro+rrTg4ODQ0NC9e/c+GoQcmqYzMjK4CZWcK1eufPzxxx9//HGFS+liY2PVanV9zg7hIQjFYvHJkydXrlx55MiRzz77bNKkSdz1iRMnqlQqAHB0dPz888/LPtIEh8TQ06N0HX39zBctJRfKAUBvrXzKtCnhgjX9DqVUiVt1McRE08UYhI1cwv9SDTkVT62oCwRFhC5tSwoeTOQpKCj48MMPJRIJANhstqKiIpqmx40bl5KS8vHHH0+ZMqVDhw7cSbwVFujv75+dnf3o9blz5yqVyri4uICAgLJrDdesWRMWFjZz5sxHH8nPz3/llVf+7//+rz5jgp8ReA8PjyVLlpS7OHbsWO6HVq1aLViwgJcXIVR7dukXBQC5UAoAekvlLcKSg9sAwGHAeNagBQC6BIOwkWsW4aq5rq231zn4SMqmIAC4urouWbJkyJAhAHDt2rV+/fpxOTRv3rx58+ZlZGRs37591qxZzZo1Gzhw4KMF5uXlVXi+RFRUlKenp0AgyMrK4vpdOZ988snWrVunT5++fv36sjNZCgsLBw0aNHny5Mclbh1pwFPREKoBe/WLAoBMWKUxQvPtq5ak66Rc6RA6RH/hEADQ2sL6qB+yH7dgJ7fgiheV1Y/AwMAbN25wQZiQkBAYGFj2U29v7zlz5vz777+nTp16NAhzc3PPnj1bYfOOO9R3woQJoaGhy5Yt44YDAcDV1fXw4cMRERFls7C4uDgqKqpfv37ltiqtB7jpNmpaTt1bRx9Uz/2iACAXygggjDYj88RlZCUHtgCAQ/hoQiyllGoAYEowCFHdmj59+rp163Q6ndVqXbVqFdeNeeTIkcuXL3M3JCcnX716tdy5ExqNJjo6evjw4e3atRs1atTjCicI4vPPP1+6dGlRUVHpRS4LY2JiZsyYwTCMXq8fMWJEp06dvv3227r5ik+CQYiaFnv1iwIASZBiSsSwrMn22GkRltSb5luXSYnMofdwAKAUKsCuUVT3IiMjx4wZ06pVq4CAgMDAwBdeeAEANBrN2LFjnZ2dAwMDO3XqNHXq1AkTJgCATCZbu3atWq0ODAx8//33R48efejQIa4rVSgUli4OUalUpeN8kZGRISEhv/zyCwAoFApui01XV9eDBw9euXJlwYIFJ0+ejIuL27Fjh7Ozs1qtVqvV9XlgL3aNoibEZDOdz7pkl35RjlwoM9FmvVUvE0orvEF7YAsAyHsPJ2UOAEAq1YBBiOrF4sWLP/jgA4ZhStfqjR07duzYsVqtVqvVenh4lA7mrV69evXq1RUW8uGHH3I/kCSp0Tz0+/bAgQPcD//991/pRXd395iYe5sOlru/PmEQoibkVMYFk81c//NFS8mEsgJTod5qrHAqujU71Xj9HCEUOfQdyV253zWqAZaFijbrQohH3KzRchQKhUKhqP/K1CfsGkVNCL/9orSJsZTYqvWITCCBxy8l1B74HVhWHjqEyz8AIIQiUurA0jbGqKtlbRFCj4NBiJoKfvtFjbnmS18nxiy9ZcyzVP0pmeCxE0dt+VmG2BMEJVD0G1v2OqnEYUJUT3JyctLT0+1dCzvAIERNBdcvyst8UV2G6eqqJHORlTYzt7aks0xVDxO4t7mMpYIg1B7aCgwtC4mk1G5lr+PEUVQPWJadNWtW7969Bw8ePGrUKIulGv+8AwCdTrdhw4Y6qls9wCBETQVf/aIlSYZrPyRZdTZVkELsJNQmGzKO5FfxWZlAChW1COmifMOFw0CSiojx5T7CiaOoHvzzzz8XL168fv16XFyc2Wxes2ZNtR4vKCh466236qhu9QAny6Amga9+0eLb+vj1KbSZce3i2PrZZsV39dd+Sk75L1cVpJB7VTDRoJzHBaE2ejtL22Rd+wlcy5/NQuHEUVT3Nm7c+OKLL3IrH6ZPn/7VV1+9/vrrAJCXl7d8+fJbt24FBQXNmzdPqVQeO3YsLi6O+zQjI+PHH39cuHDh6tWrzWbze++9BwDz5s27cuWKTqfLysr677//PvroI5FItHHjxjt37ri5ub3yyiudOnUCgMuXL8fGxorF4j///NPf33/hwoV2PAcRgxA1CbzMF9Vc197YmMrYWLdgp1aTvAmScGrl4NlLnXVSc+v39M5zWxBUJRM7uc1lDA9vN8roivRn/wOCUAyc9OgjZOnEUdR4FecnaLJi6u11CnVLF++eZa/cvn27X79+3EoGm812+/ZtACguLg4ODn711VfnzJnz999/R0VFnTp1Kjg4+JVXXlEqlVOmTHn++ecHDBggl8tDQkIoiuLOo5DJZNzu0y+//PKrr77q6up64cKFzp07jxkz5vr165GRkbGxsd7e3tevX3/nnXfGjRs3Z86cbdu2RUZGxsTEVOUQ3bqAQYiahNr3i+bGFCVuyWAZ1quPc/NRnnA/8gKe8Si6qddnmlIP5PoNqeTUmApnjWqP/sVazNIOoUJP/0cfudcixF3WGrVze2ZqC+/U2+sIUjDytduUQFx6RaPRrF69euPGjQBgtVqLi4tpml6zZk2fPn24naJ79eoVFBQUExMTHBy8devWyMjII0eOkCS5cOFCAAgJCREIBFwQctq3b1+6U5qPj4/FYsnOzhaJRCEhIfv373/ppZcAQCqVrl69WiAQhIWFtWzZ8ujRowMGDKi3/wllYRCixs9oM53LjKlNv2jWKc2dPzOBhWYDXP2feSjtSBHZerL31VVJ6Yfz1W2VCr+KV8pzHt1ulDHp9af2AoAiYkKFj1A4a7QJaBs2Pzf1RL29TqluXTYFAcDNzW3x4sVDhw4FgLi4uAEDBlAUlZCQcOTIkeDgYO6ekpKSzMxMAOjYseNzzz33/fff37lz53FtuLJ7cO/cuXPevHktWrRQq9WJiYlZWVnc9TZt2nBn9pEk2a5du9u3b2MQIlRXTmdcMNOWGveLpkfnJe/JAQICRnp4h1dwRprCX+bdzyU9Ou/WlvQu77QghY/t3nl0jFB3bBdj1EkCu4r8gyp8BDeXaQqatR7RrPUIO1YgKCgoISGBC8Lr169ze4q6uLiMGzduxYoV5W5OTEzcvHlzeHj4l19++dNPPwEAQRDswzvocgnHmTt37h9//MGdyjt+/PjSO8tuJVNQUGDHMUKcNYoav5r3i7KQ9E928p4cgiRaTfCuMAU5vlFuci+JMdecvDfnCeXJH24RshaT7sQuAKhwdJDDzRrF5ROoTs2cOXPNmjVFRUVms/n777+fMWMGAIwfP37z5s3Xrl3j7rl69arBYDCbzZMmTfr4449379595MiR33//HQBcXFxMJlNpU68cq9VqMpkA4MqVK3v37i29fuXKlSNHjgDAmTNn4uPjw8PD6/prPg62CFEjV+N+UZZh7+zIzD5bSFBE4BQfl07KJ9xMCojWk5vFLr+TeaLAub3SsaW8wtvutQjvryPUndrL6IpF/m3ELTs+tmSZghAIGaOOtVoIoahaXwGhKgoPD3/xxRcDAwMJgpg0adLUqVMBICQkZPXq1aNHjwYAg8Hg6+u7f//+pUuXtm7d+rXXXgOAzZs3T5o0KTw83Nvb+7PPPuvTp49Go7l48aJUKi17+uA333wzduxYNzc3d3f3CRMmlH7Ut2/fb775ZtasWVqt9pdffqnPI+nLwSBEjVzN+kVZmr25OT0/tpgUkUEv+qraOFT6iNxb4jvQNeW/3MStGV3mtaQkFXS3lO0aZW1W3dE/AUA56LknlUsQpEJFF+bSJRqBs0fVvwJC1bJw4cIFCxYwDINglwQAACAASURBVFO2V3PChAkTJkzQ6/UikUgoFALA4sWLSz8NDg7m5pdyj3MTZwCg9NxBzpQpUyZPnqzX65XKh/41KZfL9+7dq9VqHRwcCLtupYtdo6iRq0G/KGNjb2xKy48tFkip9rP8q5KCnGaRrgo/qUljSdpdcR/R/SDUA4D+3H66uEDo3VwSFPzkYu9tLoMTR1EdI0mybAqWksvlXArWGEVR5VKwlEKhsG8KArYIUeNWg35R2swkbEgtuqUTOgjaz/KXe1e+TL4UQRKtJze7/O2d7LOF6vZKddvye/bfnzVqBIbRHeGag5MrPVYCJ46ixqdHjx52nB1TDrYIUWN2Ov28mba0r3K/qM1IX/spueiWTqQUdHileinIkbqJ/Ya6AcDtPzJserrcpxKBmCRIk82kjTlky88UuPtIO/autMz7E0exRYgaj1atWg0ZMsTetbgHgxA1ZkdSTwFAP98q9YtatLa4VUnaFINELer4RnOZZ7VTkOPd18WxpdxSYrvzZ2a5jwgguCN5c49sBwBlxISqnDJ4f+JoQc3qgxB6MgxC1GgZbabz9/pFQyu92aK1Xf3+rj7LJPeSdHyrucS5FvMzCWg1yZsSk3mXi/Nji8t9yK2g0BakUyo3abf+VSmPwhYhQnUJgxA1WtXqF03alWUqsCj8pB1eCxApajt2LlGLAkZ6AsDtnZkW7UOH93JBaKBAETmBoKr0IlxTj1CdwiBEjVbV+0VLkgx5l4tJIRk41UcgpXh5u0dPlSpIYdPTt7dllL0utTIAYFYo5N0HVbEonDWKUJ3CWaOocap6vyjLsHd2ZgILPhEuEjWfK9ZbTfS+9HWiJl6bc67QvYeKuygu1IAI2M69qr46HmeNNjJyuXz9+vVbtmyxd0UaEqvVOmJEXe1Ch0GIGieuX7SjW7tK+0WzTmn0mSaJs8h7gCu/dRApBS3GeN38Le3uriyn1g5ildCWFC/WloAzRbdsW/nz95EKFRAErS0ChgE7nVODePT888/X3d/p9mW1WhmGEYvFld9afXJ5xRs21R4GIWqc7veLVrJ80Kqzpf6XCwDNR3mSAv5X9bp2dSy4VpIfW5y4Nb397ADzsZ0yhgAAA9gqfbYUQQlImYLRl9D6Ym4GKWrQCIJQqRrnr2OdBmHdwX9dokaoxKw9m3GBJMjwygYIk/fm2Iy0U6CDul35xe98aTHWS6QQFCXq0/5JsN25KgMKKjqk/snuDRPixFGE6gAGIWqEDiYfszK27p5dXaTqJ9ymSzPmnC8kKKLFGM+6q4xQTrWc4A0AqcfNVnBVNmsDNQ1CHCZEqC5gEKJG6L+7hwEgqvkTD/lkgZsj06y/i9S12j05eelnDm4K/3dtF01WTKU3q9spnAOMLCPIF0xRB4YAgKGaQUjiOfUI1RkcI0SNTVJRyi3NHaVI0atZ9yfcln2uUJtqFDsJm0W6AgDNws4kpshSefmEpUB+Y7E4YwcACwDHd4ztOvA73zZjnvAIYzIo8lYXw0tmxluVXALcdqPVwU0cZbBFiFAdwCBEjc2/dw4BQIR/XxH12PUJNgOdsjcbAAJGeFAikmFh2jH619vMk0smgB1g3TbFvFTMFloJ8Z/C15wgf7Bl04V9r2sLbrUNm08QFXexlPy7EUqy1K43cgtCxTkykILeoq/Wl8KuUYTqDgYhalRolj6Uchwq6xdN2Zdj1dPK5nKXTo4MC9NP0L/eZhRCmNSCfNzMUbnpbpvkhc6mUwBQqAiN91taaGuxPYOlFe2eKfzwxvkVRXnXug/9USgqP+nGmpWsO7kHSFI9oF/udjNZIgBptccISQUuJUSormAQokblTMZFjbHQ39GnjXOrx91jyDJlnykkSKLFWE+WgNdO0b/cYmQC2D1I0M+zghykbaabF1bdjP+eoa0SuVv73h/6tR0PAEYbtN9p+592cnAXv2bXZ2UnHT62bVTYyI0yZbMHD7Ns0c4fgKEdwkdJAv0AbkExAe41mCzjDDhrFKG6gZNlUKPCTZMZ0iLysXewcOfPLJZhvfo4yzwlr5+mf0pgZALYM7jiFMxOOnRgY9+Es98ytM03aNzA549yKQgAUgGs7kUBwIKksKBR/yrUrYrz46N/H5yfcbb0ccOFQ+bbcZRCpYyaInYSEhTB6FgRI6x+EGKLEKG6gkGIGo8Ss/ZsxkWKoAb693vcPbkxRcV39EKFoNlg1zdP06vjGakAdg8S9H8kBU267Av/vXHq76mGkjRH13b9J/0TErVSJHloHfTgZsTYAFJrhYXxfv0m/uPm29ds1JzYOTElfjsAMCZD8Z4NAOA4YgYpdSBIQuREAQtOFpXeUqNZoxiECNUB7BpFjcf+pCNWxhbmHeIsrXjbDtrMJO/JBoCA4R4fXoVV8YyIhO0Rggivh1KQZZnka7/HHf/MatFSAmlQz7dbd5tNkBX/YVkZSh3KYHYkMS+0Ug4b8/vVY5/cvrzu4v43i3Kv+mo86RKNKKCdLPjegKVIJTAX2NQm5zuy29X6aqRERogkrMXEmo2EWFqtZxFCT4YtQtR4/Hc3GgCimkc87obU/bmWEpvSX7acUXx9lRGRsDNSMMznoRQsyo07smXYpUPvWi1az+YDB714IjDk9celIAB4yuCjLhQAvH6aNtBUp36Lu0Z+Q5LC25fXXbq5ghawqnGvlZ6+K1YLAcDN6mJlrFbaWq1vh72jCNURDELUSNwtSrldeFcpUoR5h1R4gyHHnHmiAAjYH+i+9CrXFqSe8X2QglZzyZWji6J/H1KYEyt18Oj5zLqwkZtkCu9KX/1mO7KLM5GiY7+8QgNAQIcpvUb9KiRERZKSG/45VsWDVRwitQAAXK1uAKCr9sRR7B1FqE5gEKJGYu+dAwAQGRAupIQV3pC0K4ul2bSWqvlpIiEJ2yKoEX4Pfv8X5ycc2Nj39uV1BEEGhrw+eNpp71bDqvhqAQk/96ZIAr6+wiQUsQDgkGNpm95cQkt11pwjW4eV7j4jUgkAwNnsDNXfXAa3G0WojmAQosbAxtCHko4DwJDH9IvmXykuvKGziqhXSGeKgE3h1KgyKWjS557+e6pJn+PsFRLx3IH2vT+gBNUbhwtxJaYHkhYGZp+kaZOheM8GiU3cu9dyN9++Jn3u8R1jU2/8CfeD0MnkBAA6a3XX1GPXKEJ1AoMQNQZnMs4XmYsDnPxaq1s8+iljZZL/yQGAFc4uOgG1qR81qcWD3/m0zXzmn5cM2gxnr5C+43YoXdrUrA5fhlBuUjiezV78/Rdujoxjz+G9x/zesssM2ma+sO+1K0cXiVUUEKAwKUggDNXeZQ23G0WoTmAQosZg393DADD0Mc3BtEN5Jo3llkTyr8rxl3Dq2RZlf9uzF/e/qcmKkTv6hY7YQD5+V7ZKqcXwdXcq0JTiHrcXSFI17lUgCIKgOvVb3HnAUoIU3L687urxN4UKmmQohUVZ7c1lsEWIUN3AIEQNXqGp+FxmDEVQkf7hj35qKrCkROezBKzwdFsXLpjS8qHf89dPf5V+a7dQrOw16lextJKz7Cv1fEtiVcFPApa+0GK40PtB27RFpxd7j94skjjmJh80qrYDgNrsXLPtRnHfbYR4h0GIGryDSUdsDB3qHayuaPng4U2ZJM3uc3R8daDD860e+g2fmrDjxrnvCVLQc9gahfqxW7JVnfHi4TaF1woEqpeEk87ksmU/cvPtGzz4ewAwUucBwNmsrvYBFDhrFKG6gUGIGrz7ywcr2GV7yyGtY5pOT5EBw9xntXnod3t+xvmYg+8AsJ36LXbzq6ApWV2MyVD8z/8AILb7tCLSYfZJ2vbwaRauPr1IUmi0JTCkXm1W66s5Web+5jI4RogQzzAIUcN2U3P7TlGyUqzo6RVc7qMNN2g4lAUAxSFuM7s9NPinL049u2c6Q1tadX25RacXealJyb8buTkyE0YPbK4grmrY1QkPJaFAKFe6tGdZm1UcrzKrq73dqIMjkCSjL2ZpGy8VRghxMAhRw8btsj04oH+55YObEpnjf+f5WCwmJ/H4sQ8N/lkt2tO7XjAb8j38B3To+xEv1Sg9a0k17lWpkFgRSgHAoot0puGhDlK1V08AsIhjnc3O1Z01CiRJOTgCyzK6Yl7qjBDiYBCiBszK2A4nn4BH+kUTithFR83P5WsAIORZL4J6sH0My9jO/jOjpOCG0jmw+9AfCYLioR6lZy31vjdH5hlfYrQ/WWKFuWcfahSqPLsDgFUcq7KoqruOEB70jhbwUGeE0H0YhKgBO5V+rthc0tzJr6WqeelFKwMvHKNnZORKGMals6NjS3nZR2KPfpibelwid+s1erNQrOSlGoaLh++dtTRkaunFFaGkgxD+uMv8m/agUajy6EZSIqvwtpRhrYbq7TUKuLkMQnUDgxA1YFy/6LAWg8peXBRD08n6/iVaUkQGjPAo+1FizM93r2ykBJLQERuqsoloVZTOkXEcMZ2UOpRe95ETH3elAOCtM7SJvneRpCRqjy5AMBbxVbK44q3gngAnjiJUFzAIUUNVaCo6n3lZQFKRAX1LL57MZpddpefk5AKAT6Sr2OlB2GQnR8edWAxAdBu0TO3Rla9qlM6RkQWXX84/px3ZSU3cLmGXxtKlF119egGARRwrKKn2IWgUThxFqA5gEKKG6r+70TRLh3p3dxI7cleKLTDlKD2yoCjAZJY4i7z7uZTeXFJw8/ze2SxLt+/1nk/gKL7qYM1K1p96sI9MuU9LN+P+8gpzo+heB+m9IJRckeirfawgt7kMrqlHiF8YhKihOpB0BB7eZfu103RJsXVmfgEANB/lSQruJZNJn3Pyr2etFq1f2wmB3d/krQYsW7TzB5a2lc6ReVQPN2Jaa9LCwOxTNJeEzp7BJCm2Ce7Kq7d6AuDBdqMYhAjxCYMQNUgJBbfuFqWoJI49vLpxV3YmMZtvM6/m5UtttFNrB3U7BXedtpnO7H7JqM108e7eNfIbHutQ4RyZR33dnXKVwLEsdusdBgBISqRUdQGCdbRms8A+4cFH4QEUCNUFDELUIHG7yQwM6C8gKQDI0LMvn6Rbm0yDC4sJimgxxvP+jWzMgbma7EtyR9+ew/9Xmz21y6GLC4p2rQUAx+EvlZ0j8yi1GL7sTgHA3LN0sZUAAHef3gAgJ1LMNnO1XkrirFGE6gAGIWp4rIwtOuUEAAwO6A8ADAvPH6MLTfBZYS7Bgne4s9RNzN157eTStJt/C0WKsJGbar+n9gMMrdn0JaMrlrTpJguJrPT2aa3Jfp5EjhEWxwkAwKNVHwAghPFaYzX33VbcP4mJrV5TEiH0BBiEqOE5mXa2xKwNVLdsqQoAgGXXmOhMdqKxxLvQKFQImkW6crelxP9x88JKkhT2HL5e6RzIYwWK9/1qvhNHKVSqZ+c9OkfmUQTAyjBKSML629TFfFbt1YVgxTZBqiYjtVrvJURiUiJjrRammgmKEHoCDELU8HCnDw5uPgAArmrYDy7ScoZ5NS8PAAKGewgkFADkpZ+5dGgeAHSOWOrm24fHt5sTY7WH/wCCUE9dwA3aVUV7FfFWO5JmYeV1hiSFDNMaCDb/ztnqvv3+5jI4TIgQbzAIUQOTb9RcyLosJAWR/n3NNEw9SptpWM5oCL1N4S9z6+YEAEZt5rk9Mxna2rrbKwHtn+Px7XRJoebXr4BhlEOfF7fuXK1nX25DAsCeVMbGgE3QAgB0uTHVrcC9zWVw4ihC/MEgRA3MgaQjDMv0atbDUax87wJ9VcP2Fdva3tUAAS1GeQIBDG09u/dls7HA3b9/+z4f8vluhtH8+hVdUihu1VkZMbG6T7dyJFopWI0ZTuWwZllzADBpY6tbCE4cRYh3GISogeHmiw5uPuBQBrviGiMgYWlJLmNjPXqqHXylAHDl2CJNVoxM4R0StYog+PwdXrxvkzkxllKo1FMXAFmTkod60wCwO5WxqjxJRm6jM4y67GqVQCqdATeXQYhXGISoIYnPv5lSnKaSOLV27jr9BM0CrHIzMHe0AinlN8QNANJu/Hn3ykZKIO45fL1Yqubx1TUbGixnmDcDALtSWEJFCi3tACA//XS1SqAUuLkMQjzDIEQNCTdNJqr5gNdPQ6qO7esKwXE5AOAb5SZ0EBTnJ8QcnAcAnfovUbl34vG9tRkaLKu7M+MuhTslbKGMFJo7A0BeWjWDELcbRYhvGISowbDQFm75oIkK33aXcRTBKqrQmGeReYg9w9Q2i+7c3pdpm9E3aCy/E2RqOTRYFknAUB8SAG4ZRYw1EAByUk9WrwQcI0SIbxiEqME4kXZWZ9EHOLX6+HIzAFjdkdGdzAeAFmO8CAouHpij1dx2dG3H7z5qwMfQYFkj/QgAuF4kLaIcSdbBUJJiKEmv+uM4axQh3mEQogaD6xdNNvcvscIYf7Lr1Vzacu/o3ZsXVmUk7hVJHEOHr6cE1T7V4Ql4GRosa5A3KRPAba1UIyoSmjoCQF51hgkpXEeIEN8wCFHDkG8oiMm+QhKC05owbzmx3Necf6WYFJIBwz3y0k5dP/01QZAhUT/IHf14fClfQ4NlSQUQ4UXaWGmBWCMydwKAvLRTVX+clCkISsAYdKzVwkt9EEIYhKhh+O9uNMMyeXR3GyjWhpH5uzOBBZ9IV1akOb/vNZaxtekxxyOg/NG4tcKyfA0NljPSj7Cx8kJxgcjcGQBy06ozTEgQpEIF3I6jCCE+YBCihuG/pCMAkGXrP7cD2TmtUJ9pkjiLPMMdz+192aTPcfPtE9TzbX7fWPIvn0ODZQ33JVmQasQagTWAAkejNlNfnFL1xyk8gwIhXmEQogYgJvtKWkm6hVV5Krp8GgQp+3MBIGCkx7WTn+RnnJcpvLsP/YkgKB7faE6MLTm8jcehwbLcpNDBWa4Ra4AlRNZq947i5jII8QuDEDUA313YAgA5zNDNA4TZ+3NsetqptYNBePjOlQ0kJeo5fB2/a+dp7f2hwSG8DQ2WM9TXQSvQWimrQN8RqjtMiOfUI8QrDEL0tItOvZJeEm8FxeyOz7QwmnPOFRIk4RZefOngPADo3H+Jyp3XrHowNNhJGcnn0GBZo/xlLAGFwkKRkRsmrFaLkOsaxSBEiB8YhOhp98W5LQAglIyc11Ge9HcWy7AevSWx59+wWQ2+bcYEdJjC7+tK/t1kvsUNDb7H79BgWe1UJAtSjaRAQPuKxK4mfY6u8G4Vn+V2WcPNZRDiCwYheqp9dv6y1RJvA8XaiGfyLxUV39ELlVQusVSrSXR0adt14P/x+7o6HRosRyyQFYg0wBIOsm5QnUYhHkmIEL/4CcKLFy9OmjRpyJAhq1evZln20RsOHDjw+eefz5o1KyEhgZc3oqbgTgm769Y2AOjlM6qFVJK8JxsAhJ33Zt7dKxQreV87Xw9Dg2WpxDKNWAMAMugKAHnpVQ1CyhFnjSLEJx6CMCsrKzIyMiwsbN68eStWrPjxxx8fveeLL77Iysratm1bRkZG7d+ImgIbA88fuuRAxJOk4pPQZ9IO5lpKbJTPjeSUFQBE8KDlcid/Pt/H0JqNX9AlhZLArsqBk/gs+TGcpbJCsQYAbFpu9+1TABX8I/JR97tGC+q0egg1HTwE4f/+97/w8PA333wzIiLiq6++Wr58+aP3HD169IcfflAoFLV/HWoiPo+l9frtADCl3RgoIDOOFTCUpkCymGVsbXq85dVyCJ8vY9nCbd+bb8dRjs7qqfOBIPgs/DEcRDKNuAAAjPnuUgcPsyG/pCCxKg+SSjUQBKMrhop6XxBC1cVDEF66dCksLIz7OSwsLDExUavV1r5Y1JRdzGdXX73kSMZLhYpJrYfc2pzGMFZDwJdmU66bb5+2ofP4fV3xrrX6c/sJkcT5pUWkgxO/hT+OXCjTiIoYYEU6q6t3GADkVW2LGYISkDIFS9sYfXEd1xGhJkFQ+yJycnJUqnvTCtRqNQBkZ2fXrPGXkpJy+PDhrl27cv9JUdSSJUtKU7ZRYlnWaDRWOLDauOn1euIxDS8DTUw+LPSidgLApNbPZOwr0GWYDO5r9ebLEgfPduH/p9cbeayJKfoP09E/CUogm/S2xbmZRafjsfByyn5rEQhp0lYkZNRWgpF0BfgzK+mYe8sJVSmHkDuCvkSbnUF58PBHuK494de6ETMYDFarlaL43Ofh6We1WhmGsVqt9q7IAzKZjKxs+jcPf4ocHByMxnt/MRkMBgCocRdos2bNunbt+vXXX5deadu2rVTK54SIpw3LsiRJyuVye1ekvrEs6+DgUOFHc07QBYYYL1G8o1g5zCHq1qlMk/yATvQnSYlCh69Xu/jyWA3dyX9M0X8ASaqnzJd26ctjyRUq+62dZI4AUCI3q4tkSZYwIUBh9gW5XEYQlffTGJ1c6Nw0sc0kecz/w6fKE36tGzGSJMVicdMMQrFYbO+KVA8PQejr65uUlMT9nJSUJJVKXV1da1YURVEqlapbt261rxVqoHalMOtvMl3EOwHg2dbjUrfnWURXS5yXAQtdIr5Ue3Th8V2GC4eLdq4GglBNeLMeUrAcuVAGAIzSCEWy+AzXUKWPoSStJD/B0bVdpc/enziKKygQ4gEPY4STJk3asWNHQUEBAKxZs2b8+PHcP4K2b99+6dKl2pePmo5cI8w6SauoywoywVGs7JzQVadJLXb5jGVtrbu94t9uMo/vMsad0Wz5DljWccQMec8oHkuuIplQBgCUox4AinMtqnvDhFU6m/D+xFEMQoR4wEMQ9u/ff/jw4W3btu3UqVN0dPTixYu568uXL4+Oji69R61WZ2Zmjh49Wq1Wx8fH1/69qJFhAV46bssxQkfZTgCY5jg192JWkdsimijyCIho3+cDHt9lvhWr2bQUGFoZNUXRfyyPJVedg1AGACZ5CQC4mS2ZsjAAyK3aasL7243iUkKEeMBD1yhBED/++ONHH31UWFjYpk2b0mHJI0eOlP68a9cumqZLH1EqlbV/L2pkfohn9qax/pLLrC3Bg/LwPO2Vp37PJkh2cm3XY9jPPB4uYUm5kb/+U9ZqcegzQhnF8w5tVce1CDWSQgDwtlgPWUIjAPLTz7AsXemXxXPqEeIRb1POPD09PT09y14RiUSlP2PyoSdLKGIXnKcBoIfyzywtTM9/qZD83iK5JJG7h43cJBDyNpnImpWc//Mi1myUhUQ6jXmFr2JrwEEkB4B8UR4AeFusf2T5j3Dy1xclF+VeV7l3fPKzeCQhQjzCvUaR/VkZeOEYbbDBZN/YLG18WEkYm7/P4LCHoiShI/4nVXjx9SJbfmb+jwsZg1baIUw9eW79LJx/HJlQCgCFUCSQUlKGYQw2xrkXVG2vNRwjRIhHGITI/j6KoS/ksc0VhAOzTWlVRmS56JTrAIhug5epPbry9Ra6KD9/9UK6pFDcuov6hYVA2nleOzdr1GA1SpxFAOBttsaRPaFqZxOSjtg1ihBvMAiRnZ3MZr+5yghIWNQhNrEg8dn03iWOXwDBtO+90CdwFF9vYXTFeT++b9PkiPzbuEz/iBAI+Sq5xuRCOQDoLHqJiwgAvKzWbdpQAMjPOMcwlaxHJiVyQiRmzUbWzOfeAgg1TRiEyJ6KLTD1GE2z8EFn8lza1r65nSSS1Sxp9Gk1PjDkDb7ewpgM+T9/aMtJFXr6u7z8OSF+KrZokAulAGCwGrgWYUvGclnvJnZsYbPoinLjKn38Xu8oThxFqNYwCJE9vX6aTtay3VyISPfLmjRNV8sJhspzdOoWPIS3gwZZq6Vg7ceWtESBi5fLK0tJ2dOyxYlEIKEIykxbhGoBAAQLbACQ68CtJqxC7yhOHEWIJxiEyG7+TGZ+u83IBfB7f2pL3LbxBRqb6IZI4N1n0kaSElX+fBWwtK1gw2LznTjKycXl1aV1fdZudXHDhKQTAwD+NisAnLCFQtWCECeOIsQXDEJkH5kGmHmCBoBlPakCXUzH63cJ0TmSlfcZ+6tY6szPOxim8LdvTPEXSAdHl1e+EKjd+SmWP9zEUavCCgAyvUVCwS5DTwAiP+M8Q1ue/CxOHEWILxiEyA4YFmafF2nMMMKPnNmGPHNwpQt5mmAFncNWO3kF8fMOli3cvtJw+RgpkbvOXiJ053Orbr5wSwktMhMpIKxaW5Q7WwgutEMgbTNqsmOf/Cx2jSLEFwxCZAf/F8cczSHdpbC2N3U0ZpOfJhoAvN3eDeg5iJ8XMEzRztX6M/sIkdhl1mfCZi35KZZv3OYyOptBrBYBC6OdbABwSxQGVVhNeL9rFIMQodrCIET17b909v0LNAGwoa9AbknNP74YCJvEMjpkEj/TRFmzMX/9Z7qT/xACofNLH4kCKj/MwV4c7i0lvDdxNExsIwk4YK7SMCGF240ixBMMQlSvbhWzk6NtNAsL29si3XWHtownQCcyhfScvIQU8LDPC11ckLdqvun6WVKmcJn9haTNU32k170WoeVeEEp1lhBX4jLRAwhSk3WRtpme8Cw38Qe7RhGqPQxCVH9KrDD6IF1kgTH+5LtBxjO7ptuMqUJLCzLgdWdfHuZzWjOTcpfN4VZKuM1ZJm7ZofZl1qn7m8sYpC4iADAVWEb6kTrCqUTShraZNdlPOsWMxFmjCPEEgxDVE4aFZ6Nt8UVsJzWxqR916+ySvPQTJO1sNr82YAwPh+KaEi7mff8OXZQnCmjnNmeZwK1Z7cusa1wQ6u53jZryLSN8CQC4AJWvJqQcnIAkaV0RMPQTbkMIVQqDENWT+efpvWmssxj+HEilXVqRFv87wYgdNIsEUU5SkaSWheuO/Z2/9iPGZJB3H+j6+lekgyMvda5rXBAaS4OwwNpORbRyJKoShECSlNwRWJbWFtVLZRFqtDAIUX349TbzbRwjJGFHpMB6Y9X1U18SQDpqFpzwSR/SdUCtimaYop2ri/76CVhWGTVFNfltguLtcLG6JnvQIhQCASaNhWXY4b5EPNWdBUqTfYm2PWkrUVxBgRAvD+gydQAAIABJREFUMAhRnYvJZ2edpAFgZRjllfnTtZNfEEAqNO8kiV069m8tEYhrXDJrNuav/1R3YjchEKqnvKuMmmLfk5Wqy0F0b4yQFJIipZClWXORdYQvaSCUGcJ2DG0tyLzwhMfvraDAiaMI1Q4GIapbWQYYeZA22uCNdmR/3U9xxz8jgFJo3jHbuh8JPDy81eAal0wXF+SunGe6fo6UK11eXSrrVruWpT3cP4DCAAClvaO9PQgXCcQQXO/o6Sc8jhNHEeIFBiGqQyYaRh20ZejZPh7EK4Kf404sJoBSFLwDlj6/tNwwpvMwcU33FLWk3sz99nVr+h2Bq7fbnGXi5u35rXn94A6g0Fv1ACBxFgKAqcBCETDUh7wmqHw14f2uUWwRIlQrGISoDs04QZ/PY/0VxAqXn+NP3ktBsWnAr81/dfJQjGw1tGbFGq+eyls1nzti1+3tFQJXb36rXW+4MUK91QgA0nstQgsAjPAlEqjuDAg1ObE2i+5xj3PbjeLmMgjVEgYhqitfXmE232YchLDR86e7Zz4ngFJq5kmMkVsDtuU6Zb/b7TVBjc6I1x37u2DD56zFLO852GXWYlL6tByrVAPcXqMGqwEAuON5TfkWABjcjARKfpvqyDK2/Mzzj3ucwskyCPEBgxDVif3p7IcXaZKATZ4/5cUsIQjKsehdiT7yoN+B607X3unxqpfco9qFMnTR9lVFf/0EAMqoKapJcxvQBNEKcadP6O51jT5oEToIYYAXcY2qZBEFzhpFiBcYhIh/N4vZidE2moUfXH5irn9BEJRTyQKxNuKa/5VjzseGNI8Y6N+vumUyBl3ejx/oTu0hhCL11AXKqCl1UPH6JhfcmzUKpUGYf+/0pRF+5DVBT3hiEOKsUYR4gUGIeFZohuEH6GILfCJb7nr3C4Kg1Mb3RUX9C/3ytzn/4a3wfDP45eqWac24k7tirjkxllKqXd/8P1nXfnVQcTsQUkIhKbQxtJm2CB0ElIS0mWibgQaA4b7ELSrYCqKi3GtmY8VtPpw1ihAvMAgRn2gWnjtqSyxm36SWtc/5jiAoV9v7gry+hA+zXL2SJMkPw97h+gOriqFLDm7J/e4tW06a0CvA7e0VIp/WdVZ9OyhdSggPNwq9ZERHV1m8oCfL0rkpRyt8lhBJCLGUtVoY42Mn1CCEKoVBiPj0zjl6Xxo7jVnWt2gZQVDu5IdEZl+xu2Cl9yobaX258wttXaoRY7aCrLxV80v2bmQZWh421G3OMsrJte4qbxf3D6DQw/2Jo8aCe72jI/3Iy1Q/AMhOjn7c4xRuvY1QrWEQIt787xaz4hrzrPW7YfplBEF5yT9mk3uLHIW72v2VZc3u7tl1QtDIqpbFsvrT/+Z89Yr57nVK7eb66lLVhDeJWm9J+hRyEHITR43w8HwZABjhR1wSDACAnOQjLFvxzto4cRSh2mvYk+7Q0+N0DvvqKXqi5bsx5uUEQfm6fmaK6U5JyJQBt06knlFJHBeGvkVAlfY/ozW5mi3fmRNjAUDauY9qwlukrAGvkXiyhyaOujwUhO1VhNSpRbbR38OYXJhzRe3R9dHHMQgRqj1sESIeZBrY8YfpUcbvxpuXEwTV0m+JKaY7QREOYwQ/p28ggJjf8021tEonDhpjT+T836vmxFjSwcl5+kfOL37QiFMQyi0lfHjiKACM8CUuC/oBQHZSxb2jJJ5Tj1CtYRCi2io0wzP76b6F/8elYJugL0tOdgMC/Ma5fZW1zMrYJgaNCvMOqbQcWltYsP7Tgl+WMAadtFMfj4VrpB3C6qH+9lV6AAU80jUKACNKhwmTDlf4ODdxFDeXQag2sGsU1UquEQbvs7bL+mq05QeSFLbvtixvdxtg2YDhHr/Qm9JKMlqrW8zoPLXScoyxJwq3r2T0JaTUwXH4S/KwGu6+1uCUHlIPAGKVkKAIc7GVsbGkgACA3u5EhizMYpQU5V41G/LEsvJzhbhd1rBrFKHawBYhqrlsIwzdUxSZMWu05QeCFHYK+75gX1vGxnqEqa/5XT2QdFQikCzqNU9IPunfW4xRV/jH9wW/LGH0JZI23dwX/NR0UhBKD6m3GACAIAmxkxBYMGvuNQoFJAzylV0ThLEsk5185NHHcXMZhGoPgxDVUIqOHf1X4pTMkT1t+yiRsvvA9QX/BdmMtLqdQjqIWHFxDQDMDZntq3zSjtimGzE5X87Sn/6XEImdRs92mfU55eRSX9/gqVC2RQj358sYyw4T+hFPWERBOToDLp9AqHawaxTVxM1i9o2/Dr1S9IacLZY5tQiNWp/0O2UuNCn8pC2f83rr2EK91dDPt1dU88efEWi1FP35o+7EbmBZUUBb9XPzBC5e9fgNnhayeycx3Q/CR4YJo5qRb4sjwPxRdvJRlrERDzevsWsUodrDIETVdr2Q+WrHypd1XxPAuPoP6h6xIvG3Qn2GXuombjfTf238pvj8W14OHvN7vvG4EkzxF7R/fM8U5RFCkeOwFx3CRzesk+V5dH8d4b0glD4ycdRBCO2b+WbeaO5lvluQdcnFu3vZx0m5kqAEjFHH2qyEQFiPFUeo8cAgRNVzMUv7959vjrT8xwLRotvrbdrPS/g5TZ9lEikF7V72u1x89Y8bf1ME9WGvt7lOv3IsabeKd683J14BAJFPa9WUeUJ333r/Ek+RsrNG4ZGlhJwRvuS52wO8LHdzkqPLBSEQBKlwoovymZJCSu1WT5VGqHHBIETVcDTxTty+aV3oRCupCB26Ui3qc2VFklVrk7qK2s7wM8mNS48uZ1h2Zufn2rm0KfesrSC7ZO8vhsvHgGVJmUIcPsZ54ASo0ZGEjUnZvUahoq5RABjhR/xM9XsG1mUmHW7X671yJVAKNV2UT2s1GIQI1QwGIaqqAzEHc0687sWWFEuCxk38nyVVFbcumbEy/9/encdHVd77A/8+Z5l9spOZrASCAcIii5RFpBgQBStYRa3rVa9t/dV6rfdel171drG1rYJ9WatWa6+tWpW6ASLKjoqIEHZlDUkgyWSyzGT2MzNneX5/nGQIWYBMJpnD5Pt++fJ1cnJm5jkZks885zzP98kYbRlzRxFrZH699RmX0DYpd/wt5dd1fqAS8vs3vRv4fCUVo4TlzJdenbbw9qBEMQWhe48wFoQUYnV48k3EaJ8ZrjFDy7dCwGm0nLGUIw4cRaifMAjR+aArN/xZ+ub3JlCc2Yt/eNMfnVsDp9bXAQX7zKzS6/IIS/51eOX2hp3p+rQnZv83Q9pHI1NZCm5b4/v0TUUIACHGSZelX/PvXLYdAMDvT+YJaYblzFGjrJ7hLZwYkKI+UZd++p7fouGGg3Wzpkkbmmo3l4y/pfMzsOlYdxuhfsEgROcgRQMfvv9TxrmOAltd9PP/uvanVe84WvZ4CENGXGvPvywbAKo9J/+6/00C5OEZ9+cYswAAKBX2b/N+9DfJ5QQAfdnkjMX38IWlyT0XDeq8+oTKkKMTA5LQGu0chEuGk//ecfk0aUNj9yBUB45ilTWE4oVBiM7G33bik/fuYgPH/SSzsfyF/5px2cE/1wROCayBGXN7UeZYKwC0hlyPf/5UVI5eN/p7swunA0Dk2F7PqlfFhhMAwNuHp19zt2Hc9CSfiVaZeRMBIkgCBaoWJTdm6/y1obArml5qjh02IYu0ps+DMDhrP1cUkWFOZySDy/Mi1D8YhKhXjdXrv1x7Pyv6atly8ZK//cdw+/7nTkTaREO2rvye4SabHgBO+er/e/MvmoIto7NG/b/Jd4rOk97Vfwsf2gkAbMawtAU3m2dcBQzWbegVQxgDpxekcEgU1HG2PY6XAYA5IwvqWsqKxGMux65hhaersLYvQOHFIEQoThiEqEf06K4Xvvnyd4QqX3KLS7777L+BvP/5aiWqpI0wjb2rmLdwAHDEdfyRLb/2RLzlOWW/nXxf8P2/BHd8CorCmCzWeTda5lxLeF2yT+QCYOZNghQOiaH2IOxpBgUALB7OvLa7oih6zFmzuXsQKn4MQoTihEGIuooI7spP73fWbpaB+6fh8SUVP17Q4DrycRNQsM/ILL0+n7AEAHY79z/++VMhUZiZO/FBf57w9H9QSSS8zjJ3sXX+D1J77aTEMvEmENxBUVArandfjEk1x04eM14O0b+crNo04bLHY/tZK44aRahfMAhRZ7T2m7cPfvGbaLjNR7L/ZHrhfypmX7zTUVvpIQwZfnVuYUX76gfra7b+YcdzkiLPVXLu/nSvKFcCIaZLKtIX3Ymz2fqqx6mEQrceIUtg1IjvCPut4DkS8jeYrO1FXJm0TCBE8XuA0iFboAeh/sAgRO28rYf3bnrE5dgFAPu5Of9nevrl2QVFn55qrg6yemb0bUVZ46zqke8eWvXivr8pFK5sUm6vczAMZ5w2z1pxA59XkswTuGB1TCVsHziqs3KsjpGCsiTInPGMqZbXlOh2fXPpdOlTZ83mkRPbF7ciHM8YLUrIrwR9jCV9kBuPUArAIEQgS8LRXS8c3fUnRRZ9TO4/dD//xnL9uxdL1nerfe6oPoMf++/DLQUGAJADnlc3LXsrcIBQuLleWhywmiuutFy2eKgtGZFYHeVGhfavCeizdCFnOOwWLQVnBOFVhczfdJdPlz49VX06CAGATctUQn7Z34ZBiFAcMAiHusbq9fu2PBby1QNhNvC3vKH/n0sL03fkCS3v1IXDirXEVH5XMW/lpOZ677bVz9d/uiUbOAo/caVdOXOpeeZCotMn+wwueOoCFJ2nEhpzdCFnONwaVT9/xFh5MBfMo8eJ69Tnihxl2PaxSExaFjhPyV4XdsoRigMG4dAlBJz7tz7RcHwNADj4cc/rnqrjJz85gVzX0OJ4wwUUci/JGHVDvlizv/Wdld4jO/80ktuXTfSU+d/SG2ffejPejkqULgtQwOkZFJHuB88vzTtZPbZEPtTasCO3eI66Ux0vo+CceoTigkE4FFFFOrH/tW+//IMkBmUu7U3uwY/5O8dlcdtyAvIap8MnEYYUL8jMsu5rXvZrqelUkIPlo/VHzTSNM/9+3i+6F9RG/aHeIwzGLo3GZlB0GzgKAEuGk4e4ihL5UH315tNBmI4DRxGKHwbhkNPasHPvpkd8riMAUGWa/wzzVBtj/3mReGN1vW9bAADMdsaWtx+2vNcWDgKAz5b7h1JSK7bZzbnPVPzq7CvOoziY29fm7VRlrb1HKHY/ON9EhKzLofHPtVWbpsz9pbqTaV+eF3uECMUDg3AIiYa9h3csO7Hv/yhVZHPJH5knd8B3R5voh3o3u7HVJ1GWl7MM282O1YqDAoC+dIJrxmW/qF/ZHGodkV78TMUvh5lwUEzidfQIu18a7aFHCACTR08LODMs/qqgp9acUQKxOfXYI0QoLhiEQwQ9eei9g5//KiK4CMsfzLz3qfCDIugfNoWuO+4QvbIC1AJ7MiOr2UiQMacZL55tmbXouEF5dOuvvRHfpNzxT819vMeFdlH/WXRm6BKEWTxhSMQjUpmq5Qs6WzKc+wc7e5a0prF2y6hJdwEAq5YbxeIyCMUFgzD1eVq+3bf55+oEQcid85j4myORkunUu6z5KLj0IoCO1mfTDw18o37sZPO0+YYJMwnLbW/Y+atNz4SlyOzC6f87+yE9i8XSBoqp/dLo6SAkLNGlc5E2MewWjcO6/uQnZpE6SwV41hw9tkkNQgbLjSLUDxiEqayl/qtjlS84azYDUL3Ztsv2i983Xznft/eP3m+sgREU9AwVMsiGnFEBy3duME68lOiN6gPXVW9++uvnJUVeOHLeQzN+yhJcQXcAdR81CgDGHF2kTQy7ot2DEABKRl1OK5lQ45eyFGY5Q0e5UbxHiFA8MAhTEKWKo+qTY5Uvup17AIDjTcbSO16tXzB1/9e7vL8JKQslchEFmmY5UTybSZt1X+dZ2G1hz6v73/y4agMF+m8TfnD3xFt6fx2UGOb2JQnPCEJDtg6OB3u7TXhVqa1y7/hS+UBL/XZ7SQVjtBBep4RDNBomOkOPD0EI9QaDMKUoslh3dOXRXc/73ccBQG/Iys2d3+LIy92472n5uIu51gfzgYApPVK6ND993ITOjxUV6f2jH71+cEVQDPEMd/8lP1xy0cIkncfQYm4fLBPsvLO30tuqy+zkDd3lpcKBo0c320sqAIC1ZkruJtnXxuXkDXyTEUopGIQpQoz6T377zrHKF4WAEwCMuuxccUROlcQdPZoHJ7ykooGpoMBxBlJ8lT1vdhZhzhiCsdu5/0+Vr9R66wBgqv3i+y/54Yj04uScydDTfR4hxKYSunsOQp4Ba1EFHHvOWbMR4Deg3iZ0N8k+NwYhQn2FQXjBCwebqw+8XrXnFTHqBwCzZLZ5snOCGQTECGQd5a/WwzhO5oCAfXrm8KvtvPmMG351voYX9vztq4ZKAChKK7hvyr/PLLgkOWcyVBl5A0OYsBSWqRy7HXv2HiEAzB4zxXs8J104GWirtmSOVAeO4qqECMUBg/ACFnBXHfvyuVPVK2VFAgBrxJTntWWG01q5zA3WeVSZfFHYYJYpAFiHG0dem28dbjzj4dHgP799790jq0RFsujMt45besOYJTyD/yQGGwFi4o2BaDAkClZd+zqOp5fnpQA9FbNbVMz9Lzf7UnHlieObLv7OSFynHqG44V+9C4/kcjorV9Qcf7dFrKZACZBMIS0/UNiUNuOlzKkhZcLlPigLRgCAMJA51po/Jzuj7IxlchVKN9RseWnva21hL0PIghGX/2TK3ZkGXLggacy8KRANBsVQLAg5A8uZWSkoRwOSztrD76mFh2huBTSsPHps48Xf+SGjLs+LA0cR6jsMwgsAlUSxvipaezhQvafRua2ZqQvoQwDAUiZXKcnMu/Y9/vI3XGXz3KFrXJ40OQIAvJWzTcvMm52lz+C7PNu+poPP7361qq0GACbZJtw/9Z5RmSMG/6RQZ+ZuxWUAwJCtCwSFsCvaYxACwLiyy5UGFlp3SGJILTeKxWUQigMGoUbJXle09nC05lDk5BGh4YhL1+Iye/36ADVSAOCJcXjBApj44LOOUUcO+L/X5n3VX89QCgCWQqN9ZmbutEyG63pBrTnU+td9b6yv2QIAuaacey6+7cqRFYN/aqi79iCMnjFw1JitC5wSwq3RtJKea/osvij79c8vLpP3NJzcZsNyowjFC4NQMxRZbK6PVn8bqflWrDsuOk/JRHabfW6Tx2v3U0IBgCFcrn168cQfuIctemYPH1jtucF94oeRKAAQjmSPSy+Ym20d3sMfzbAUefvQB28dej8qRw2c/gdjr7t13PU6LBajGT0PHD1rxVEAKDATZ1pFWduevd9uWjTlHsAFKBCKCwZh8iiK1OqINlQLNYdC9VXRumNUjAKAQhSvIdCWG3AZ2xSQAIAQNjtvSmHZNZml161vyfzrHsF+2HWnz29QFADgM3UFs7Ps0zM5Uw/1X/zRwOaTX7x+cEWr4CZAFoy4/MeT/y3HmDXI54rOztLjVMLeF2OKKRxZAbuXBes3snMfBhw1ilBcMAgHjxL0iQ3VYmON6KgVHTWis1ZNvvbvEgjk69qswZZotayEAYAQJjtvWmHZNRkjr13vzFq1O8h+4p/mPf5DUQQASsB0kaVkTnZWubX7qEJRFnc4KtfXbP2qoVJURAAozym7f+qPynPKBu+E0Xkzd6u7DR09QqH3HiEAXDFu4u49wzKiDn+kGQiRA15QFGCYAW0tQikGg3CgUFmSnCdFR63YWCM6akRHTffLVlyWjdiLPFbBTeqaWyslUYAwEMIMK5xZULY4reR7G6utW/b4Lav8kwMt1yuK+ijJwNqmZJTMyTLm6ru+KNCDzYfW12zdeupLfzQAAAxhLsmbdM2oK79bPIv0OAwfaUBvg2UAQGiKhF1Rdbu7SdnsCsPcS4R3Kw9vKTenKwGPHPCoUykQQucJgzAxaDQstTik1gapxSE6asTGWqm5nspS52OI3sjnj+DzSog9z8/7PZE6V9PuNucWJaiuv0qy86cVli22Fl/92TeWnZ8HbCvaRoSdRerzEwhlG/MnWIZfbLUUGbvUhQGAk966zSe3ra/Z4gg41T0l6UVzi2cvLJ1nN+cO+Pmj/lEXoAideY9Qn85bCo2BemHfH0+Mub0oY7Sl58cWVsDxd+urN01ItykBj+JrwyBEqE8wCPusI/McUkuD1KpuOGSvq+txDMPlFvL5I/j8kXxeCc3O8oZqnI4drfVrPXu+oVRWjyKEzbBNLh692GBbtPObtKOb/MVOT77iygcAgCjLCPmmUVPSSi62dp8IAQAuoW3LyW1bT2072HJY3TPMlDOnaObC0nkXZY4csJ8BSrAey40CgQn3jTj2dr3rgO/bv54cvii3sGJY98fOGD+37Thn8u6i1psAQPa5eSgdlFYjlCIwCM9GCQdlV1NH2jX0mnkAhNdx2XncsAJuWD5nK+LzR/D2kqjoczv3NDp2NR/8u6f5G0rbr20ShsvMnZid/53MYdOCkanffgOuNYLd67OBzwYAAK1mHSm1XjwtbfgYU/d1WQEgIke/ati1rnrz1449MpUBwKIzzyr4zlUjK6bYJ+Il0AuOmVfvEQpd9rN6Zuy/FZ9a33xqfXPtmqZgY+SiG/MZ/oxbgHOLM57lJ48SdzWzgSxcnhehvsMgBCpGZU+L7GmVPS2Su1n2tsptLXJbs+xpUcKh7scTjudy1Mwr4HLyuZx8blg+mzEMCAEAwe9odXzdeuxvrVt2+FzHYo9iOX2mbXK2fQbPTmpqGdtWB6HDYV8oSmibHQAAIgxTl2WyjrHMmpE2O7+HG0IylWs8J79tPbqv6ZvtDTvDUgQAeIafVfidBSPmzsy/hGd76DKiC4JZXZv3zHmE7QgUX5lrzjcce6u+ZbdHaIqMvbu48+UBnoFITgU07joZbcjCGRQI9d1QCUIaEWSfW/Z7ZG+r7GmV25rltpb2/Ou9KhXRG7nMXG6YmnbtscdmtmceAIgRn7f1iK/uU+/eQ77Woz7X4WjYG3s4x5syc6dazFMjwkSP8yLhoOLZEQUABnzZAAAQJeSkSe/NNuSNT5s/wzLP2nWwn1toO+Q6dqj16LctR464q8JSuL1hQCbmjlswYu7c4ktjRbnQhaujR9jDBy9V9oS0ix8oPfR/JwP1wr5nT4y9syhtpDn23TGj50HjH0LiMSAlCs6pR6iPUiQIaTQi+92Kr00OeGWfS/F7FL9H9rnlgEfxtcl+N41GenssYTk2YxibkcNm5aobXMYwNsvGpucwpjMyRpYifvcx7+HPfK4j3tbDPtdRwe/o8mw6faY1baKOTIp4xstNI5hqqv5ts0IYAMIMc8Kgb0nXE5uxqNQ4dbThu1kkLITM5vY/aqIiHXdXH2o9esh19NuWI85gc+cnL7Tml+eUleeMnlUwzYZDYFJIj6NGuzDZ9ZMeLD36Rl3bkcDBl2pHfj8vb1b7oJirxo//1+e2TKUpxAtG7BEi1EcXcBBG6457V70ie92yz00jXW+udEF0etaaxaRlsmnZbEYO15F5bGYua82M9fDOeP6wN9TybdBT63Md8bYe8bUeDnhqY4NcVCxrNOpLWVKqhEsgWMx7i/hoJgAoADwAD1RgmON6vSPNwOQZ80oME0YZbs9lTJ1+6pRSV7htv+fbg82HD7YcOuY+EZFPzxszcoZRmSPKskZNzC2/OHc81sVOVeqo0bMHIQBwRrb8nuG1Hzc1bGk98Z4jWC+UXp9PWJLGkyZrRab3bY/Rn4E9QoT66EIOwupvIlUH1W3C6xhrBpuWxVoymLQs1prJWNLZ9GzGksFaM9i0LKI39vI0NBxsDvrqBF9DyF8f8jWE/PVBb13IXy9FA10OJYTTcSOIOJIVhuuE4VykhJHtBM64null2Rq9vtpkkHP12SXGCaWGJbmkwEwAQJTFxmDjwWZnY6DJEXA6Ak2NgabGgLPznz+GkBHpxWNzysbljBmXM3p4ejHTU0ijFGNRJ9RHzxGEAEAYMuIau6XAcHyFw7mjLdQUGXNnsc7K2UbOg71ve4w+rLuNUgGlihBUwkGq/j8cUsJBRQjRcFARgvrSCYbyaQl8tcQEoaIoR48eNZvNxcW9Lmve0NDg8XjGjBnDsj1UAouD5bIl+tKJhNcx6dmMoeeqxB3NEyPBpojgjoRahYAz5DsV6hR7itxz5Q4CJka2sWIeHy3mxBGcOJwVh5OOn5hISCPPOYw6B8+3GXmSoTPm6HPs/PAsdnY2udHgaQ41OQLOxkDTG06nGnsuwaVQ2v2FrDrLuJwx5Tmjx+WMHptTpl4lQ0PK+Vwa7WzYlAxjrv7wa6d8NaF9z54ov6t4/qTLdu7lA/pQ1NMykC1FqA+oJNJoRBH8NBqhYoSGQ0pEoNEIjQhKOESjYSpGFCFIw0FFCLUHnvplTwMVY4RvvrJrLQibmpoWLFggy7LH47nsssvefPPNLlFHKb333ntXrlyZl5cnCML69euHDx/e/9cFhuELSxVZjAquSGttJNQaCbVGBJcaeGF/S0RwR0KuaMQlRr1nexolgxVtjJLLyrmsaGMlG6PYWNHG0PYbhB6OO8VzDhPv4HVBM89lgTErkpbhyzb40nn/eMYvyX5P2OuJ+Fr9vupW31uBpkhP4coSNt8yLM9iy7PY8i32fIs9z2LLM9s4mY3dI0RDk47V8QwnKmJUjp5nMXRLoXHSg6VH/lHnPRE88OfqUTcW1BunDRe2t7GtheEgY8B/USgeihAACjQSoopMxaj6H0hRJSyAIishP6UKDYfUhKPRMJUlKgSpIitCEGRJiQhKNAxilEZCSjgEHfWw4sAYLcRgYgwmxmgmBhNjMBOjmTGYGaNZP3pyAk8ZEhKETz31VHl5+dtvvx0MBqdOnfrhhx8uXbq08wFbtmz5+OOPDx8+nJWVdd999z3xxBOvv/56/1+3/vCG3Rv+Q5I95z6UMgzNYKQ0RkmGNAL5AAAawElEQVRnlBxWtrGSjZVzWSmXkexBRu9miY+nXk726GW/WQrrhKghoOhbZUNANvlYzsMSn6L4w6LPF/GJiggegLO+bJreGou6fIstz2LPs9hs5mEs6dobppSGQufbD0ApzMSbvBFfUBTOf1UQ3sKNv7ekemVj45fuY2/VF5ROA9juMfoVXxsG4YWLRgQqywAAiqR0jH6gQqh9IrIkUrF96J8aWgCghPwAQBWFRkIA0J5e6lMpcseRVL3eCAA0HARKlWgYJIlKUSpGQFHO3gmLD2E5ojcwBjPR6QlvIEYzozcQXq8mHOH1RKdnjBaiNzJGMzGYmfbAMzHGQR0Mn4AgXLFixdtvvw0AZrP5lltuWbFiRZcgVPdkZWUBwI9+9KOZM2e+9tpr/b9AumvHKUXyAmEZOYNR0oicwcqZREljaAYrpck0PcJYRDAGwRxh9AITEbhohA2FdCGB9wmcJ6ir87OeIBeUGanX16AA3WZ2GThDuj4tw5CWoU9P11vT9Wlpemu6Pi3TkKFu55py8PIm6iszb/JGfCEx1KchUYQlpdfnmwuM1R84bKfGu+zgNQQkn4vLLRy4pp5F7E/wGTuFoNK9LgRVlHBP8ya7PZb2dEOh49uU9vIklFIq9PKtaJhKYtedYpRKXVuuCEE489XVPlCnI7qeBY2GqSQBgCzLDMOAJELH01JFpuHTw/oUIQBAOx4V6d6kpGAMZmAI0RkIy1PODLwBWJ7wOkavB4ZjDCaZ6hmdiXAc4XWE0wPHAWukwBKDCQijsJwMEgAvEV0YJCCKHFXkqJ9SJSwrUsQLMqVUVpSgFKFSIEppAMClRH2SLCtKAOSooghUCVMaoVSgiggQBJAVxQ9EBhIEKlES1ukuv+a+txJ41v0Nwkgk0tzcPHJkezWvkSNHrlmzpssxJ0+eXLRoUewAQRCam5vz8vK6P5soii0tLRs3blS/ZBhm+vTpvV02PFSSHW67L0JIRB8OM5EIGw6zkTAXCrPuCBtWoPdfnk6sOoue1ek5vYU3GzmDjtOZOKOZN+k5vYHVW3RmA2fI0KepyZeuT0vXp+EyfmggqB+eAuK5s6E7+4xMk11/5O+sR7aJbNPHHzwEtP//Snv+DaKUS8STDzhCCCgJ/jzKUKbL4Li+P0WnDoC50yeebn/kYj99AgxDz7zZRACoQplzdOAYyhLa8fmDyJTpYWg9oYSATEm40z51W6BMBEgUFAAFQAQapJQ5/Y+TMiJ0ehQlIQrxXwU9t9hPnQAAhAIJ7rz2NwiDwSCl1GAwqF+aTKZAoOtgy2AwGDvAaDQCQPdjVI2NjYcPH37qqadie37+85/PmDGjx4OvG1P+PnfCAIQhJo5hDawBAMy8iRCiY3Q6licd85QNnJ4jHEMYE29kCWvkDCbOqGf1+r5GmgyRUCQCvU5JjAOlVBAEpR9X0i9Qvf0bSG1nOWv1H3Cr15XP2+J4ZpINo35oq399ErDrgD8YfxPP+UID99Sal6zfUvnchyTsUQlAeQAeABSqB2ApUEr1QBgAkKmREgqUKMSggKwAo4AOAETCKkAk0EmMIoNOJCASohA2wihRwlKiFzigQEIMK7KSDFxx4bhFfv95NsdkMp3zAmR/gzAzM1On07ndbpvNBgAul8tut3c5xmaztbW1z21yu90A0P0YVXFx8Zw5c957773zeemxVuvjef8Vf9O1gVLKskN0sIzVak12E5Kgt7NOM1gAQOFo/D8WK2y5OMtcc+no9Ikmrn8/W0Io0/MfBwISKOEev9UbWZYTNVb8/FFKiZLgD1uUAXreHwQURWEYEt8nB4YAd7oPRGn3XihhCXuO/q7Cnu5IsayRcKeP1zHAqt9iAHiOYXuaXcYaoKNqo5klQAjDn75vRxgd4fQAwDEwzGhhdWZCGACQJElRFJ2u5z6GjuH1XNfF42I4hjNyhrOf1ADpbxASQiZPnrx9+/axY8cCwPbt26dMmdLlmClTpmzfvl3d3r59e1lZ2dD8C4jQWaiXRkPnPYOiR7WhJl9G9KHr7s82ZiaoXQng9/uH4K98KBTS6/WD/wkguURRVBRFr+817bQpAYNlHnjggYceeigvL8/hcHzwwQe7d+8GAI/HM3HixM8//7ykpOTuu+9+5plnnnvuufLy8kceeeSBBx7o/4silGJM7VMJz1Ej6SyCYsgX8Rs4fZYxI3HtQij1JSAIb775ZkmSXnrpJbPZvHbt2lGjRgEAz/Pf+9731Ct+drt906ZNy5cvX79+/X/+53/+6Ec/6v+LIpRiLP3uEdb7HQCQb7HjOlwI9UliKsvcfvvtt99+e+c9ZrP5xRdfjH05ZcqUf/7znwl5LYRSkqmPxWW6c/idAFBg7WE8NkLoLPo3FBghlCB9rbLWXUPACQAFFgxChPoGgxAhTTDr+t8jbASAfGvPQ7IRQr3BIERIEzp6hPFMqFc1+BsBe4QI9R0GIUKa0LFIffyjRtVLo9gjRKivMAgR0gSzujZvNM4eYVSOugQXS1ibeVhC24VQ6sMgREgT+jlq1BFoUii1W3K7r3CCEDo7DEKENMHCmwEgFO+lUUcAbxAiFCcMQoQ0IdYjpOe3cEoXDTiJEKF4YRAipAkcw+pZnUzliBTP8iZqjxBHyiAUBwxChLTCrIt/4Gg9zp1AKF4YhAhpRX+mEqr11bBHiFAcMAgR0oq4q6wpVGkKNhMg+RYMQoT6DIMQIa1oD8Jon4OwKdgiKlK2KUvP9rwgKkLoLDAIEdKKuHuEjoATAArxBiFCccEgREgr4g7C9pUI8QYhQnHBIERIK+IuLtO+EiH2CBGKCwYhQlph0cU5ahTLbSPUHxiECGmF2iOMo8qauhIhlpVBKD4YhAhpRdz3CBuDTQCAcycQig8GIUJaEV8QuoW2kCik6axWnWVg2oVQisMgREgr4ptHqN4gxOuiCMUNgxAhrehYpL6PQejHctsI9QsGIUJa0b5IfR9HjeJKhAj1EwYhQloR3+oTDVhuG6H+wSBESCviW31C7REW4j1ChOKFQYiQVhg5I0OIIIYV2odF6tWVCPPx0ihC8cIgREgrGEIMnIECFaTzvToaFEO+iN/A6bOMGQPaNoRSGAYhQhrS14Gj7UNGLXYCZACbhVBKwyBESEPaB45Gz/c2YQMWV0Oo3zAIEdKQjh7h+V4abS+3jcXVEOoHDEKENKSvA0ex3DZC/YdBiJCGmHV9KzfaXl8Nh4wi1A8YhAhpSF/rbmN9NYT6D4MQIQ3pUxBG5ahLcLGEzTUNG+B2IZTKMAgR0hATbwSA0PkFoSPQpFBqt+RyDDvA7UIolWEQIqQh6qjRwPmtxITlthFKCAxChDREvTR6nj1Ctdw2DhlFqJ8wCBHSkD7dI1R7hDhSBqF+wiBESEP6FIRquW28NIpQP2EQIqQhfZpH6MCVCBFKBAxChDTk/HuEClWags0ESJ7ZNvDtQiiVYRAipCHnv/pEU7BFVKRsU5aB0w98uxBKZRiECGnI+a8+4WgvrobXRRHqLwxChDTEwBlYwkbkqKhIZz8SF2BCKFEwCBHSFvU2oXCulZgcWG4boQTBIERIW9Qqa4FzrcRUj+W2EUoQDEKEtOU8i8s4cBIhQgmCQYiQtph157VIfWOwCbBHiFAiYBAipC3tA0fPemnULbSFRCFNZ7XqLIPVLoRSFgYhQtpyPlMJ1YXpsTuIUEJgECKkLe3FZc66EpMD504glDgYhAhpy/lUWWvAuRMIJQ4GIULacj6jRhtw7gRCiYNBiJC2mHgTAATOGoQda9NjECKUABiECGmLOmr0XD1CdW36/EFqE0IpDYMQIW3pmEfYaxAGxZA34jNw+ixjxiC2C6GUhUGIkLacc9Ro+w1Ci50AGbxmIZS6MAgR0pZzjhrFdScQSiwMQoS05dxBqM6mx5EyCCUIBiFC2mLWnSMIcTY9QomFQYiQtpg5NQh7rTWKs+kRSiwMQoS0hWd5nuElRY7I0R4PcOBseoQSCoMQIc2x6HotLhOVo62CiyVsrmnYoLcLodSEQYiQ5rQXl4n2cHW0MdCkUGq35HIMO+jtQig1YRAipDkd5UZ7WJu3IYAL0yOUYBiECGmOub3caA89QrW4Gt4gRCiBMAgR0pyzLECB5bYRSjgMQoQ0p6PcaE+XRrHcNkKJhkGIkOaoC1D0OJUQVyJEKOEwCBHSHDPf8wIUClWags0ESJ7Zlox2IZSaMAgR0pzeRo02h1pFRco2ZRk4fTLahVBqwiBESHNMvBF6mkfYvu4EjpRBKKEwCBHSHAtvhp5GjXbcIMRJhAglEgYhQprTXlmmWxA62sttY48QoUTCIERIc9RRo731CHEBJoQSi0vIs5w4ceKll15yuVxLliy59tprux9w8ODBysrK5ubmm266qaSkJCEvilCq6phH2EsQYn01hBIqAT1Ct9s9a9YsjuMqKiruu+++t956q/sx119//erVq5988smqqqr+vyJCqa23Reobg02AkwgRSrQE9Aj/8Y9/TJw48fe//z0AsCz79NNP33LLLV2OOXbsGAAUFRX1/+UQSnk9BqFbaAuJQprOatVZktQuhFJTAnqEX3311dy5c9XtuXPn7t+/PxjsdXFthNA5mXkTARISQxRobKe6MD12BxFKuPPqEUqS5Pf7u+9PT09nGMbpdM6bN0/dM2zYMABobGwcNWpUHK1xOBw7duy4/vrrY3sefPDBKVOmxPFUFwpKaSgUIoQkuyGDTRAElh1yK+qd/1nrOX1YCrt9bUbOoO6pdZ0EALspNxTqoRi3lg3N9zoUCsmyPNROXBRFRVFkWU52Q04zGAwMc44u33kF4ddff93jEJht27aNHj1ar9dHo1F1TyQSAQCj0djHprbLysoqKiq66aab2hvHcWPHjjUYDPE92wWBUqooSmqfY49EUcSzPgsLbwpLYZmRY8c3R1wAUJRWcMH93Ibme60oil6vH2pByLKseuLJbshp50xBOM8gvPTSS1taWnr7bmFhYV1dnbpdV1fH87zNFmchRIPBUFBQcOONN8b38AsRpZRhmPN5q1IMnvXZmXgTCO6QHI4d3xhoAoCCtLwL7uc2lN/roXbi6vlecGedgOZed911H3zwgXq55s0331y8eDHHcQCwdetWdYwMQqivLLquSxI2BByAs+kRGgAJCMJFixaNGzfukksuWbRo0WuvvfarX/1K3f/YY4+tXr1a3V6yZElpaanT6bzjjjtKS0uPHj3a/9dFKIWZui1S37E2PU4iRCjBEjB9gmXZlStX7tmzp62tbcaMGRZL+9juf/3rX2azWd1+8cUX1duHqsLCwv6/LkIprMsCFEEx5I34DJw+25iZ1HYhlIISU1mGEDJ16tQuOwsKCnrcRgidkxqEsQUo1JoyeRY7gSE3wBihgXaB3dJEaIjo6BG23yNUy20X4nVRhAYABiFCWtSxSH37pdF6dQEmHCmD0ADAIERIi7osQOHAdScQGjAYhAhpUZdRo+311bBHiNAAwCBESIssOnWR+vZLo9gjRGjgYBAipEUm3ggdPUJRFlsFF0vYXNOwZLcLoRSEQYiQFll4tUcYAgBHwKlQajMP45ihVbgSocGBQYiQFrXfI4yGoOMGYaE1P8ltQihFYRAipEWdR42qs+lxJUKEBggGIUJa1DGPUL002ghYbhuhAYNBiJAWGXkDQ5iwFJGpjOW2ERpQGIQIaREBYuQMFGhIFNT6ajh3AqEBgkGIkEapUwn90YAz0ESA5JnjXO8aIXR2GIQIaZQ6cLTGc0pUpGxTloHTJ7tFCKUmDEKENMrCmwDgmPsE4EgZhAYSBiFCGmXqFIQ4UgahgYNBiJBGqT3C423VgD1ChAYSBiFCGmXWmQGgJdQK2CNEaCBhECKkUSbOGNsutGAQIjRQMAgR0ij1HqEK66shNHAwCBHSKIuuPQitOotVZ0luYxBKYRiECGlUrEeINWUQGlAYhAhplCUWhDhkFKGBhEGIkEZ16hHiSoQIDSAMQoQ0Sl2kHnCkDEIDDIMQIY0y8e3TJ/DSKEIDCoMQIY1SV58AnE2P0ADjkt0AhFDPLDoLz3B6Tp9tzEx2WxBKZRiECGkUz3DL5z3JMzwBkuy2IJTKMAgR0q6Lc8cluwkIpT68R4gQQmhIwyBECCE0pGEQJlltbe0777yT7FYMtmAw+MILLyS7FUmwbNkySmmyWzHY/vrXv7rd7mS3YrCtWrXq0KFDyW7FYPv66683b96c7Fb0GQZhkh04cODDDz9MdisGm9PpfOWVV5LdiiR4+umnI5FIslsx2N54442amppkt2Kwffzxx5WVlcluxWD74osvNm3alOxW9BkGIUIIoSENgxAhhNCQhkGIEEJoSCOaunX/3HPPPf7447m5ucluyOARBMHn89lstmQ3ZFBJkuR0OgsLC5PdkMF28uTJ4uJiQobWBHmHw5GTk6PT6ZLdkEHV0tJiNBotlqG1orLX65VlOSsrK9kNOe2WW2558sknz36MtoJQUZTjx4/zPJ/shgweSqkoikPtbwQARCIRvV6f7FYMNjzroUMURZZlGWZoXXWTZZlSynEaKtWSl5dnNBrPfoy2ghAhhBAaZEPr0wpCCCHUBQYhQgihIQ2DECGE0JCGQYgQQmhI09DYnqHG6XSuXr069mVFRcWoUaOS2J4BVVVVtWfPHo/Hc9ttt5lMptj+HTt2vP/++1ar9a677ioqKkpiCwdCS0vL7t27T506NWfOnDFjxqg7a2pqNmzYEDvm6quvLigoSFIDB8Thw4c/+eSThoaGkpKSO+64Iz09Xd0fiUReffXVEydOTJky5dZbb02xOSStra0fffTRoUOH0tPTly5dGnu7V69e7XQ61e2srKylS5cmr42J53A4Vq1aVVVVZTKZ5s2bN3fu3Ni31q1bt27dOrvdfs8992hqNkWPsEeYNFVVVY899lh1h0AgkOwWDZTq6uoZM2b85S9/+fGPf+zxeGL7N23adNVVV9ntdrfbPW3atJaWliQ2ciBceeWVv/zlLx977LEvvvgitnPv3r2//e1vY++7IAhJbOFAWLhwYXV1dXFx8YYNGyZPntzW1qbuv+GGGz788MOysrJly5Y9/PDDyW1kwv3sZz9bu3at3W53uVxTp0797LPP1P3PPPPM+vXr1fe6vr4+uY1MuMrKyoMHDxYXF3Mcd8MNN/z5z39W9//973+/++67R4wYceDAgdmzZ0ej0eS289woSpIvvviivLw82a0YDOrUIq/XCwANDQ2x/VdcccXy5cvV7WuuueZ3v/tdcto3YNQTv+yyy1555ZXYzvfff3/u3LnJa9SAEwRB3ZBledSoUW+99Ral9MCBAxaLxe/3U0qPHTtmMpna2tqS2cpEi501pfT++++/9dZb1e3Zs2evWrUqSY0aVC+++OKsWbMopYqilJWVffDBB+r2xIkT33777WS37hywR5hMPp9v+fLlL7/8cm1tbbLbMoB6nFNMKf38888XLFigfnnFFVfEPkSnjN4mUzc1NS1btuzVV19tbGwc5CYNAoPBENsOh8NqaZXPPvts5syZ6vZFF11ks9l27dqVtCYOgB7PWvXJJ58sX7587dq1NHUnbYui+PXXX0+YMAEAGhsbjx07Nn/+fAAghMyfP1/7v9oYhEmj1+unT5/e1tb22WefTZgwYd26dclu0aByu92RSGTYsGHql7m5uSmZCt1ZLJZJkyZ5PJ61a9eOHTt2586dyW7RQPnDH/6QnZ191VVXAYDT6Yy91wCQm5vrcDiS17QBVFlZ+c477/zsZz9TvywvLzcYDM3Nzffff/8111yjKEpym5dwVVVVpaWlGRkZR44cWb58OQA0NjYajUar1aoeYLPZtP9e42CZpJk2bdp7772nbi9btuyRRx658sork9ukwaQW0pMkSf1SkqQhUmduwYIFsX7www8//MQTT6TkZ6B33333+eef37Jli/pGcxwny3Lsu6laVvD48ePf//73X3755dhgmZdfflndePTRR8vKytavX69+MkgZI0aMqKysrK+vf+SRR+6999433niD53lJkiil6ngoURS1X2APe4SaMGvWrOrq6mS3YlClpaVZLJaGhgb1y4aGhry8vOQ2afCl6vv+3nvvPfDAAxs3bhw9erS6p6CgIPZeU0odDkd+fn7yGjggqqur58+f/+tf//rmm2/u/t3MzMzy8vLUW6CYZdnMzMwJEyb89re/XbFiBaU0Pz9fFMXY2LcL4lcbgzBpOg8X/Oijj8aPH5/ExiTF4sWL3333XQCQZfmDDz5YvHhxsls0GDq/72vWrEm9933lypU/+clP1qxZU15eHtu5cOHC3bt319XVAcC2bdtkWZ4xY0by2ph4p06duuKKKx599NG77rortlMUxdg1j7q6un379o0bNy5JDRwQoVAotl1ZWVlUVEQIycnJmTVrlnq5KxQKffzxx9r/1cai20lz7733Hjx4cOTIkSdOnKitrV21atW0adOS3aiBcumll4ZCoX379k2cOJHn+Z07dzIMc+jQoblz51ZUVDgcjkgksnXr1nMWib+wPPLII5s2bTp69Gh2dnZOTs6zzz47Z86cpUuXulyuwsLCQ4cOud3udevWlZWVJbulCaMoitFozM7OjnX4fvrTn955550A8Oijj65YseLyyy9fu3btb37zm3vuuSeZDU20JUuWbNy4cezYseqXkyZNevXVV6urq2fPnj1z5kyO4zZs2HDzzTe/8MILyW1nYt12222nTp0qKSlxOBx79+598803Fy5cCABbt25dunTp1VdffeDAgeLi4pUrV2p82igGYdIEAoGvv/66ubk5Nzd3xowZZrM52S0aQHv37u08TGDKlCnqL4bb7d64cWNaWlpFRUXq3TSqrq6OzaIDAHVMgcfj2blzp8vlysvLmzlzpvZvn/QJpXTPnj2d9xQUFNjtdnV7165dVVVVkydPjt1CSxnHjx/3+XyxLy0Wi3pZ+PDhw4cPH1ZnEaTSJx5VOBzeuXNnQ0NDdnb29OnTY8UTAKChoeGLL76w2Wzf/e53tb8WFQYhQgihIU3rQY0QQggNKAxChBBCQxoGIUIIoSENgxAhhNCQhkGIEEJoSMMgRAghNKRhECKEEBrSMAgRQggNaRiECCGEhjQMQoQQQkMaBiFCCKEh7f8DczHS7XmfYrkAAAAASUVORK5CYII=", - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/html": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plot((Φ^m)[nx ÷ 2, :], label=\"Φ^m\", linewidth=2)\n", - "plot!(Φ_m[nx ÷ 2, :], label=\"euler\", linewidth=2)\n", - "plot!(Φ_sdirk[nx ÷ 2, :], label=\"SDIRK2\", linewidth=2)\n", - "plot!(Φ_θ[nx ÷ 2, :], label=\"θSDIRK2\", linewidth=2)\n", - "plot!(Φ_θ_extrap[nx ÷ 2, :], label=\"θextrap\", linewidth=2)\n", - "# plot!(Φ_both[nx ÷ 2, :], label=\"θboth\", linewidth=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "32×32 Matrix{Float64}:\n", - " 0.0651948 0.00451346 0.00572298 … -0.021391 0.0172369\n", - " 0.0172369 0.0651948 0.00451346 -0.0390968 -0.021391\n", - " -0.021391 0.0172369 0.0651948 -0.0375144 -0.0390968\n", - " -0.0390968 -0.021391 0.0172369 -0.0227642 -0.0375144\n", - " -0.0375144 -0.0390968 -0.021391 -0.00132928 -0.0227642\n", - " -0.0227642 -0.0375144 -0.0390968 … 0.0215937 -0.00132928\n", - " -0.00132928 -0.0227642 -0.0375144 0.0425005 0.0215937\n", - " 0.0215937 -0.00132928 -0.0227642 0.0594241 0.0425005\n", - " 0.0425005 0.0215937 -0.00132928 0.07158 0.0594241\n", - " 0.0594241 0.0425005 0.0215937 0.0789898 0.07158\n", - " ⋮ ⋱ ⋮ \n", - " 0.0246593 0.0294486 0.0348455 0.0168616 0.0204731\n", - " 0.0204731 0.0246593 0.0294486 0.0137817 0.0168616\n", - " 0.0168616 0.0204731 0.0246593 … 0.0111819 0.0137817\n", - " 0.0137817 0.0168616 0.0204731 0.00900795 0.0111819\n", - " 0.0111819 0.0137817 0.0168616 0.00720551 0.00900795\n", - " 0.00900795 0.0111819 0.0137817 0.00572298 0.00720551\n", - " 0.00720551 0.00900795 0.0111819 0.00451346 0.00572298\n", - " 0.00572298 0.00720551 0.00900795 … 0.0651948 0.00451346\n", - " 0.00451346 0.00572298 0.00720551 0.0172369 0.0651948" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Φ_both" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.9.0", - "language": "julia", - "name": "julia-1.9" - }, - "language_info": { - "file_extension": ".jl", - "mimetype": "application/julia", - "name": "julia", - "version": "1.9.0" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} From dd83362c19a5ab55557334dd875341c900285397 Mon Sep 17 00:00:00 2001 From: Rob Falgout Date: Thu, 1 Jun 2023 07:10:34 -0700 Subject: [PATCH 07/12] Updates to makefiles, Julia example README, and Fortran interface --- Makefile | 6 +++--- braid/Makefile | 10 ++++++++-- braid/braid.h | 13 ++++++++----- braid/braid_F90_iface.c | 5 ++++- examples/julia/README.md | 8 ++++++-- examples/julia/ex-01.jl | 2 +- 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 4bca0295..d1e469fe 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,9 @@ all: braid examples braid: cd braid; $(MAKE) -ifeq ($(shared),yes) - cd braid; $(MAKE) libbraid.so -endif +# ifeq ($(shared),yes) +# cd braid; $(MAKE) libbraid.so +# endif examples: ./braid/libbraid.a cd examples; $(MAKE) diff --git a/braid/Makefile b/braid/Makefile index df720c9f..67f342e2 100644 --- a/braid/Makefile +++ b/braid/Makefile @@ -74,6 +74,12 @@ else SEQFLAGS = endif +ifeq ($(shared),yes) + BRAID_LIBFILES = libbraid.a libbraid.so +else + BRAID_LIBFILES = libbraid.a +endif + BRAID_OBJ = $(BRAID_FILES:.c=.o) .PHONY: all clean @@ -84,6 +90,8 @@ BRAID_OBJ = $(BRAID_FILES:.c=.o) %.o: %.c $(BRAID_HEADERS) $(MPICC) $(SEQFLAGS) $(CFLAGS) -c $< -o $@ +all: $(BRAID_LIBFILES) + libbraid.a: $(BRAID_HEADERS) $(BRAID_OBJ) @echo "Building" $@ "..." ar cruv libbraid.a $(BRAID_OBJ) @@ -93,8 +101,6 @@ libbraid.so: $(BRAID_HEADERS) $(BRAID_OBJ) @echo "Building" $@ "..." $(MPICC) $(SEQFLAGS) $(CFLAGS) $(SHAREDFLAGS) $(BRAID_OBJ) -o $@ -all: libbraid.a - clean: rm -f *.o libbraid.a libbraid.so diff --git a/braid/braid.h b/braid/braid.h index fb378ffd..e4f12483 100644 --- a/braid/braid.h +++ b/braid/braid.h @@ -57,14 +57,17 @@ extern "C" { */ /** Define Fortran name-mangling schema, there are four supported options, see braid_F90_iface.c */ -#define braid_FMANGLE 0 -/** Turn on the optional user-defined spatial coarsening and refinement functions */ +#define braid_FMANGLE 1 +/** Turn on/off the Fortran interface (useful for avoiding undefined symbol + * errors in shared library builds) */ +#define braid_Fortran_Iface 0 +/** Turn on/off the optional user-defined spatial coarsening and refinement functions */ #define braid_Fortran_SpatialCoarsen 0 -/** Turn on the optional user-defined residual function */ +/** Turn on/off the optional user-defined residual function */ #define braid_Fortran_Residual 0 -/** Turn on the optional user-defined time-grid function */ +/** Turn on/off the optional user-defined time-grid function */ #define braid_Fortran_TimeGrid 0 -/** Turn on the optional user-defined sync function */ +/** Turn on/off the optional user-defined sync function */ #define braid_Fortran_Sync 0 /** @} */ diff --git a/braid/braid_F90_iface.c b/braid/braid_F90_iface.c index 8622a94b..85e88b70 100644 --- a/braid/braid_F90_iface.c +++ b/braid/braid_F90_iface.c @@ -18,7 +18,9 @@ * Temple Place, Suite 330, Boston, MA 02111-1307 USA * ***********************************************************************EHEADER*/ - + +/* This interface is optionally available */ +#if (braid_Fortran_Iface) /** \file _braid_F90_iface.c * \brief Define F90 interface for user routines. @@ -1480,3 +1482,4 @@ braid_F90_Name(braid_set_sync_f90, BRAID_SET_SYNC_F90)( #endif +#endif diff --git a/examples/julia/README.md b/examples/julia/README.md index 97fdc1a9..ccf7b16d 100644 --- a/examples/julia/README.md +++ b/examples/julia/README.md @@ -27,9 +27,13 @@ To run examples in this directory: 2. install and configure MPI.jl: https://juliaparallel.org/MPI.jl/v0.20/ - julia> using Pkg; Pkg.add(["MPI, MPIPreferences]") + julia> using Pkg; Pkg.add(["MPI", "MPIPreferences"]) julia> using MPIPreferences; MPIPreferences.use_system_binary() + NOTE: It might be necessary to give the path to the system MPI library, e.g.: + + julia> MPIPreferences.use_system_binary(; library_names="/opt/homebrew/lib/libmpi") + 3. run from the command line: - shell> mpirun -n 4 ex-01.jl \ No newline at end of file + shell> mpirun -n 4 julia ex-01.jl diff --git a/examples/julia/ex-01.jl b/examples/julia/ex-01.jl index 808fb227..a2b7ccc4 100644 --- a/examples/julia/ex-01.jl +++ b/examples/julia/ex-01.jl @@ -87,4 +87,4 @@ XBraid.SetCFactor(core, -1, 2) XBraid.Drive(core) # no need for braid_Destroy(core), julia will take care of it :) -MPI.Finalize() \ No newline at end of file +MPI.Finalize() From 8bba2f35809e89b65dc5ca2fe8b1876c758058d8 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Thu, 1 Jun 2023 11:29:15 -0600 Subject: [PATCH 08/12] Added makefile option to turn on/off fortran interface - this currently just removes braid_F90_iface.c from the list of braid files/linked objects --- Makefile | 3 --- braid/Makefile | 5 ++++- makefile.inc | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d1e469fe..12f13736 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,6 @@ all: braid examples braid: cd braid; $(MAKE) -# ifeq ($(shared),yes) -# cd braid; $(MAKE) libbraid.so -# endif examples: ./braid/libbraid.a cd examples; $(MAKE) diff --git a/braid/Makefile b/braid/Makefile index 67f342e2..a86b5db6 100644 --- a/braid/Makefile +++ b/braid/Makefile @@ -45,7 +45,6 @@ BRAID_FILES =\ base.c\ braid.c\ braid_status.c\ - braid_F90_iface.c\ braid_test.c\ communication.c\ delta.c\ @@ -66,6 +65,10 @@ BRAID_FILES =\ util.c\ uvector.c +ifeq ($(fortran),yes) + BRAID_FILES += braid_F90_iface.c +endif + ifeq ($(sequential),yes) BRAID_HEADERS += mpistubs.h BRAID_FILES += mpistubs.c diff --git a/makefile.inc b/makefile.inc index e9dad806..dd783eff 100644 --- a/makefile.inc +++ b/makefile.inc @@ -22,11 +22,12 @@ # #EHEADER********************************************************************** -# four compile time options +# five compile time options # make debug=yes|no # make valgrind=yes|no # make sequential=yes|no # make shared=yes|no +# make fortran=yes|no # Was DEBUG specified? ifeq ($(debug),no) From c03e47ad0dbee2d8f8b3e684bfc690d5e3f7856b Mon Sep 17 00:00:00 2001 From: David Vargas Date: Thu, 1 Jun 2023 11:39:37 -0600 Subject: [PATCH 09/12] Removed -undefined dynamic_lookup from SHAREDFLAGS --- makefile.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile.inc b/makefile.inc index dd783eff..fbeff554 100644 --- a/makefile.inc +++ b/makefile.inc @@ -83,7 +83,7 @@ else ifeq ($(shell uname -s), Darwin) MPICXX = mpicxx MPIF90 = mpif90 LFLAGS = -lm -lstdc++ - SHAREDFLAGS = -shared -undefined dynamic_lookup + SHAREDFLAGS = -shared ifeq ($(optlevel),DEBUG) CFLAGS = -g -Wall -fPIC CXXFLAGS = -g -Wall -fPIC From 4fd15b850b85370cce087f7955c0d3e13eb49e82 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 7 Jun 2023 17:20:57 -0600 Subject: [PATCH 10/12] Renamed examples/julia/ to examples/ex-01-julia/ Added .dSYM file extension to .gitignore --- .gitignore | 1 + examples/{julia => ex-01-julia}/README.md | 0 examples/{julia => ex-01-julia}/ex-01.jl | 0 3 files changed, 1 insertion(+) rename examples/{julia => ex-01-julia}/README.md (100%) rename examples/{julia => ex-01-julia}/ex-01.jl (100%) diff --git a/.gitignore b/.gitignore index df34340c..e3fa7c50 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ wrapperTests.txt *.o *.so *.a +*.dSYM ex-01 ex-01-adjoint ex-01-expanded diff --git a/examples/julia/README.md b/examples/ex-01-julia/README.md similarity index 100% rename from examples/julia/README.md rename to examples/ex-01-julia/README.md diff --git a/examples/julia/ex-01.jl b/examples/ex-01-julia/ex-01.jl similarity index 100% rename from examples/julia/ex-01.jl rename to examples/ex-01-julia/ex-01.jl From 197026bffbb3d9e19d53cfe6eaa39afd63d5dfc5 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 7 Jun 2023 17:52:49 -0600 Subject: [PATCH 11/12] Added entry to examples readme --- examples/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/README.md b/examples/README.md index 66242155..acb2b0b2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -80,6 +80,14 @@ For the Cython examples, see the corresponding *.pyx file. examples/ex-01-cython-alt/ex_01_alt-setup.py + + *ex-01-julia/*: is a directory containing an example using the Braid-Julia + interface defined in braid.jl ( braid/braid.jl ). It solves the same scalar + ODE equation as the ex-01 series described above. + + For instructions on running see + + examples/ex-01-julia/README.md + 2. ex-02 implements the 1D heat equation on a regular grid, using a very simple implementation. This is the next example to read after the various ex-01 cases. From 407744f2c371e6c89d319afc066d3fccf0440dc0 Mon Sep 17 00:00:00 2001 From: David Vargas Date: Wed, 7 Jun 2023 21:03:17 -0600 Subject: [PATCH 12/12] Added kwargs to XBraid.Init This prevents having to make calls to XBraid.set functions to set common parameters like max_iters, max_levels, etc. --- braid/braid.jl/XBraid.jl | 165 +++++++++++++++++++++++++++++++--- examples/ex-01-julia/ex-01.jl | 11 +-- 2 files changed, 156 insertions(+), 20 deletions(-) diff --git a/braid/braid.jl/XBraid.jl b/braid/braid.jl/XBraid.jl index bb8c5b5e..aff02192 100644 --- a/braid/braid.jl/XBraid.jl +++ b/braid/braid.jl/XBraid.jl @@ -101,7 +101,6 @@ function BraidApp(app, comm::MPI.Comm, step, init, access; comm_t=comm, sum=defa end - """ Stores all the information needed to run XBraid. Create this object with Init(), then pass this to Drive() to run XBraid. @@ -134,10 +133,10 @@ using .Status, .Wrappers function postInitPrecompile(app::BraidApp) #= - # This is a hack to make sure every processor knows how big the user's vector will be. - # We can also take this time to precompile the user's step function so it doesn't happen - # in serial on the coarse grid. - =# + # This is a hack to make sure every processor knows how big the user's vector will be. + # We can also take this time to precompile the user's step function so it doesn't happen + # in serial on the coarse grid. + =# GC.enable(false) # disable garbage collection app_ptr = pointer_from_objref(app)::Ptr{Cvoid} pp = malloc_null_double_ptr(Cvoid) @@ -198,6 +197,8 @@ end """ Create a new BraidCore object. The BraidCore object is used to run the XBraid solver, and is destroyed when the object is garbage collected. + + """ function Init( comm_world::MPI.Comm, @@ -207,11 +208,12 @@ function Init( step::Function, init::Function, access::Function; - sum = default_sum!, - spatialnorm = default_norm, - sync = nothing, - comm_t::MPI.Comm = comm_world, - app = nothing + sum=default_sum!, + spatialnorm=default_norm, + sync=nothing, + comm_t::MPI.Comm=comm_world, + app=nothing, + kwargs... )::BraidCore _app = BraidApp(app, comm_world, comm_t, step, init, sum, spatialnorm, access, sync, nothing, nothing) _core_ptr = malloc_null_double_ptr(Cvoid) @@ -238,7 +240,146 @@ function Init( end end - return BraidCore(_core, _app, tstart, tstop, ntime) + core = BraidCore(_core, _app, tstart, tstop, ntime) + + # parse kwargs + if haskey(kwargs, :timer_file) + timer_file = kwargs[:timer_file]::String + setTimerFile(core, timer_file) + end + if haskey(kwargs, :timing_level) + timing_level = kwargs[:timing_level]::Integer + setTimingLevel(core, timing_level) + end + if haskey(kwargs, :write_conv_hist) + write_conv_hist = kwargs[:write_conv_hist]::String + setWriteConvHistory(core, write_conv_hist) + end + if haskey(kwargs, :max_levels) + max_levels = kwargs[:max_levels]::Integer + setMaxLevels(core, max_levels) + end + if haskey(kwargs, :incr_max_levels) + incr_max_levels = kwargs[:incr_max_levels]::Bool + incr_max_levels && setIncrMaxLevels(core) + end + if haskey(kwargs, :skip) + skip = kwargs[:skip]::Bool + setSkip(core, skip) + end + if haskey(kwargs, :refine) + refine = kwargs[:refine]::Bool + setRefine(core, refine) + end + if haskey(kwargs, :max_refinements) + max_refinements = kwargs[:max_refinements]::Integer + setMaxRefinements(core, max_refinements) + end + if haskey(kwargs, :tpoints_cutoff) + tpoints_cutoff = kwargs[:tpoints_cutoff]::Integer + setTPointsCutoff(core, tpoints_cutoff) + end + if haskey(kwargs, :min_coarse) + min_coarse = kwargs[:min_coarse]::Integer + setMinCoarse(core, min_coarse) + end + if haskey(kwargs, :relax_only_cg) + relax_only_cg = kwargs[:relax_only_cg]::Bool + setRelaxOnlyCG(core, relax_only_cg) + end + if haskey(kwargs, :abs_tol) + abs_tol = kwargs[:abs_tol]::Float64 + setAbsTol(core, abs_tol) + end + if haskey(kwargs, :rel_tol) + rel_tol = kwargs[:rel_tol]::Float64 + setRelTol(core, rel_tol) + end + if haskey(kwargs, :n_relax_fine) + n_relax = kwargs[:n_relax]::Integer + setNRelax(core, 0, n_relax) + end + if haskey(kwargs, :cfactor) + cfactor = kwargs[:cfactor] + if isa(cfactor, Integer) + setCFactor(core, -1, cfactor) + elseif isa(cfactor, Vector{Integer}) + for (i, cf) in enumerate(cfactor) + setCFactor(core, i-1, cf) + end + else + error("cfactor must be an Integer or Vector{Integer}") + end + end + if haskey(kwargs, :cfactor_fine) + cfactor_fine = kwargs[:cfactor_fine]::Integer + setCFactorFine(core, 0, cfactor_fine) + end + if haskey(kwargs, :max_iter) + max_iter = kwargs[:max_iter]::Integer + setMaxIter(core, max_iter) + end + if haskey(kwargs, :fmg) + fmg = kwargs[:fmg]::Bool + fmg && setFMG(core) + end + if haskey(kwargs, :n_fmg_vcycles) + fmg_cycle = kwargs[:fmg_cycle]::Integer + setNFMGVcyc(core, fmg_cycle) + end + if haskey(kwargs, :storage) + storage = kwargs[:storage]::Integer + setStorage(core, storage) + end + if haskey(kwargs, :temporal_norm) + norm = kwargs[:temporal_norm]::Symbol + if norm == :One + setTemporalNorm(core, 1) + elseif norm == :Two + setTemporalNorm(core, 2) + elseif norm == :Inf + setTemporalNorm(core, 3) + else + error("temporal_norm must be one of :One, :Two, or :Inf") + end + end + if haskey(kwargs, :periodic) + periodic = kwargs[:periodic]::Bool + setPeriodic(core, periodic) + end + if haskey(kwargs, :print_level) + print_level = kwargs[:print_level]::Integer + print_level < 0 && error("print_level must be non-negative") + setPrintLevel(core, print_level) + end + if haskey(kwargs, :file_io_level) + file_io_level = kwargs[:file_io_level]::Integer + file_io_level < 0 && error("file_io_level must be non-negative") + setFileIOLevel(core, file_io_level) + end + if haskey(kwargs, :save_cycle) + save_cycle = kwargs[:save_cycle]::Bool + setFileIOLevel(core, save_cycle ? 1 : 0) + end + if haskey(kwargs, :print_file) + print_file = kwargs[:print_file]::String + setPrintFile(core, print_file) + end + if haskey(kwargs, :access_level) + access_level = kwargs[:access_level]::Integer + access_level < 0 && error("access_level must be non-negative") + setAccessLevel(core, access_level) + end + if haskey(kwargs, :final_fc_relax) + final_fc_relax = kwargs[:final_fc_relax]::Bool + final_fc_relax && setFinalFCRelax(core) + end + if haskey(kwargs, :seq_soln) + seq_soln = kwargs[:seq_soln]::Bool + setSeqSoln(core, seq_soln) + end + + return core end """ @@ -307,6 +448,7 @@ function Drive(core::BraidCore; warmup=true) end end _Drive(core) + return nothing end function printStats(core::BraidCore) @@ -383,7 +525,6 @@ function setTPointsCutoff(core::BraidCore, tpoints_cutoff::Integer) end end - function setMinCoarse(core::BraidCore, min_coarse::Integer) GC.@preserve core begin @ccall libbraid.braid_SetMinCoarse(core._braid_core::Ptr{Cvoid}, min_coarse::Cint)::Cint diff --git a/examples/ex-01-julia/ex-01.jl b/examples/ex-01-julia/ex-01.jl index 7f7a74df..f53a5489 100644 --- a/examples/ex-01-julia/ex-01.jl +++ b/examples/ex-01-julia/ex-01.jl @@ -48,7 +48,7 @@ called by the solver to give access to solution values. =# function my_access(app, status, u) - XBraid.Status.getWrapperTest(status) && throw("Test error") + XBraid.Status.getWrapperTest(status) && return t = XBraid.Status.getT(status) ti = XBraid.Status.getTIndex(status) print("t: $(t[]),\tu: $(u[])\n") @@ -77,14 +77,9 @@ end ntime = 10 tstart = 0.0 tstop = tstart + ntime / 2.0; -core = XBraid.Init(comm, tstart, tstop, ntime, my_step!, my_init, my_access) - -XBraid.setPrintLevel(core, 2) -XBraid.setMaxLevels(core, 2) -XBraid.setAbsTol(core, 1.e-6) -XBraid.setCFactor(core, -1, 2) +core = XBraid.Init(comm, tstart, tstop, ntime, my_step!, my_init, my_access; + print_level=2, max_levels=2, abs_tol=1.e-6, c_factor=2) XBraid.Drive(core) # no need for braid_Destroy(core), julia will take care of it :) -MPI.Finalize()