Merge branch 'feature/subcycle' of github.com:lwJi/CarpetX into featu…

…re/subcycle

TestSubcyclingMC2/README

@@ -0,0 +1,9 @@
+Cactus Code Thorn TestSubcyclingMC2
+Author(s)    : Liwei Ji <[email protected]>
+Maintainer(s): Liwei Ji <[email protected]>
+Licence      : LGPL
+--------------------------------------------------------------------------
+
+1. Purpose
+
+Solve the scalar wave equation using staggered grids.

TestSubcyclingMC2/configuration.ccl

@@ -0,0 +1,3 @@
+# Configuration definitions for thorn TestSubcyclingMC2
+
+REQUIRES Loop Subcycling

TestSubcyclingMC2/interface.ccl

@@ -0,0 +1,71 @@
+# Interface definition for thorn TestSubcyclingMC2
+
+IMPLEMENTS: TestSubcyclingMC2
+
+INHERITS: CarpetX Subcycling
+
+USES INCLUDE HEADER: loop_device.hxx
+USES INCLUDE HEADER: subcycling.hxx
+USES INCLUDE HEADER: vect.hxx
+
+
+
+CCTK_REAL ustate TYPE=gf CENTERING={vvv} TAGS='dependents="uerror"'
+{
+  u
+  rho
+} "Scalar wave state vector"
+
+CCTK_REAL pstate TYPE=gf TAGS='checkpoint="no"'
+{
+  u_p
+  rho_p
+} "Scalar wave old state vector"
+
+CCTK_REAL wstate TYPE=gf TAGS='checkpoint="no"'
+{
+  u_w
+  rho_w
+} "Ys of scalar wave state vector"
+
+CCTK_REAL k1 TYPE=gf TAGS='checkpoint="no"'
+{
+    u_k1
+    rho_k1
+} "The Runge-Kutta k_1 variables"
+
+CCTK_REAL k2 TYPE=gf TAGS='checkpoint="no"'
+{
+    u_k2
+    rho_k2
+} "The Runge-Kutta k_2 variables"
+
+CCTK_REAL k3 TYPE=gf TAGS='checkpoint="no"'
+{
+    u_k3
+    rho_k3
+} "The Runge-Kutta k_3 variables"
+
+CCTK_REAL k4 TYPE=gf TAGS='checkpoint="no"'
+{
+    u_k4
+    rho_k4
+} "The Runge-Kutta k_4 variables"
+
+
+
+CCTK_REAL energy CENTERING={CCC} TYPE=gf TAGS='checkpoint="no"'
+{
+  eps
+} "Energy density"
+
+
+
+CCTK_REAL error TYPE=gf TAGS='checkpoint="no"'
+{
+  u_err
+  rho_err
+} "Error in scalar wave state vector"
+
+
+

TestSubcyclingMC2/par/gaussian.par

@@ -0,0 +1,93 @@
+ActiveThorns = "
+    CarpetX
+    BoxInBox
+    IOUtil
+    TestSubcyclingMC2
+    TimerReport
+"
+
+$out_every = 32 #16
+$nlevels = 2
+$ncells = 64
+
+TestSubcyclingMC2::initial_condition = "Gaussian"
+TestSubcyclingMC2::gaussian_width = 0.17677669529 # sqrt(2)*W = 0.25
+
+CarpetX::use_subcycling_wip = yes
+CarpetX::restrict_during_sync = no
+
+CarpetX::poison_undefined_values = no
+CarpetX::verbose = no
+
+Cactus::cctk_show_schedule = yes
+Cactus::presync_mode = "mixed-error"
+
+CarpetX::xmin = -4.0
+CarpetX::ymin = -4.0
+CarpetX::zmin = -4.0
+
+CarpetX::xmax = +4.0
+CarpetX::ymax = +4.0
+CarpetX::zmax = +4.0
+
+CarpetX::ncells_x = $ncells
+CarpetX::ncells_y = $ncells
+CarpetX::ncells_z = $ncells
+
+CarpetX::periodic_x = yes
+CarpetX::periodic_y = yes
+CarpetX::periodic_z = yes
+
+CarpetX::max_num_levels = $nlevels
+CarpetX::regrid_every = 128
+CarpetX::regrid_error_threshold = 0.9
+
+CarpetX::prolongation_type = "ddf"
+#CarpetX::prolongation_order = 3
+CarpetX::ghost_size = 3
+CarpetX::dtfac = 0.25
+
+CarpetX::blocking_factor_x = 2
+CarpetX::blocking_factor_y = 2
+CarpetX::blocking_factor_z = 2
+CarpetX::max_grid_size_x = 10000000
+CarpetX::max_grid_size_y = 10000000
+CarpetX::max_grid_size_z = 10000000
+CarpetX::max_tile_size_x = 10000000
+CarpetX::max_tile_size_y = 10000000
+CarpetX::max_tile_size_z = 10000000
+
+BoxInBox::num_regions = 1
+# Region 1
+BoxInBox::shape_1 = "cube"
+BoxInBox::num_levels_1 = $nlevels
+BoxInBox::radius_1 = [ -1.0, 1.0, 0.5 ]
+
+
+Cactus::terminate = "time"
+Cactus::cctk_final_time = 2.0
+#Cactus::cctk_itlast = $itlast
+
+IO::out_dir = $parfile
+IO::out_every = $out_every
+
+#CarpetX::out_norm_vars = "all"
+
+CarpetX::out_tsv_vars = "
+    TestSubcyclingMC2::ustate
+    #TestSubcyclingMC2::energy
+    TestSubcyclingMC2::error
+"
+
+CarpetX::out_silo_vars = "
+    TestSubcyclingMC2::ustate
+    #TestSubcyclingMC2::energy
+    TestSubcyclingMC2::error
+"
+
+TimerReport::out_every = 128 # $out_every
+TimerReport::out_filename = "TimerReport"
+TimerReport::output_all_timers_together = yes
+TimerReport::output_all_timers_readable = yes
+#TimerReport::output_schedule_timers = no
+TimerReport::n_top_timers = 50

TestSubcyclingMC2/par/standing.par

@@ -0,0 +1,54 @@
+ActiveThorns = "
+    CarpetX
+    IOUtil
+    TestSubcyclingMC2
+    TimerReport
+"
+
+$out_every = 16
+
+Cactus::cctk_show_schedule = no
+Cactus::presync_mode = "mixed-error"
+
+CarpetX::poison_undefined_values = no
+
+CarpetX::periodic_x = yes
+CarpetX::periodic_y = yes
+CarpetX::periodic_z = yes
+
+Cactus::terminate = "time"
+Cactus::cctk_final_time = 1.0
+
+TestSubcyclingMC2::initial_condition = "standing wave"
+
+IO::out_dir = $parfile
+IO::out_every = $out_every
+
+CarpetX::out_norm_vars = "all"
+
+CarpetX::out_tsv_vars = "
+    TestSubcyclingMC2::ustate
+    TestSubcyclingMC2::ftstate
+    TestSubcyclingMC2::fxstate
+    TestSubcyclingMC2::fystate
+    TestSubcyclingMC2::fzstate
+    TestSubcyclingMC2::energy
+    TestSubcyclingMC2::uerror
+    TestSubcyclingMC2::fterror
+    TestSubcyclingMC2::fxerror
+    TestSubcyclingMC2::fyerror
+    TestSubcyclingMC2::fzerror
+"
+
+CarpetX::out_silo_vars = "
+    TestSubcyclingMC2::ustate
+    TestSubcyclingMC2::ftstate
+    TestSubcyclingMC2::energy
+    TestSubcyclingMC2::uerror
+    TestSubcyclingMC2::fterror
+"
+
+TimerReport::out_every = $out_every
+TimerReport::out_filename = "TimerReport"
+TimerReport::output_schedule_timers = no
+TimerReport::n_top_timers = 100

TestSubcyclingMC2/param.ccl

@@ -0,0 +1,37 @@
+# Parameter definitions for thorn TestSubcyclingMC2
+
+KEYWORD initial_condition "Initial condition"
+{
+  "standing wave" :: "Standing wave"
+  "Gaussian" :: "Gaussian"
+} "standing wave"
+
+CCTK_REAL amplitude "Initial amplitude"
+{
+  *:* :: ""
+} 1.0
+
+CCTK_REAL standing_wave_kx "kx for standing wave"
+{
+  *:* :: ""
+} 0.5
+
+CCTK_REAL standing_wave_ky "kz for standing wave"
+{
+  *:* :: ""
+} 0.5
+
+CCTK_REAL standing_wave_kz "ky for standing wave"
+{
+  *:* :: ""
+} 0.5
+
+CCTK_REAL gaussian_width "width of Gaussian"
+{
+  (0:* :: ""
+} 1.0
+
+
+SHARES: CarpetX
+
+USES BOOLEAN use_subcycling_wip

TestSubcyclingMC2/schedule.ccl

@@ -0,0 +1,107 @@
+# Schedule definitions for thorn TestSubcyclingMC2
+
+STORAGE: ustate pstate wstate k1 k2 k3 k4
+
+
+SCHEDULE TestSubcyclingMC2_Initial AT initial
+{
+  LANG: C
+  WRITES: ustate(interior)
+  WRITES: k1(interior) k2(interior) k3(interior) k4(interior)
+  SYNC: ustate
+  SYNC: k1 k2 k3 k4
+} "Initialize scalar wave state"
+
+SCHEDULE TestSubcyclingMC2_Sync AT postregrid
+{
+  LANG: C
+  OPTIONS: global
+  SYNC: ustate
+} "Synchronize"
+
+
+
+SCHEDULE GROUP TestSubcyclingMC2_RK4Group AT evol
+{
+} "RK4"
+
+SCHEDULE TestSubcyclingMC2_SetP IN TestSubcyclingMC2_RK4Group
+{
+  LANG: C
+  READS: ustate(interior)
+  WRITES: pstate(interior)
+  SYNC: pstate k1 k2 k3 k4
+} "Save previous ustate and Sync ks"
+
+SCHEDULE TestSubcyclingMC2_CalcK1 IN TestSubcyclingMC2_RK4Group AFTER TestSubcyclingMC2_SetP
+{
+  LANG: C
+  READS: pstate(everywhere)
+  READS: k1(everywhere) k2(everywhere) k3(everywhere) k4(everywhere)
+  # calculate k1 and update ustate
+  READS: ustate(everywhere)
+  WRITES: k1(interior) ustate(everywhere)
+  WRITES: k2(interior) k3(interior) k4(interior) #fake
+  # calculate Y2
+  WRITES: wstate(interior)
+  SYNC: wstate
+} "Calculate k1 and Y2"
+
+SCHEDULE TestSubcyclingMC2_CalcK2 IN TestSubcyclingMC2_RK4Group AFTER TestSubcyclingMC2_CalcK1
+{
+  LANG: C
+  READS: pstate(everywhere)
+  READS: k1(everywhere) k2(everywhere) k3(everywhere) k4(everywhere)
+  # calculate k2 and update ustate
+  READS: wstate(everywhere) ustate(interior)
+  WRITES: k2(interior) ustate(interior)
+  WRITES: k1(interior) k3(interior) k4(interior) #fake
+  # calculate Y3
+  WRITES: wstate(everywhere)
+  SYNC: wstate
+} "Calculate k2 and Y3"
+
+SCHEDULE TestSubcyclingMC2_CalcK3 IN TestSubcyclingMC2_RK4Group AFTER TestSubcyclingMC2_CalcK2
+{
+  LANG: C
+  READS: pstate(everywhere)
+  READS: k1(everywhere) k2(everywhere) k3(everywhere) k4(everywhere)
+  # calculate k3 and update ustate
+  READS: wstate(everywhere) ustate(interior)
+  WRITES: k3(interior) ustate(interior)
+  WRITES: k1(interior) k2(interior) k4(interior) #fake
+  # calculate Y4
+  WRITES: wstate(everywhere)
+  SYNC: wstate
+} "Calculate k3 and Y4"
+
+SCHEDULE TestSubcyclingMC2_CalcK4 IN TestSubcyclingMC2_RK4Group AFTER TestSubcyclingMC2_CalcK3
+{
+  LANG: C
+  READS: pstate(everywhere)
+  READS: k1(everywhere) k2(everywhere) k3(everywhere) k4(everywhere)
+  WRITES: wstate(everywhere) # write ghost points only here
+  # calculate k4 and update ustate
+  READS: wstate(everywhere) ustate(interior)
+  WRITES: k4(interior) ustate(interior)
+  WRITES: k1(interior) k2(interior) k3(interior) #fake
+  SYNC: ustate
+} "Calculate k4"
+
+
+
+SCHEDULE GROUP TestSubcyclingMC2_PostStepGroup AT postinitial
+{
+} "Apply boundary conditions to state vector, and project if necessary"
+
+SCHEDULE GROUP TestSubcyclingMC2_PostStepGroup AT poststep
+{
+} "Apply boundary conditions to state vector, and project if necessary"
+
+SCHEDULE TestSubcyclingMC2_Error IN TestSubcyclingMC2_PostStepGroup
+{
+  LANG: C
+  READS: ustate(interior)
+  WRITES: error(interior)
+  SYNC: error
+} "Calculate error in scalar wave state"

TestSubcyclingMC2/src/make.code.defn

@@ -0,0 +1,7 @@
+# Main make.code.defn file for thorn TestSubcyclingMC
+
+# Source files in this directory
+SRCS = testsubcyclingmc.cxx
+
+# Subdirectories containing source files
+SUBDIRS =