From 3787f7069332a6589ff9702ab7e017f3c390afa2 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 12:00:40 +1100 Subject: [PATCH 01/28] Lay down some initial structure for memory functions --- src/py21cmfast/_memory.py | 92 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 src/py21cmfast/_memory.py diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py new file mode 100644 index 000000000..22b154059 --- /dev/null +++ b/src/py21cmfast/_memory.py @@ -0,0 +1,92 @@ +"""Function to estimate total memory usage.""" + +import numpy as np + +from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params + + +def estimate_memory_coeval( + *, + user_params=None, + cosmo_params=None, + astro_params=None, + flag_options=None, +): + """Compute an estimate of the requisite memory needed by the user for a run_coeval call.""" + """ + # Initial conditions + mem_initial_conditions(user_params=self.user_params) + + # Perturb field + + # Halo field + if flag_options.USE_HALO_FIELD: + + # Photon non-conservation + if flag_options.PHOTON_CONS: + + # Note, if any of below are set the require current and + # previous boxes, thus need to be careful + + # Spin temperature + if flag_options.USE_TS_FLUCT: + + # Mini-halos + if flag_options.USE_MINI_HALOS: + + # Inhomogeneous recombinations + if flag_options.INHOMO_RECO: + + + # Output a summary of information in human readable format + """ + return {} + + +def estimate_memory_lightcone( + *, + user_params=None, + cosmo_params=None, + astro_params=None, + flag_options=None, +): + """Compute an estimate of the requisite memory needed by the user for a run_lightcone call.""" + return {} + + +def mem_initial_conditions( + *, + user_params=None, +): + """Memory usage of Python initial conditions class.""" + # lowres_density, lowres_vx, lowres_vy, lowres_vz, lowres_vcb, + # lowres_vx_2LPT, lowres_vy_2LPT, lowres_vz_2LPT + num_py_boxes_HII_DIM = 8.0 + + # hires_density, hires_vx, hires_vy, hires_vz, hires_vcb, + # hires_vx_2LPT, hires_vy_2LPT, hires_vz_2LPT + num_py_boxes_DIM = 8.0 + + size_py = num_py_boxes_DIM * (user_params.DIM) ** 3 + size_py += num_py_boxes_HII_DIM * (user_params.HII_DIM) ** 3 + + size_py = (np.float32(1.0).nbytes) * size_py + + """Memory usage of C initial conditions""" + KSPACE_NUM_PIXELS = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 + + # HIRES_box, HIRES_box_saved + num_c_boxes = 2 + + # HIRES_box_vcb_x, HIRES_box_vcb_y, HIRES_box_vcb_z + if user_params.USE_RELATIVE_VELOCITIES: + num_c_boxes += 3 + + # phi_1 (6 components) + if global_params.SECOND_ORDER_LPT_CORRECTIONS: + num_c_boxes += 6 + + # These are all fftwf complex arrays (thus 2 * size) + size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * KSPACE_NUM_PIXELS + + return {"python": size_py, "c": size_c} From 06617dfd468bf556825a1a5e231534425964c76a Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 12:23:45 +1100 Subject: [PATCH 02/28] Finalise ICs memory function --- src/py21cmfast/_memory.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 22b154059..454249319 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -51,7 +51,9 @@ def estimate_memory_lightcone( flag_options=None, ): """Compute an estimate of the requisite memory needed by the user for a run_lightcone call.""" - return {} + memory_ics = mem_initial_conditions(user_params=user_params) + + return memory_ics def mem_initial_conditions( @@ -59,34 +61,35 @@ def mem_initial_conditions( user_params=None, ): """Memory usage of Python initial conditions class.""" - # lowres_density, lowres_vx, lowres_vy, lowres_vz, lowres_vcb, + """All declared HII_DIM boxes""" + # lowres_density, lowres_vx, lowres_vy, lowres_vz, lowres_vcb # lowres_vx_2LPT, lowres_vy_2LPT, lowres_vz_2LPT num_py_boxes_HII_DIM = 8.0 - # hires_density, hires_vx, hires_vy, hires_vz, hires_vcb, + """All declared DIM boxes""" + # hires_density, hires_vx, hires_vy, hires_vz # hires_vx_2LPT, hires_vy_2LPT, hires_vz_2LPT - num_py_boxes_DIM = 8.0 + num_py_boxes_DIM = 7.0 size_py = num_py_boxes_DIM * (user_params.DIM) ** 3 size_py += num_py_boxes_HII_DIM * (user_params.HII_DIM) ** 3 + # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py """Memory usage of C initial conditions""" - KSPACE_NUM_PIXELS = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 + kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 + """All declared DIM boxes""" # HIRES_box, HIRES_box_saved num_c_boxes = 2 - # HIRES_box_vcb_x, HIRES_box_vcb_y, HIRES_box_vcb_z - if user_params.USE_RELATIVE_VELOCITIES: - num_c_boxes += 3 - + """All declared fftwf boxes (DIM)""" # phi_1 (6 components) if global_params.SECOND_ORDER_LPT_CORRECTIONS: num_c_boxes += 6 # These are all fftwf complex arrays (thus 2 * size) - size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * KSPACE_NUM_PIXELS + size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * kspace_num_pixels return {"python": size_py, "c": size_c} From 6860415bf3d7da1175719bdfe06cd942e4345369 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 13:07:00 +1100 Subject: [PATCH 03/28] Add function for perturb_field --- src/py21cmfast/_memory.py | 63 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 454249319..0efae83f3 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -53,14 +53,22 @@ def estimate_memory_lightcone( """Compute an estimate of the requisite memory needed by the user for a run_lightcone call.""" memory_ics = mem_initial_conditions(user_params=user_params) - return memory_ics + memory_pf = mem_perturb_field(user_params=user_params) + + return { + "ics_python": memory_ics["python"], + "ics_c": memory_ics["c"], + "pf_python": memory_pf["python"], + "pf_c": memory_pf["c"], + } def mem_initial_conditions( *, user_params=None, ): - """Memory usage of Python initial conditions class.""" + """A function to estimate total memory usage of an initial_conditions call.""" + """Memory usage of Python InitialConditions class.""" """All declared HII_DIM boxes""" # lowres_density, lowres_vx, lowres_vy, lowres_vz, lowres_vcb # lowres_vx_2LPT, lowres_vy_2LPT, lowres_vz_2LPT @@ -77,14 +85,14 @@ def mem_initial_conditions( # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py - """Memory usage of C initial conditions""" + """Memory usage within GenerateICs""" kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 """All declared DIM boxes""" # HIRES_box, HIRES_box_saved num_c_boxes = 2 - """All declared fftwf boxes (DIM)""" + """All declared 2LPT boxes (DIM)""" # phi_1 (6 components) if global_params.SECOND_ORDER_LPT_CORRECTIONS: num_c_boxes += 6 @@ -93,3 +101,50 @@ def mem_initial_conditions( size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * kspace_num_pixels return {"python": size_py, "c": size_c} + + +def mem_perturb_field( + *, + user_params=None, +): + """A function to estimate total memory usage of a perturb_field call.""" + """Memory usage of Python PerturbedField class.""" + """All declared HII_DIM boxes""" + # density, velocity + num_py_boxes_HII_DIM = 2.0 + + size_py = num_py_boxes_HII_DIM * (user_params.HII_DIM) ** 3 + + # These are all float arrays + size_py = (np.float32(1.0).nbytes) * size_py + + """Memory usage within PerturbField.c""" + kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 + hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( + user_params.HII_DIM + ) ** 2 + + tot_num_pixels = float(user_params.DIM) ** 3 + hii_tot_num_pixels = float(user_params.HII_DIM) ** 3 + + # LOWRES_density_perturb, LOWRES_density_perturb_saved + num_c_boxes_HII_DIM = 2 + + size_c = ( + (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes_HII_DIM * hii_kspace_num_pixels + ) + + num_c_boxes_DIM = 0 + if user_params.PERTURB_ON_HIGH_RES: + # HIRES_density_perturb, HIRES_density_perturb_saved + num_c_boxes_DIM += 2 + + size_c += (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes_DIM * kspace_num_pixels + + # For resampled_box (double) + size_c += (np.float64(1.0).nbytes) * tot_num_pixels + else: + # For resampled_box (double) + size_c += (np.float64(1.0).nbytes) * hii_tot_num_pixels + + return {"python": size_py, "c": size_c} From 42697c9ec8a5273f4b55d62502fb78b28d1c430f Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 15:02:39 +1100 Subject: [PATCH 04/28] Added memory function for ionize_box --- src/py21cmfast/_memory.py | 90 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 0efae83f3..b7a7ae6ef 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -55,11 +55,19 @@ def estimate_memory_lightcone( memory_pf = mem_perturb_field(user_params=user_params) + memory_ib = mem_ionize_box( + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + return { "ics_python": memory_ics["python"], "ics_c": memory_ics["c"], "pf_python": memory_pf["python"], "pf_c": memory_pf["c"], + "ib_python": memory_ib["python"], + "ib_c": memory_ib["c"], } @@ -148,3 +156,85 @@ def mem_perturb_field( size_c += (np.float64(1.0).nbytes) * hii_tot_num_pixels return {"python": size_py, "c": size_c} + + +def mem_ionize_box( + *, + user_params=None, + astro_params=None, + flag_options=None, +): + """A function to estimate total memory usage of an ionize_box call.""" + # First do check on inhomogeneous recombinations + astro_params = AstroParams(astro_params, INHOMO_RECO=flag_options.INHOMO_RECO) + + # determine number of filtering scales (for USE_MINI_HALOS) + if flag_options.USE_MINI_HALOS: + n_filtering = ( + int( + np.log( + min(astro_params.R_BUBBLE_MAX, 0.620350491 * user_params.BOX_LEN) + / max( + global_params.R_BUBBLE_MIN, + 0.620350491 + * user_params.BOX_LEN + / np.float64(user_params.HII_DIM), + ) + ) + / np.log(global_params.DELTA_R_HII_FACTOR) + ) + ) + 1 + else: + n_filtering = 1 + + """Memory usage of Python IonizedBox class.""" + + """All declared HII_DIM boxes""" + # xH_box, Gamma12_box, MFP_box, z_re_box, dNrec_box, temp_kinetic_all_gas + num_py_boxes = 6.0 + + # Fcoll + num_py_boxes += n_filtering + + # Fcoll_MINI + if flag_options.USE_MINI_HALOS: + num_py_boxes += n_filtering + + size_py = num_py_boxes * (user_params.HII_DIM) ** 3 + + # These are all float arrays + size_py = (np.float32(1.0).nbytes) * size_py + + """Memory usage within IonisationBox.c""" + hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( + user_params.HII_DIM + ) ** 2 + + # deltax_unfiltered, delta_unfiltered_original, deltax_filtered + num_c_boxes = 3.0 + + if flag_options.USE_MINI_HALOS: + # prev_deltax_unfiltered, prev_deltax_filtered + num_c_boxes += 2.0 + + if flag_options.USE_TS_FLUCT: + # xe_unfiltered, xe_filtered + num_c_boxes += 2.0 + + if flag_options.INHOMO_RECO: + # N_rec_unfiltered, N_rec_filtered + num_c_boxes += 2.0 + + # There are a bunch of 1 and 2D interpolation tables, but ignore those as they are small relative to 3D grids + + if flag_options.USE_MASS_DEPENDENT_ZETA and flag_options.USE_MINI_HALOS: + # log10_Mturnover_unfiltered, log10_Mturnover_filtered, log10_Mturnover_MINI_unfiltered, log10_Mturnover_MINI_filtered + num_c_boxes += 4.0 + + if flag_options.USE_HALO_FIELD: + # M_coll_unfiltered, M_coll_filtered + num_c_boxes += 2.0 + + size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels + + return {"python": size_py, "c": size_c} From 43a6e8999c92bf39d765442e71286a688518110a Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 16:06:40 +1100 Subject: [PATCH 05/28] Added memory function for the spin temperature calculation --- src/py21cmfast/_memory.py | 103 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index b7a7ae6ef..a429b556c 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -61,6 +61,12 @@ def estimate_memory_lightcone( flag_options=flag_options, ) + memory_st = mem_spin_temperature( + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + return { "ics_python": memory_ics["python"], "ics_c": memory_ics["c"], @@ -68,6 +74,8 @@ def estimate_memory_lightcone( "pf_c": memory_pf["c"], "ib_python": memory_ib["python"], "ib_c": memory_ib["c"], + "st_python": memory_st["python"], + "st_c": memory_st["c"], } @@ -138,6 +146,7 @@ def mem_perturb_field( # LOWRES_density_perturb, LOWRES_density_perturb_saved num_c_boxes_HII_DIM = 2 + # These are all fftwf complex arrays (thus 2 * size) size_c = ( (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes_HII_DIM * hii_kspace_num_pixels ) @@ -147,6 +156,7 @@ def mem_perturb_field( # HIRES_density_perturb, HIRES_density_perturb_saved num_c_boxes_DIM += 2 + # These are all fftwf complex arrays (thus 2 * size) size_c += (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes_DIM * kspace_num_pixels # For resampled_box (double) @@ -235,6 +245,99 @@ def mem_ionize_box( # M_coll_unfiltered, M_coll_filtered num_c_boxes += 2.0 + # These are all fftwf complex arrays (thus 2 * size) + size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels + + return {"python": size_py, "c": size_c} + + +def mem_spin_temperature( + *, + user_params=None, + astro_params=None, + flag_options=None, +): + """A function to estimate total memory usage of a spin_temperature call.""" + """Memory usage of Python IonizedBox class.""" + + """All declared HII_DIM boxes""" + # Ts_bx, x_e_box, Tk_box, J_21_LW_box + num_py_boxes = 4.0 + + size_py = num_py_boxes * (user_params.HII_DIM) ** 3 + + # These are all float arrays + size_py = (np.float32(1.0).nbytes) * size_py + + """Memory usage within SpinTemperatureBox.c""" + hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( + user_params.HII_DIM + ) ** 2 + + # box, unfiltered_box + num_c_boxes = 2.0 + num_c_boxes_alt = 0.0 + + mem_c_interp = 0.0 + + if flag_options.USE_MINI_HALOS: + # log10_Mcrit_LW_unfiltered, log10_Mcrit_LW_filtered + num_c_boxes += 2.0 + + num_c_boxes_alt += global_params.NUM_FILTER_STEPS_FOR_Ts + + # There are a bunch of 1 and 2D interpolation tables, but ignore those as they are small relative to 3D grids + # I will add any that could end up considerably large. + + if flag_options.USE_MASS_DEPENDENT_ZETA: + # delNL0 + num_c_boxes_alt += global_params.NUM_FILTER_STEPS_FOR_Ts + + # del_fcoll_Rct, m_xHII_low_box, inverse_val_box + num_c_boxes_alt += ( + 3.0 # m_xHII_low_box is an int, but I'll count it as 4 bytes just in case + ) + + # dxheat_dt_box, dxion_source_dt_box, dxlya_dt_box, dstarlya_dt_box + num_c_boxes_alt += 2.0 * 4.0 # factor of 2. as these are doubles + + if flag_options.USE_MINI_HALOS: + # del_fcoll_Rct_MINI + num_c_boxes_alt += 1.0 + + # dstarlyLW_dt_box, dxheat_dt_box_MINI, dxion_source_dt_box_MINI + # dxlya_dt_box_MINI, dstarlya_dt_box_MINI, dstarlyLW_dt_box_MINI + num_c_boxes_alt += ( + 2.0 * 6.0 + ) # factor of 2. taking into account that these are doubles + + if user_params.USE_INTERPOLATION_TABLES: + # log10_SFRD_z_low_table_MINI, SFRD_z_high_table_MINI (factor of 4 as these are float tables) + mem_c_interp += ( + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * 250.0 * 50.0 + ) # NSFR_low = 250, NMTURN = 50 + + mem_c_interp += ( + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * 200.0 * 50.0 + ) # NSFR_high = 200, NMTURN = 50 + else: + # delNL0_rev + num_c_boxes_alt += global_params.NUM_FILTER_STEPS_FOR_Ts + + if user_params.USE_INTERPOLATION_TABLES: + # fcoll_R_grid, dfcoll_dz_grid (factor of 8. as these are double) + mem_c_interp += ( + 2.0 * 8.0 * (global_params.NUM_FILTER_STEPS_FOR_Ts * 400 * 400) + ) # zpp_interp_points_SFR = 400, dens_Ninterp = 400 + + # dens_grid_int_vals + num_c_boxes_alt += 0.5 # 0.5 as it is a short + + # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels + size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.HII_DIM ** 3.0) + + size_c += mem_c_interp + return {"python": size_py, "c": size_c} From 8e4f6af8ebc6902a7c4a59f0d9f2aa53a499f0ae Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 16:16:30 +1100 Subject: [PATCH 06/28] Added memory function for brightness temperature --- src/py21cmfast/_memory.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index a429b556c..c41777104 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -67,6 +67,8 @@ def estimate_memory_lightcone( flag_options=flag_options, ) + memory_bt = mem_brightness_temperature(user_params=user_params) + return { "ics_python": memory_ics["python"], "ics_c": memory_ics["c"], @@ -75,7 +77,9 @@ def estimate_memory_lightcone( "ib_python": memory_ib["python"], "ib_c": memory_ib["c"], "st_python": memory_st["python"], - "st_c": memory_st["c"], + "st_c": memory_bt["c"], + "bt_python": memory_st["python"], + "bt_c": memory_bt["c"], } @@ -341,3 +345,32 @@ def mem_spin_temperature( size_c += mem_c_interp return {"python": size_py, "c": size_c} + + +def mem_brightness_temperature( + *, + user_params=None, +): + """A function to estimate total memory usage of a brightness_temperature call.""" + """Memory usage of Python BrightnessTemp class.""" + + """All declared HII_DIM boxes""" + # brightness_temp + num_py_boxes = 1.0 + + size_py = num_py_boxes * (user_params.HII_DIM) ** 3 + + # These are all float arrays + size_py = (np.float32(1.0).nbytes) * size_py + + """Memory usage within BrightnessTemperatureBox.c""" + hii_tot_fft_num_pixels = ( + 2.0 * (float(user_params.HII_DIM) / 2.0 + 1.0) * (user_params.HII_DIM) ** 2 + ) + + # box, unfiltered_box + num_c_boxes = 2.0 + + size_c = (np.float32(1.0).nbytes) * num_c_boxes * hii_tot_fft_num_pixels + + return {"python": size_py, "c": size_c} From 89f62698830feb4da9bf8891aeaca798f9f7bb55 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 17:04:25 +1100 Subject: [PATCH 07/28] Add memory function for determine_halo_list --- src/py21cmfast/_memory.py | 49 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index c41777104..e9dfabb77 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -69,6 +69,8 @@ def estimate_memory_lightcone( memory_bt = mem_brightness_temperature(user_params=user_params) + memory_hf = mem_halo_field(user_params=user_params) + return { "ics_python": memory_ics["python"], "ics_c": memory_ics["c"], @@ -80,6 +82,8 @@ def estimate_memory_lightcone( "st_c": memory_bt["c"], "bt_python": memory_st["python"], "bt_c": memory_bt["c"], + "hf_python": memory_hf["python"], + "hf_c": memory_hf["c"], } @@ -374,3 +378,48 @@ def mem_brightness_temperature( size_c = (np.float32(1.0).nbytes) * num_c_boxes * hii_tot_fft_num_pixels return {"python": size_py, "c": size_c} + + +def mem_halo_field( + *, + user_params=None, +): + """A function to estimate total memory usage of a determine_halo_list call.""" + """Memory usage of Python HaloField class.""" + + """All declared DIM boxes""" + # halo_field + num_py_boxes = 1.0 + + size_py = num_py_boxes * (user_params.DIM) ** 3 + + # These are all float arrays + size_py = (np.float32(1.0).nbytes) * size_py + + """Memory usage within FindHaloes.c""" + kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 + + # density_field, density_field_saved + num_c_boxes = 2.0 + + # halo_field, in_halo + num_c_boxes_alt = ( + 1.25 # in_halo is size 1 (char *) so count it as 0.25 of float (4) + ) + + if global_params.OPTIMIZE: + # forbidden + num_c_boxes_alt += 0.25 + + # These are fftwf complex arrays (thus 2 * size) + size_c = (2.0 * np.float32(1.0).nbytes) * num_c_boxes * kspace_num_pixels + + size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.DIM) ** 3 + + # We don't know a priori how many haloes that will be found, but we'll estimate the memory usage + # at 10 per cent of the total number of pixels (likely an over estimate) + # Below the factor of 4 corresponds to the mass and three spatial locations. It is defined as an + # int but I'll leave it as 4 bytes in case + size_py += 0.1 * 4.0 * (np.float32(1.0).nbytes) * (user_params.DIM) ** 3 + + return {"python": size_py, "c": size_c} From b9423bc5ad78f054c08490c546e7c03d9f11dde5 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 25 Mar 2021 17:16:21 +1100 Subject: [PATCH 08/28] Added function for perturbed halo field --- src/py21cmfast/_memory.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index e9dfabb77..ddfa08099 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -71,6 +71,8 @@ def estimate_memory_lightcone( memory_hf = mem_halo_field(user_params=user_params) + memory_phf = mem_perturb_halo(user_params=user_params) + return { "ics_python": memory_ics["python"], "ics_c": memory_ics["c"], @@ -84,6 +86,8 @@ def estimate_memory_lightcone( "bt_c": memory_bt["c"], "hf_python": memory_hf["python"], "hf_c": memory_hf["c"], + "phf_python": memory_phf["python"], + "phf_c": memory_phf["c"], } @@ -417,9 +421,25 @@ def mem_halo_field( size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.DIM) ** 3 # We don't know a priori how many haloes that will be found, but we'll estimate the memory usage - # at 10 per cent of the total number of pixels (likely an over estimate) + # at 10 per cent of the total number of pixels (HII_DIM, likely an over estimate) # Below the factor of 4 corresponds to the mass and three spatial locations. It is defined as an # int but I'll leave it as 4 bytes in case - size_py += 0.1 * 4.0 * (np.float32(1.0).nbytes) * (user_params.DIM) ** 3 + size_c += 0.1 * 4.0 * (np.float32(1.0).nbytes) * (user_params.HII_DIM) ** 3 return {"python": size_py, "c": size_c} + + +def mem_perturb_halo( + *, + user_params=None, +): + """A function to estimate total memory usage of a perturb_halo_list call.""" + """Memory usage of Python PerturbHaloField class.""" + + # We don't know a priori how many haloes that will be found, but we'll estimate the memory usage + # at 10 per cent of the total number of pixels (HII_DIM, likely an over estimate) + # Below the factor of 4 corresponds to the mass and three spatial locations. It is defined as an + # int but I'll leave it as 4 bytes in case + size_c = 0.1 * 4.0 * (np.float32(1.0).nbytes) * (user_params.HII_DIM) ** 3 + + return {"python": 0.0, "c": size_c} From 71f24c0563c97c74e854a63b3d7bfd3c7a161ea1 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Fri, 26 Mar 2021 12:15:05 +1100 Subject: [PATCH 09/28] Add in code to calculate the size of all stored lightcones --- src/py21cmfast/_memory.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index ddfa08099..d28270f6a 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -3,6 +3,7 @@ import numpy as np from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params +from .wrapper import _logscroll_redshifts, _setup_lightcone def estimate_memory_coeval( @@ -45,10 +46,13 @@ def estimate_memory_coeval( def estimate_memory_lightcone( *, + redshift=None, + max_redshift=None, user_params=None, cosmo_params=None, astro_params=None, flag_options=None, + lightcone_quantities=("brightness_temp",), ): """Compute an estimate of the requisite memory needed by the user for a run_lightcone call.""" memory_ics = mem_initial_conditions(user_params=user_params) @@ -73,6 +77,41 @@ def estimate_memory_lightcone( memory_phf = mem_perturb_halo(user_params=user_params) + # Now need to determine the size of the light-cone and how many types are to be stored in memory. + # Below are taken from the run_lightcone function. + # Determine the maximum redshift (starting point) for the light-cone. + max_redshift = ( + global_params.Z_HEAT_MAX + if ( + flag_options.INHOMO_RECO + or flag_options.USE_TS_FLUCT + or max_redshift is None + ) + else max_redshift + ) + + # Get the redshift through which we scroll and evaluate the ionization field. + scrollz = _logscroll_redshifts( + redshift, global_params.ZPRIME_STEP_FACTOR, max_redshift + ) + + # Obtain the size of the light-cone object (n_lightcone) + d_at_redshift, lc_distances, n_lightcone = _setup_lightcone( + cosmo_params, + max_redshift, + redshift, + scrollz, + user_params, + global_params.ZPRIME_STEP_FACTOR, + ) + + # Total number of light-cones to be stored in memory + num_lightcones = len(lightcone_quantities) + + # Calculate memory footprint for all light-cones + size_lightcones = n_lightcone * (user_params.HII_DIM) ** 2 + size_lightcones = (np.float32(1.0).nbytes) * num_lightcones * size_lightcones + return { "ics_python": memory_ics["python"], "ics_c": memory_ics["c"], From 1b9355d3768451e3c20bef352121893592ec6367 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Fri, 26 Mar 2021 15:30:42 +1100 Subject: [PATCH 10/28] Add in the calculation of the memory for the lightcone function --- src/py21cmfast/_memory.py | 213 +++++++++++++++++++++++++++++++------- 1 file changed, 175 insertions(+), 38 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index d28270f6a..8972644f8 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -1,6 +1,7 @@ """Function to estimate total memory usage.""" import numpy as np +from copy import deepcopy from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params from .wrapper import _logscroll_redshifts, _setup_lightcone @@ -55,27 +56,67 @@ def estimate_memory_lightcone( lightcone_quantities=("brightness_temp",), ): """Compute an estimate of the requisite memory needed by the user for a run_lightcone call.""" + # First, calculate the memory usage for the initial conditions memory_ics = mem_initial_conditions(user_params=user_params) + memory_data = {"ics_%s" % k: memory_ics["%s" % k] for k in memory_ics.keys()} + + # Maximum memory while running ICs + peak_memory = memory_ics["c"] + memory_ics["python"] + + # Now the perturb field memory_pf = mem_perturb_field(user_params=user_params) - memory_ib = mem_ionize_box( - user_params=user_params, - astro_params=astro_params, - flag_options=flag_options, - ) + memory_data.update({"pf_%s" % k: memory_pf["%s" % k] for k in memory_pf.keys()}) - memory_st = mem_spin_temperature( - user_params=user_params, - astro_params=astro_params, - flag_options=flag_options, - ) + # Stored ICs in python + allocated C and Python memory for perturb_field + current_memory = memory_ics["python"] + memory_pf["python"] + memory_pf["c"] - memory_bt = mem_brightness_temperature(user_params=user_params) + # Check if running perturb_field requires more memory than generating ICs + peak_memory = peak_memory if peak_memory > current_memory else current_memory - memory_hf = mem_halo_field(user_params=user_params) + # If we are using the photon non-conservation correction + if flag_options.PHOTON_CONS: + # First need to create new structs for photon the photon-conservation + astro_params_photoncons = deepcopy(astro_params) + astro_params_photoncons._R_BUBBLE_MAX = astro_params.R_BUBBLE_MAX + + flag_options_photoncons = FlagOptions( + USE_MASS_DEPENDENT_ZETA=flag_options.USE_MASS_DEPENDENT_ZETA, + M_MIN_in_Mass=flag_options.M_MIN_in_Mass, + USE_VELS_AUX=user_params.USE_RELATIVE_VELOCITIES, + ) - memory_phf = mem_perturb_halo(user_params=user_params) + # First perturb_field + memory_pf = mem_perturb_field(user_params=user_params) + + # First ionize_box + memory_ib = mem_ionize_box( + user_params=user_params, + astro_params=astro_params_photoncons, + flag_options=flag_options_photoncons, + ) + + # As we iterate through we storing the python memory of two + # perturb_field and ionize_boxes plus the C memory of either + # of the ionize_box or perturb field as it is being calculated + peak_memory_photoncons = memory_ics[ + "python" + ] # We have the initial conditions in memory + peak_memory_photoncons += 2 * ( + memory_pf["python"] + memory_ib["python"] + ) # The python memory + peak_memory_photoncons += ( + memory_pf["c"] if memory_pf["c"] > memory_ib["c"] else memory_ib["c"] + ) # Maximum C memory, as it is freed after usage + + # Check if the memory required to do the photon non-conservation correction exceeds + # current peak memory usage + peak_memory = ( + peak_memory + if peak_memory > peak_memory_photoncons + else peak_memory_photoncons + ) # Now need to determine the size of the light-cone and how many types are to be stored in memory. # Below are taken from the run_lightcone function. @@ -112,22 +153,110 @@ def estimate_memory_lightcone( size_lightcones = n_lightcone * (user_params.HII_DIM) ** 2 size_lightcones = (np.float32(1.0).nbytes) * num_lightcones * size_lightcones - return { - "ics_python": memory_ics["python"], - "ics_c": memory_ics["c"], - "pf_python": memory_pf["python"], - "pf_c": memory_pf["c"], - "ib_python": memory_ib["python"], - "ib_c": memory_ib["c"], - "st_python": memory_st["python"], - "st_c": memory_bt["c"], - "bt_python": memory_st["python"], - "bt_c": memory_bt["c"], - "hf_python": memory_hf["python"], - "hf_c": memory_hf["c"], - "phf_python": memory_phf["python"], - "phf_c": memory_phf["c"], - } + memory_data.update({"python_lc": size_lightcones}) + + # All the data kept in memory at this point in Python + current_memory = memory_ics["python"] + memory_pf["python"] + size_lightcones + + # Check if we now exceed the peak memory usage thus far + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Now start generating the data to populate the light-cones + + # Calculate the memory for a determine_halo_list call + memory_hf = mem_halo_field(user_params=user_params) + + memory_data.update({"hf_%s" % k: memory_hf["%s" % k] for k in memory_hf.keys()}) + + # Calculate the memory for a perturb_halo_list call + memory_phf = mem_perturb_halo(user_params=user_params) + + memory_data.update({"phf_%s" % k: memory_phf["%s" % k] for k in memory_phf.keys()}) + + # Calculate the memory for an ionize_box call + memory_ib = mem_ionize_box( + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + + memory_data.update({"ib_%s" % k: memory_ib["%s" % k] for k in memory_ib.keys()}) + + # Calculate the memory for a spin_temperature call + memory_st = mem_spin_temperature( + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + + memory_data.update({"st_%s" % k: memory_st["%s" % k] for k in memory_st.keys()}) + + # Calculate the memory for a brightness_temperature call + memory_bt = mem_brightness_temperature(user_params=user_params) + + memory_data.update({"bt_%s" % k: memory_bt["%s" % k] for k in memory_bt.keys()}) + + # Now, when calculating the light-cone we always need two concurrent boxes (current and previous redshift) + + # This is the all the data currently in memory at this point + stored_memory = current_memory # Corresponds to ICs, one perturb_field and the allocated light-cones + + # Add second perturb_field + stored_memory += memory_pf["python"] + + # Add the memory from perturb_haloes + if flag_options.USE_HALO_FIELD: + # We'll have two copies of this at any one time, one of the original haloes + # and one for the current redshift we are evaluating + # Note this is approximate as we can only guess the total memory due to the + # fact that the amount of memory required is proportional to the number of haloes + # found. + stored_memory += 2.0 * memory_phf["c"] + + # Add the two ionized boxes + stored_memory += 2.0 * memory_ib["python"] + + # Add (if necessary) the two spin temperature boxes + if flag_options.USE_TS_FLUCT: + stored_memory += 2.0 * memory_st["python"] + + # We also have initialied C memory that is retained until the end of the calculation + stored_memory += memory_st["c_init"] + + # Add the two brightness_temperature boxes + stored_memory += 2.0 * memory_bt["python"] + + # Now we have an estimate for the data retained in memory, now we just need to check + # the peak memory usage (which includes the additional C memory that is allocated + # and then freed on a single redshift call) + # First check perturb field + current_memory = stored_memory + memory_pf["c"] # Add the temporary C memory + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Check spin temperature + if flag_options.USE_TS_FLUCT: + current_memory = stored_memory + memory_st["c_per_z"] + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Check ionized box + current_memory = stored_memory + memory_ib["c"] + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Check brightness temperature + current_memory = stored_memory + memory_bt["c"] + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + memory_data.update({"peak_memory": peak_memory}) + + return memory_data def mem_initial_conditions( @@ -328,6 +457,7 @@ def mem_spin_temperature( # box, unfiltered_box num_c_boxes = 2.0 num_c_boxes_alt = 0.0 + num_c_boxes_initialised = 0.0 mem_c_interp = 0.0 @@ -335,6 +465,7 @@ def mem_spin_temperature( # log10_Mcrit_LW_unfiltered, log10_Mcrit_LW_filtered num_c_boxes += 2.0 + # log10_Mcrit_LW num_c_boxes_alt += global_params.NUM_FILTER_STEPS_FOR_Ts # There are a bunch of 1 and 2D interpolation tables, but ignore those as they are small relative to 3D grids @@ -342,23 +473,23 @@ def mem_spin_temperature( if flag_options.USE_MASS_DEPENDENT_ZETA: # delNL0 - num_c_boxes_alt += global_params.NUM_FILTER_STEPS_FOR_Ts + num_c_boxes_initialised += global_params.NUM_FILTER_STEPS_FOR_Ts # del_fcoll_Rct, m_xHII_low_box, inverse_val_box - num_c_boxes_alt += ( + num_c_boxes_initialised += ( 3.0 # m_xHII_low_box is an int, but I'll count it as 4 bytes just in case ) # dxheat_dt_box, dxion_source_dt_box, dxlya_dt_box, dstarlya_dt_box - num_c_boxes_alt += 2.0 * 4.0 # factor of 2. as these are doubles + num_c_boxes_initialised += 2.0 * 4.0 # factor of 2. as these are doubles if flag_options.USE_MINI_HALOS: # del_fcoll_Rct_MINI - num_c_boxes_alt += 1.0 + num_c_boxes_initialised += 1.0 # dstarlyLW_dt_box, dxheat_dt_box_MINI, dxion_source_dt_box_MINI # dxlya_dt_box_MINI, dstarlya_dt_box_MINI, dstarlyLW_dt_box_MINI - num_c_boxes_alt += ( + num_c_boxes_initialised += ( 2.0 * 6.0 ) # factor of 2. taking into account that these are doubles @@ -373,7 +504,7 @@ def mem_spin_temperature( ) # NSFR_high = 200, NMTURN = 50 else: # delNL0_rev - num_c_boxes_alt += global_params.NUM_FILTER_STEPS_FOR_Ts + num_c_boxes_initialised += global_params.NUM_FILTER_STEPS_FOR_Ts if user_params.USE_INTERPOLATION_TABLES: # fcoll_R_grid, dfcoll_dz_grid (factor of 8. as these are double) @@ -382,16 +513,22 @@ def mem_spin_temperature( ) # zpp_interp_points_SFR = 400, dens_Ninterp = 400 # dens_grid_int_vals - num_c_boxes_alt += 0.5 # 0.5 as it is a short + num_c_boxes_initialised += 0.5 # 0.5 as it is a short # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.HII_DIM ** 3.0) - size_c += mem_c_interp + size_c_init = ( + (np.float32(1.0).nbytes) + * num_c_boxes_initialised + * (user_params.HII_DIM ** 3.0) + ) + + size_c_init += mem_c_interp - return {"python": size_py, "c": size_c} + return {"python": size_py, "c_init": size_c_init, "c_per_z": size_c} def mem_brightness_temperature( From 2ecdb64075d82a1aeea41a1f21e9c34d520607ca Mon Sep 17 00:00:00 2001 From: BradGreig Date: Fri, 26 Mar 2021 16:58:46 +1100 Subject: [PATCH 11/28] Add some output messaging for the lightcone function --- src/py21cmfast/_memory.py | 85 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 8972644f8..ecc17db3c 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -1,11 +1,14 @@ """Function to estimate total memory usage.""" +import logging import numpy as np from copy import deepcopy from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params from .wrapper import _logscroll_redshifts, _setup_lightcone +logger = logging.getLogger("21cmFAST") + def estimate_memory_coeval( *, @@ -256,6 +259,13 @@ def estimate_memory_lightcone( memory_data.update({"peak_memory": peak_memory}) + format_output( + memory_data=memory_data, + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + return memory_data @@ -619,3 +629,78 @@ def mem_perturb_halo( size_c = 0.1 * 4.0 * (np.float32(1.0).nbytes) * (user_params.HII_DIM) ** 3 return {"python": 0.0, "c": size_c} + + +def format_output( + *, + memory_data=None, + user_params=None, + astro_params=None, + flag_options=None, +): + """Function to output information in a manageable format.""" + bytes_in_gb = 1024 ** 3 + + print("") + if "python_lc" in memory_data.keys(): + print("Memory info for run_lightcone") + else: + print("Memory info for run_coeval") + print("") + print("%s" % (user_params)) + print("%s" % (astro_params)) + print("%s" % (flag_options)) + print("") + print("Peak memory usage: %g (GB)" % (memory_data["peak_memory"] / bytes_in_gb)) + print("") + if "python_lc" in memory_data.keys(): + print( + "Memory for stored lightcones: %g (GB)" + % (memory_data["python_lc"] / bytes_in_gb) + ) + """logger.info("Peak memory usage: %g (GB)"%(memory_data['peak_memory']/bytes_in_gb))""" + print( + "Memory for ICs: %g (GB; Python) %g (GB; C)" + % (memory_data["ics_python"] / bytes_in_gb, memory_data["ics_c"] / bytes_in_gb) + ) + print( + "Memory for single perturbed field: %g (GB; Python) %g (GB; C)" + % (memory_data["pf_python"] / bytes_in_gb, memory_data["pf_c"] / bytes_in_gb) + ) + if flag_options.USE_HALO_FIELD: + print( + "Note these are approximations as we don't know a priori how many haloes there are (assume 10 per cent of volume)" + ) + print( + "Memory for generating halo list: %g (GB; Python) %g (GB; C)" + % ( + memory_data["hf_python"] / bytes_in_gb, + memory_data["hf_c"] / bytes_in_gb, + ) + ) + print( + "Memory for perturbing halo list: %g (GB; Python) %g (GB; C)" + % ( + memory_data["phf_python"] / bytes_in_gb, + memory_data["phf_c"] / bytes_in_gb, + ) + ) + + print( + "Memory for single ionized box: %g (GB; Python) %g (GB; C)" + % (memory_data["ib_python"] / bytes_in_gb, memory_data["ib_c"] / bytes_in_gb) + ) + if flag_options.USE_TS_FLUCT: + print( + "Memory for single spin temperature box: %g (GB; Python) %g (GB; C per z) %g (GB; C retained)" + % ( + memory_data["st_python"] / bytes_in_gb, + memory_data["st_c_per_z"] / bytes_in_gb, + memory_data["st_c_init"] / bytes_in_gb, + ) + ) + print( + "Memory for single brightness temperature box: %g (GB; Python) %g (GB; C)" + % (memory_data["bt_python"] / bytes_in_gb, memory_data["bt_c"] / bytes_in_gb) + ) + print("") From 15a2b3c3762329a54ea2873ca2c8bbc4c3257f70 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 8 Apr 2021 16:17:54 +1000 Subject: [PATCH 12/28] Some minor modifications (formatting) following Steven's feedback --- src/py21cmfast/_memory.py | 66 +++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index ecc17db3c..7a2025e98 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -59,10 +59,13 @@ def estimate_memory_lightcone( lightcone_quantities=("brightness_temp",), ): """Compute an estimate of the requisite memory needed by the user for a run_lightcone call.""" + # Deal with AstroParams and INHOMO_RECO + astro_params = AstroParams(astro_params, INHOMO_RECO=flag_options.INHOMO_RECO) + # First, calculate the memory usage for the initial conditions memory_ics = mem_initial_conditions(user_params=user_params) - memory_data = {"ics_%s" % k: memory_ics["%s" % k] for k in memory_ics.keys()} + memory_data = {"ics_%s" % k: memory_ics[k] for k in memory_ics.keys()} # Maximum memory while running ICs peak_memory = memory_ics["c"] + memory_ics["python"] @@ -70,7 +73,7 @@ def estimate_memory_lightcone( # Now the perturb field memory_pf = mem_perturb_field(user_params=user_params) - memory_data.update({"pf_%s" % k: memory_pf["%s" % k] for k in memory_pf.keys()}) + memory_data.update({"pf_%s" % k: memory_pf[k] for k in memory_pf.keys()}) # Stored ICs in python + allocated C and Python memory for perturb_field current_memory = memory_ics["python"] + memory_pf["python"] + memory_pf["c"] @@ -82,7 +85,6 @@ def estimate_memory_lightcone( if flag_options.PHOTON_CONS: # First need to create new structs for photon the photon-conservation astro_params_photoncons = deepcopy(astro_params) - astro_params_photoncons._R_BUBBLE_MAX = astro_params.R_BUBBLE_MAX flag_options_photoncons = FlagOptions( USE_MASS_DEPENDENT_ZETA=flag_options.USE_MASS_DEPENDENT_ZETA, @@ -169,12 +171,12 @@ def estimate_memory_lightcone( # Calculate the memory for a determine_halo_list call memory_hf = mem_halo_field(user_params=user_params) - memory_data.update({"hf_%s" % k: memory_hf["%s" % k] for k in memory_hf.keys()}) + memory_data.update({"hf_%s" % k: memory_hf[k] for k in memory_hf.keys()}) # Calculate the memory for a perturb_halo_list call memory_phf = mem_perturb_halo(user_params=user_params) - memory_data.update({"phf_%s" % k: memory_phf["%s" % k] for k in memory_phf.keys()}) + memory_data.update({"phf_%s" % k: memory_phf[k] for k in memory_phf.keys()}) # Calculate the memory for an ionize_box call memory_ib = mem_ionize_box( @@ -183,7 +185,7 @@ def estimate_memory_lightcone( flag_options=flag_options, ) - memory_data.update({"ib_%s" % k: memory_ib["%s" % k] for k in memory_ib.keys()}) + memory_data.update({"ib_%s" % k: memory_ib[k] for k in memory_ib.keys()}) # Calculate the memory for a spin_temperature call memory_st = mem_spin_temperature( @@ -192,12 +194,12 @@ def estimate_memory_lightcone( flag_options=flag_options, ) - memory_data.update({"st_%s" % k: memory_st["%s" % k] for k in memory_st.keys()}) + memory_data.update({"st_%s" % k: memory_st[k] for k in memory_st.keys()}) # Calculate the memory for a brightness_temperature call memory_bt = mem_brightness_temperature(user_params=user_params) - memory_data.update({"bt_%s" % k: memory_bt["%s" % k] for k in memory_bt.keys()}) + memory_data.update({"bt_%s" % k: memory_bt[k] for k in memory_bt.keys()}) # Now, when calculating the light-cone we always need two concurrent boxes (current and previous redshift) @@ -274,13 +276,13 @@ def mem_initial_conditions( user_params=None, ): """A function to estimate total memory usage of an initial_conditions call.""" - """Memory usage of Python InitialConditions class.""" - """All declared HII_DIM boxes""" + # Memory usage of Python InitialConditions class. + # All declared HII_DIM boxes # lowres_density, lowres_vx, lowres_vy, lowres_vz, lowres_vcb # lowres_vx_2LPT, lowres_vy_2LPT, lowres_vz_2LPT num_py_boxes_HII_DIM = 8.0 - """All declared DIM boxes""" + # All declared DIM boxes # hires_density, hires_vx, hires_vy, hires_vz # hires_vx_2LPT, hires_vy_2LPT, hires_vz_2LPT num_py_boxes_DIM = 7.0 @@ -291,14 +293,14 @@ def mem_initial_conditions( # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py - """Memory usage within GenerateICs""" + # Memory usage within GenerateICs kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 - """All declared DIM boxes""" + # All declared DIM boxes # HIRES_box, HIRES_box_saved num_c_boxes = 2 - """All declared 2LPT boxes (DIM)""" + # All declared 2LPT boxes (DIM) # phi_1 (6 components) if global_params.SECOND_ORDER_LPT_CORRECTIONS: num_c_boxes += 6 @@ -314,8 +316,8 @@ def mem_perturb_field( user_params=None, ): """A function to estimate total memory usage of a perturb_field call.""" - """Memory usage of Python PerturbedField class.""" - """All declared HII_DIM boxes""" + # Memory usage of Python PerturbedField class. + # All declared HII_DIM boxes""" # density, velocity num_py_boxes_HII_DIM = 2.0 @@ -324,7 +326,7 @@ def mem_perturb_field( # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py - """Memory usage within PerturbField.c""" + # Memory usage within PerturbField.c kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( user_params.HII_DIM @@ -365,9 +367,7 @@ def mem_ionize_box( flag_options=None, ): """A function to estimate total memory usage of an ionize_box call.""" - # First do check on inhomogeneous recombinations - astro_params = AstroParams(astro_params, INHOMO_RECO=flag_options.INHOMO_RECO) - + # Implicitly have dealt with INHOMO_RECO earlier # determine number of filtering scales (for USE_MINI_HALOS) if flag_options.USE_MINI_HALOS: n_filtering = ( @@ -387,9 +387,9 @@ def mem_ionize_box( else: n_filtering = 1 - """Memory usage of Python IonizedBox class.""" + # Memory usage of Python IonizedBox class. - """All declared HII_DIM boxes""" + # All declared HII_DIM boxes # xH_box, Gamma12_box, MFP_box, z_re_box, dNrec_box, temp_kinetic_all_gas num_py_boxes = 6.0 @@ -405,7 +405,7 @@ def mem_ionize_box( # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py - """Memory usage within IonisationBox.c""" + # Memory usage within IonisationBox.c hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( user_params.HII_DIM ) ** 2 @@ -448,9 +448,9 @@ def mem_spin_temperature( flag_options=None, ): """A function to estimate total memory usage of a spin_temperature call.""" - """Memory usage of Python IonizedBox class.""" + # Memory usage of Python IonizedBox class. - """All declared HII_DIM boxes""" + # All declared HII_DIM boxes # Ts_bx, x_e_box, Tk_box, J_21_LW_box num_py_boxes = 4.0 @@ -459,7 +459,7 @@ def mem_spin_temperature( # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py - """Memory usage within SpinTemperatureBox.c""" + # Memory usage within SpinTemperatureBox.c hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( user_params.HII_DIM ) ** 2 @@ -546,9 +546,9 @@ def mem_brightness_temperature( user_params=None, ): """A function to estimate total memory usage of a brightness_temperature call.""" - """Memory usage of Python BrightnessTemp class.""" + # Memory usage of Python BrightnessTemp class. - """All declared HII_DIM boxes""" + # All declared HII_DIM boxes # brightness_temp num_py_boxes = 1.0 @@ -557,7 +557,7 @@ def mem_brightness_temperature( # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py - """Memory usage within BrightnessTemperatureBox.c""" + # Memory usage within BrightnessTemperatureBox.c hii_tot_fft_num_pixels = ( 2.0 * (float(user_params.HII_DIM) / 2.0 + 1.0) * (user_params.HII_DIM) ** 2 ) @@ -575,9 +575,9 @@ def mem_halo_field( user_params=None, ): """A function to estimate total memory usage of a determine_halo_list call.""" - """Memory usage of Python HaloField class.""" + # Memory usage of Python HaloField class. - """All declared DIM boxes""" + # All declared DIM boxes # halo_field num_py_boxes = 1.0 @@ -586,7 +586,7 @@ def mem_halo_field( # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py - """Memory usage within FindHaloes.c""" + # Memory usage within FindHaloes.c kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 # density_field, density_field_saved @@ -620,7 +620,7 @@ def mem_perturb_halo( user_params=None, ): """A function to estimate total memory usage of a perturb_halo_list call.""" - """Memory usage of Python PerturbHaloField class.""" + # Memory usage of Python PerturbHaloField class. # We don't know a priori how many haloes that will be found, but we'll estimate the memory usage # at 10 per cent of the total number of pixels (HII_DIM, likely an over estimate) From ff4b86ac9c2bff073c21d6d418c02959ef641f2f Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 8 Apr 2021 17:15:20 +1000 Subject: [PATCH 13/28] Add interpolation table data to IonisationBox (+modify double declared array) --- src/py21cmfast/_memory.py | 66 +++++++++++++++++++++++++++--- src/py21cmfast/src/IonisationBox.c | 7 ++-- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 7a2025e98..352b82460 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -9,6 +9,18 @@ logger = logging.getLogger("21cmFAST") +# Constants defining interpolation table lengths (from C side) +# Not ideal to be here, but unlikely to ever change (names in lower case) +nmass = 300.0 +nsfr_high = 200.0 +nsfr_low = 250.0 +ngl_sfr = 100.0 +nmturn = 50.0 + +zpp_interp_points_sfr = 400.0 +dens_ninterp = 400.0 +erfc_num_points = 10000.0 + def estimate_memory_coeval( *, @@ -435,9 +447,45 @@ def mem_ionize_box( # M_coll_unfiltered, M_coll_filtered num_c_boxes += 2.0 + tables_float = tables_double = 0.0 + if flag_options.USE_MASS_DEPENDENT_ZETA: + tables_float += 2.0 * (ngl_sfr + 1) # xi_SFR, wi_SFR + + if user_params.USE_INTERPOLATION_TABLES: + tables_double += nsfr_low # log10_overdense_spline_SFR + tables_float += nsfr_high # Overdense_spline_SFR + + if flag_options.USE_MINI_HALOS: + tables_double += nsfr_low # prev_log10_overdense_spline_SFR + tables_float += nsfr_high # prev_Overdense_spline_SFR + tables_float += ( + 4.0 * nsfr_low * nmturn + ) # log10_Nion_spline, log10_Nion_spline_MINI, prev_log10_Nion_spline, prev_log10_Nion_spline_MINI + tables_float += ( + 4.0 * nsfr_high * nmturn + ) # Nion_spline, Nion_spline_MINI, prev_Nion_spline, prev_Nion_spline_MINI + else: + tables_float += nsfr_high + nsfr_low # log10_Nion_spline, Nion_spline + + if flag_options.USE_MINI_HALOS: + tables_float += 2.0 * nmturn # Mturns, Mturns_MINI + else: + tables_double += 2.0 * erfc_num_points # ERFC_VALS, ERFC_VALS_DIFF + + # These can only exist in ionisation box if spin temperature is not being computed (otherwise it exists there) + if user_params.USE_INTERPOLATION_TABLES and not flag_options.USE_TS_FLUCT: + tables_float += ( + 3.0 * nmass + ) # Mass_InterpTable, Sigma_InterpTable, dSigmadm_InterpTable + # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels + # Now add in the additional interpolation tables + size_c += (np.float32(1.0).nbytes) * tables_float + ( + np.float64(1.0).nbytes + ) * tables_double + return {"python": size_py, "c": size_c} @@ -506,12 +554,12 @@ def mem_spin_temperature( if user_params.USE_INTERPOLATION_TABLES: # log10_SFRD_z_low_table_MINI, SFRD_z_high_table_MINI (factor of 4 as these are float tables) mem_c_interp += ( - 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * 250.0 * 50.0 - ) # NSFR_low = 250, NMTURN = 50 + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * nsfr_low * nmturn + ) mem_c_interp += ( - 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * 200.0 * 50.0 - ) # NSFR_high = 200, NMTURN = 50 + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * nsfr_high * nmturn + ) else: # delNL0_rev num_c_boxes_initialised += global_params.NUM_FILTER_STEPS_FOR_Ts @@ -519,8 +567,14 @@ def mem_spin_temperature( if user_params.USE_INTERPOLATION_TABLES: # fcoll_R_grid, dfcoll_dz_grid (factor of 8. as these are double) mem_c_interp += ( - 2.0 * 8.0 * (global_params.NUM_FILTER_STEPS_FOR_Ts * 400 * 400) - ) # zpp_interp_points_SFR = 400, dens_Ninterp = 400 + 2.0 + * 8.0 + * ( + global_params.NUM_FILTER_STEPS_FOR_Ts + * zpp_interp_points_sfr + * dens_ninterp + ) + ) # dens_grid_int_vals num_c_boxes_initialised += 0.5 # 0.5 as it is a short diff --git a/src/py21cmfast/src/IonisationBox.c b/src/py21cmfast/src/IonisationBox.c index b6e5922f8..cd7678e23 100644 --- a/src/py21cmfast/src/IonisationBox.c +++ b/src/py21cmfast/src/IonisationBox.c @@ -267,9 +267,6 @@ LOG_SUPER_DEBUG("erfc interpolation done"); log10_overdense_spline_SFR = calloc(NSFR_low,sizeof(double)); Overdense_spline_SFR = calloc(NSFR_high,sizeof(float)); - log10_Nion_spline = calloc(NSFR_low,sizeof(float)); - Nion_spline = calloc(NSFR_high,sizeof(float)); - if (flag_options->USE_MINI_HALOS){ prev_log10_overdense_spline_SFR = calloc(NSFR_low,sizeof(double)); prev_Overdense_spline_SFR = calloc(NSFR_high,sizeof(float)); @@ -282,6 +279,10 @@ LOG_SUPER_DEBUG("erfc interpolation done"); prev_log10_Nion_spline_MINI = calloc(NSFR_low*NMTURN,sizeof(float)); prev_Nion_spline_MINI = calloc(NSFR_high*NMTURN,sizeof(float)); } + else { + log10_Nion_spline = calloc(NSFR_low,sizeof(float)); + Nion_spline = calloc(NSFR_high,sizeof(float)); + } } if (flag_options->USE_MINI_HALOS){ From f8045cc9724215e21e20a8f322a38a4916c61bc4 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Fri, 9 Apr 2021 15:00:48 +1000 Subject: [PATCH 14/28] Add in some final interpolation tables --- src/py21cmfast/_memory.py | 120 +++++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 20 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 352b82460..d34f654f3 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -21,6 +21,8 @@ dens_ninterp = 400.0 erfc_num_points = 10000.0 +x_int_nxhii = 14.0 + def estimate_memory_coeval( *, @@ -517,8 +519,6 @@ def mem_spin_temperature( num_c_boxes_alt = 0.0 num_c_boxes_initialised = 0.0 - mem_c_interp = 0.0 - if flag_options.USE_MINI_HALOS: # log10_Mcrit_LW_unfiltered, log10_Mcrit_LW_filtered num_c_boxes += 2.0 @@ -526,8 +526,17 @@ def mem_spin_temperature( # log10_Mcrit_LW num_c_boxes_alt += global_params.NUM_FILTER_STEPS_FOR_Ts - # There are a bunch of 1 and 2D interpolation tables, but ignore those as they are small relative to 3D grids - # I will add any that could end up considerably large. + # Now go through and add in all the interpolation table information + tables_float = tables_double = 0.0 + + tables_float += global_params.NUM_FILTER_STEPS_FOR_Ts # R_values + tables_double += 2.0 * global_params.NUM_FILTER_STEPS_FOR_Ts # zpp_edge, sigma_atR + + if user_params.USE_INTERPOLATION_TABLES: + tables_float += ( + 2.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # min_densities, max_densities + tables_float += zpp_interp_points_sfr # zpp_interp_table if flag_options.USE_MASS_DEPENDENT_ZETA: # delNL0 @@ -541,6 +550,18 @@ def mem_spin_temperature( # dxheat_dt_box, dxion_source_dt_box, dxlya_dt_box, dstarlya_dt_box num_c_boxes_initialised += 2.0 * 4.0 # factor of 2. as these are doubles + tables_float += global_params.NUM_FILTER_STEPS_FOR_Ts # SFR_timescale_factor + tables_double += 2.0 * (ngl_sfr + 1.0) # xi_SFR_Xray, wi_SFR_Xray + + if user_params.USE_INTERPOLATION_TABLES: + tables_double += ( + nsfr_low + nsfr_high + ) # overdense_low_table, overdense_high_table + + tables_float += global_params.NUM_FILTER_STEPS_FOR_Ts * ( + nsfr_low + nsfr_high + ) # log10_SFRD_z_low_table, SFRD_z_high_table + if flag_options.USE_MINI_HALOS: # del_fcoll_Rct_MINI num_c_boxes_initialised += 1.0 @@ -551,34 +572,90 @@ def mem_spin_temperature( 2.0 * 6.0 ) # factor of 2. taking into account that these are doubles + tables_double += ( + global_params.NUM_FILTER_STEPS_FOR_Ts + ) # log10_Mcrit_LW_ave_list + if user_params.USE_INTERPOLATION_TABLES: - # log10_SFRD_z_low_table_MINI, SFRD_z_high_table_MINI (factor of 4 as these are float tables) - mem_c_interp += ( - 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * nsfr_low * nmturn + # log10_SFRD_z_low_table_MINI, SFRD_z_high_table_MINI + tables_float += ( + global_params.NUM_FILTER_STEPS_FOR_Ts + * nmturn + * (nsfr_low + nsfr_high) ) - mem_c_interp += ( - 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * nsfr_high * nmturn - ) else: # delNL0_rev num_c_boxes_initialised += global_params.NUM_FILTER_STEPS_FOR_Ts if user_params.USE_INTERPOLATION_TABLES: - # fcoll_R_grid, dfcoll_dz_grid (factor of 8. as these are double) - mem_c_interp += ( - 2.0 - * 8.0 - * ( - global_params.NUM_FILTER_STEPS_FOR_Ts - * zpp_interp_points_sfr - * dens_ninterp - ) + tables_double += zpp_interp_points_sfr # Sigma_Tmin_grid + + # fcoll_R_grid, dfcoll_dz_grid + tables_double += 2.0 * ( + global_params.NUM_FILTER_STEPS_FOR_Ts + * zpp_interp_points_sfr + * dens_ninterp ) + tables_double += ( + 6.0 * global_params.NUM_FILTER_STEPS_FOR_Ts * dens_ninterp + ) # grid_dens, density_gridpoints, fcoll_interp1, fcoll_interp2, dfcoll_interp1, dfcoll_interp2 + tables_double += zpp_interp_points_sfr # ST_over_PS_arg_grid + + tables_float += ( + 7.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # delNL0_bw, delNL0_Offset, delNL0_LL, delNL0_UL, delNL0_ibw, log10delNL0_diff, log10delNL0_diff_UL + # dens_grid_int_vals num_c_boxes_initialised += 0.5 # 0.5 as it is a short + tables_double += global_params.NUM_FILTER_STEPS_FOR_Ts # dstarlya_dt_prefactor + if flag_options.USE_MINI_HALOS: + tables_double += ( + 3.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # dstarlya_dt_prefactor_MINI, dstarlyLW_dt_prefactor, dstarlyLW_dt_prefactor_MINI + + tables_double += ( + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # ST_over_PS_MINI, sum_lyn_MINI, sum_lyLWn, sum_lyLWn_MINI, + tables_float += global_params.NUM_FILTER_STEPS_FOR_Ts # Mcrit_atom_interp_table + + tables_float += ( + 1.5 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # zpp_for_evolve_list, SingleVal_int (0.5 as it is a short) + + tables_double += ( + 6.0 * x_int_nxhii * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # freq_int_heat_tbl, freq_int_ion_tbl, freq_int_lya_tbl, freq_int_heat_tbl_diff, freq_int_ion_tbl_diff, freq_int_lya_tbl_diff + + tables_double += ( + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # fcoll_R_array, sigma_Tmin, ST_over_PS, sum_lyn + tables_float += ( + x_int_nxhii + global_params.NUM_FILTER_STEPS_FOR_Ts + ) # inverse_diff, zpp_growth + + if user_params.USE_MASS_DEPENDENT_ZETA: + if flag_options.USE_MINI_HALOS: + tables_double += ( + 4.0 * zpp_interp_points_sfr + 2.0 * zpp_interp_points_sfr * nmturn + ) # z_val, Nion_z_val, Nion_z_val_MINI, z_X_val, SFRD_val, SFRD_val_MINI + else: + tables_double += ( + 4.0 * zpp_interp_points_sfr + ) # z_val, Nion_z_val, z_X_val, SFRD_val + else: + # This is dependent on the user defined input, that cannot be captured here. + # However, approximating it as ~ 1000 (almost certainly an overestimate) should be sufficient for almost all uses. + tables_double += 1000.0 # FgtrM_1DTable_linear + + # These supersede usage in IonisationBox + if user_params.USE_INTERPOLATION_TABLES: + tables_float += ( + 3.0 * nmass + ) # Mass_InterpTable, Sigma_InterpTable, dSigmadm_InterpTable + # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels @@ -590,7 +667,10 @@ def mem_spin_temperature( * (user_params.HII_DIM ** 3.0) ) - size_c_init += mem_c_interp + # Now, add all the table data (which are kept throughout the calculation) + size_c_init += (np.float32(1.0).nbytes) * tables_float + ( + np.float64(1.0).nbytes + ) * tables_double return {"python": size_py, "c_init": size_c_init, "c_per_z": size_c} From 4bcc750dce126abdab0e9414d20481ddb12b535a Mon Sep 17 00:00:00 2001 From: BradGreig Date: Fri, 9 Apr 2021 16:54:36 +1000 Subject: [PATCH 15/28] Add in memory from generating a large number of ints (shuffling in RNG) --- src/py21cmfast/_memory.py | 11 ++++++++++- src/py21cmfast/src/GenerateICs.c | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index d34f654f3..00d28735a 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -322,6 +322,15 @@ def mem_initial_conditions( # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * kspace_num_pixels + # Storage of large number of integers (for seeding in multithreading) + # Note, this is somewhat hard coded as it is C compiler/architecture specific (INT_MAX) + size_c_RNG = ( + 134217727.0 * 4.0 + ) # First, is INT_MAX/16, factor of 4. is for an unsigned int + + if size_c_RNG > size_c: + size_c = size_c_RNG + return {"python": size_py, "c": size_c} @@ -636,7 +645,7 @@ def mem_spin_temperature( x_int_nxhii + global_params.NUM_FILTER_STEPS_FOR_Ts ) # inverse_diff, zpp_growth - if user_params.USE_MASS_DEPENDENT_ZETA: + if flag_options.USE_MASS_DEPENDENT_ZETA: if flag_options.USE_MINI_HALOS: tables_double += ( 4.0 * zpp_interp_points_sfr + 2.0 * zpp_interp_points_sfr * nmturn diff --git a/src/py21cmfast/src/GenerateICs.c b/src/py21cmfast/src/GenerateICs.c index 887805190..90f473689 100644 --- a/src/py21cmfast/src/GenerateICs.c +++ b/src/py21cmfast/src/GenerateICs.c @@ -787,6 +787,7 @@ int ComputeInitialConditions( gsl_rng_free (r[i]); } gsl_rng_free(rseed); + LOG_DEBUG("Cleaned Up."); } // End of Try{} From 7ed89f17c1eae6e45ace0a4773533391b62d99ab Mon Sep 17 00:00:00 2001 From: BradGreig Date: Fri, 9 Apr 2021 19:32:02 +1000 Subject: [PATCH 16/28] Add in recombination tables --- src/py21cmfast/_memory.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 00d28735a..bc9521d91 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -22,6 +22,8 @@ erfc_num_points = 10000.0 x_int_nxhii = 14.0 +rr_z_npts = 300.0 +rr_lngamma_npts = 250.0 def estimate_memory_coeval( @@ -483,6 +485,11 @@ def mem_ionize_box( else: tables_double += 2.0 * erfc_num_points # ERFC_VALS, ERFC_VALS_DIFF + if flag_options.INHOMO_RECO: + tables_double += ( + rr_z_npts * rr_lngamma_npts + rr_lngamma_npts + ) # RR_table, lnGamma_values + # These can only exist in ionisation box if spin temperature is not being computed (otherwise it exists there) if user_params.USE_INTERPOLATION_TABLES and not flag_options.USE_TS_FLUCT: tables_float += ( From b04faa239ee06e9a4f41c17b7b75d35a7bc66c33 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Sat, 10 Apr 2021 14:20:42 +1000 Subject: [PATCH 17/28] Add some description to the function to estimate memory usage for a light-cone --- src/py21cmfast/_memory.py | 57 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index bc9521d91..4b58bc323 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -74,7 +74,62 @@ def estimate_memory_lightcone( flag_options=None, lightcone_quantities=("brightness_temp",), ): - """Compute an estimate of the requisite memory needed by the user for a run_lightcone call.""" + r""" + Compute an estimate of the requisite memory needed by the user for a run_lightcone call. + + Note, this is an upper-limit as it assumes all requisite data needs to be generated. + Actual memory usage may be less than this estimate if using pre-computed data. + + Parameters + ---------- + redshift : float + The minimum redshift of the lightcone. + max_redshift : float, optional + The maximum redshift at which to keep lightcone information. By default, this is equal to + `z_heat_max`. Note that this is not *exact*, but will be typically slightly exceeded. + user_params : `~UserParams`, optional + Defines the overall options and parameters of the run. + astro_params : :class:`~AstroParams`, optional + Defines the astrophysical parameters of the run. + cosmo_params : :class:`~CosmoParams`, optional + Defines the cosmological parameters used to compute initial conditions. + flag_options : :class:`~FlagOptions`, optional + Options concerning how the reionization process is run, eg. if spin temperature + fluctuations are required. + lightcone_quantities : tuple of str, optional + The quantities to form into a lightcone. By default, just the brightness + temperature. Note that these quantities must exist in one of the output + structures: + + * :class:`~InitialConditions` + * :class:`~PerturbField` + * :class:`~TsBox` + * :class:`~IonizedBox` + * :class:`BrightnessTemp` + + To get a full list of possible quantities, run :func:`get_all_fieldnames`. + + Returns + ------- + dict : + ics_python: Estimate of python allocated memory for initial conditions (in Bytes) + ics_c: Estimate of C allocated memory for initial conditions (in Bytes) + pf_python: Estimate of python allocated memory for a single perturb field (in Bytes) + pf_c: Estimate of C allocated memory for a single perturb field (in Bytes) + hf_python: Estimate of python allocated memory for determine halo list (in Bytes) + hf_c: Estimate of C allocated memory for determine halo list (in Bytes) + phf_python: Estimate of python allocated memory for a single perturb halo list (in Bytes) + phf_c: Estimate of C allocated memory for a single perturb halo list (in Bytes) + ib_python: Estimate of python allocated memory for a single ionized box (in Bytes) + ib_c: Estimate of C allocated memory for a single ionized box (in Bytes) + st_python: Estimate of python allocated memory for a single spin temperature box (in Bytes) + st_c_init: Estimate of retained memory for any spin temperature box (in Bytes) + st_c_per_z: Estimate of C allocated memory for a single spin temperature box (in Bytes) + bt_python: Estimate of python allocated memory for a single brightness temperature box (in Bytes) + bt_c: Estimate of C allocated memory for a single brightness temperature box (in Bytes) + peak_memory: As estimate of the peak memory usage for running a lightcone (generating all data) (in Bytes) + + """ # Deal with AstroParams and INHOMO_RECO astro_params = AstroParams(astro_params, INHOMO_RECO=flag_options.INHOMO_RECO) From 698b14bff35cb8719c8ba9534b977553ceb72b82 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Sat, 10 Apr 2021 16:51:09 +1000 Subject: [PATCH 18/28] Add memory function for running a coeval box --- src/py21cmfast/_memory.py | 240 +++++++++++++++++++++++++++++++++++--- 1 file changed, 223 insertions(+), 17 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 4b58bc323..47e2f4cd7 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -28,40 +28,245 @@ def estimate_memory_coeval( *, + redshift=None, user_params=None, cosmo_params=None, astro_params=None, flag_options=None, ): - """Compute an estimate of the requisite memory needed by the user for a run_coeval call.""" + r""" + Compute an estimate of the requisite memory needed by the user for a run_coeval call. + + Note, this is an upper-limit as it assumes all requisite data needs to be generated. + Actual memory usage may be less than this estimate if using pre-computed data. + + Parameters + ---------- + redshift : array_like + A single redshift, or multiple redshift, at which to return results. The minimum of these + will define the log-scrolling behaviour (if necessary). + user_params : `~UserParams`, optional + Defines the overall options and parameters of the run. + astro_params : :class:`~AstroParams`, optional + Defines the astrophysical parameters of the run. + cosmo_params : :class:`~CosmoParams`, optional + Defines the cosmological parameters used to compute initial conditions. + flag_options : :class:`~FlagOptions`, optional + Options concerning how the reionization process is run, eg. if spin temperature + fluctuations are required. + + Returns + ------- + dict : + ics_python: Estimate of python allocated memory for initial conditions (in Bytes) + ics_c: Estimate of C allocated memory for initial conditions (in Bytes) + pf_python: Estimate of python allocated memory for a single perturb field (in Bytes) + pf_c: Estimate of C allocated memory for a single perturb field (in Bytes) + hf_python: Estimate of python allocated memory for determine halo list (in Bytes) + hf_c: Estimate of C allocated memory for determine halo list (in Bytes) + phf_python: Estimate of python allocated memory for a single perturb halo list (in Bytes) + phf_c: Estimate of C allocated memory for a single perturb halo list (in Bytes) + ib_python: Estimate of python allocated memory for a single ionized box (in Bytes) + ib_c: Estimate of C allocated memory for a single ionized box (in Bytes) + st_python: Estimate of python allocated memory for a single spin temperature box (in Bytes) + st_c_init: Estimate of retained memory for any spin temperature box (in Bytes) + st_c_per_z: Estimate of C allocated memory for a single spin temperature box (in Bytes) + bt_python: Estimate of python allocated memory for a single brightness temperature box (in Bytes) + bt_c: Estimate of C allocated memory for a single brightness temperature box (in Bytes) + peak_memory: As estimate of the peak memory usage for running a lightcone (generating all data) (in Bytes) + """ - # Initial conditions - mem_initial_conditions(user_params=self.user_params) + # Deal with AstroParams and INHOMO_RECO + astro_params = AstroParams(astro_params, INHOMO_RECO=flag_options.INHOMO_RECO) - # Perturb field + # First, calculate the memory usage for the initial conditions + memory_ics = mem_initial_conditions(user_params=user_params) - # Halo field - if flag_options.USE_HALO_FIELD: + memory_data = {"ics_%s" % k: memory_ics[k] for k in memory_ics.keys()} + + # Maximum memory while running ICs + peak_memory = memory_ics["c"] + memory_ics["python"] + + # Now the perturb field + memory_pf = mem_perturb_field(user_params=user_params) + + memory_data.update({"pf_%s" % k: memory_pf[k] for k in memory_pf.keys()}) - # Photon non-conservation + # If we are using the photon non-conservation correction if flag_options.PHOTON_CONS: + # First need to create new structs for photon the photon-conservation + astro_params_photoncons = deepcopy(astro_params) - # Note, if any of below are set the require current and - # previous boxes, thus need to be careful + flag_options_photoncons = FlagOptions( + USE_MASS_DEPENDENT_ZETA=flag_options.USE_MASS_DEPENDENT_ZETA, + M_MIN_in_Mass=flag_options.M_MIN_in_Mass, + USE_VELS_AUX=user_params.USE_RELATIVE_VELOCITIES, + ) - # Spin temperature - if flag_options.USE_TS_FLUCT: + # First perturb_field + memory_pf = mem_perturb_field(user_params=user_params) + + # First ionize_box + memory_ib = mem_ionize_box( + user_params=user_params, + astro_params=astro_params_photoncons, + flag_options=flag_options_photoncons, + ) + + # As we iterate through we storing the python memory of two + # perturb_field and ionize_boxes plus the C memory of either + # of the ionize_box or perturb field as it is being calculated + peak_memory_photoncons = memory_ics[ + "python" + ] # We have the initial conditions in memory + peak_memory_photoncons += 2 * ( + memory_pf["python"] + memory_ib["python"] + ) # The python memory + peak_memory_photoncons += ( + memory_pf["c"] if memory_pf["c"] > memory_ib["c"] else memory_ib["c"] + ) # Maximum C memory, as it is freed after usage + + # Check if the memory required to do the photon non-conservation correction exceeds + # current peak memory usage + peak_memory = ( + peak_memory + if peak_memory > peak_memory_photoncons + else peak_memory_photoncons + ) + + # Determine the number of redshifts passed + if not hasattr(redshift, "__len__"): + redshift = [redshift] + + n_redshifts = len(redshift) + + current_memory = memory_ics["python"] # We have the initial conditions in memory + current_memory += ( + n_redshifts * memory_pf["python"] + ) # Python memory for all perturb fields + current_memory += memory_pf["c"] # Plus C memory for a single perturb field + + # Check if running perturb_field requires more memory than generating ICs + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Now start generating estimates for all other data products + + # Calculate the memory for a determine_halo_list call + memory_hf = mem_halo_field(user_params=user_params) + + memory_data.update({"hf_%s" % k: memory_hf[k] for k in memory_hf.keys()}) + + # Calculate the memory for a perturb_halo_list call + memory_phf = mem_perturb_halo(user_params=user_params) + + memory_data.update({"phf_%s" % k: memory_phf[k] for k in memory_phf.keys()}) + + # Calculate the memory for an ionize_box call + memory_ib = mem_ionize_box( + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + + memory_data.update({"ib_%s" % k: memory_ib[k] for k in memory_ib.keys()}) + + # Calculate the memory for a spin_temperature call + memory_st = mem_spin_temperature( + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + + memory_data.update({"st_%s" % k: memory_st[k] for k in memory_st.keys()}) + + # Calculate the memory for a brightness_temperature call + memory_bt = mem_brightness_temperature(user_params=user_params) + + memory_data.update({"bt_%s" % k: memory_bt[k] for k in memory_bt.keys()}) + + # All the data kept in memory at this point in Python + stored_memory = memory_ics["python"] # We have the initial conditions in memory + stored_memory += ( + n_redshifts * memory_pf["python"] + ) # Python memory for all perturb fields + + current_memory = stored_memory + + # If using haloes, then need to add all of these too. + if flag_options.USE_HALO_FIELD: + current_memory += n_redshifts * memory_phf["python"] + memory_hf["python"] + + if memory_phf["c"] > memory_hf["c"]: + current_memory += memory_phf["c"] + else: + current_memory += memory_hf["c"] + + # Check if running perturb_halos requires more memory than generating ICs + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # We keep all the python memory for the haloes + stored_memory += n_redshifts * memory_phf["python"] + memory_hf["python"] - # Mini-halos if flag_options.USE_MINI_HALOS: + # If using minihaloes we (might) need 2 perturbs, so two python + one in C + current_memory = stored_memory + 2.0 * memory_pf["python"] + memory_pf["c"] - # Inhomogeneous recombinations - if flag_options.INHOMO_RECO: + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + stored_memory += 2.0 * memory_pf["python"] # We (might) need two perturb's - # Output a summary of information in human readable format - """ - return {} + # Add the two ionized boxes + stored_memory += 2.0 * memory_ib["python"] + + # Add (if necessary) the two spin temperature boxes + if flag_options.USE_TS_FLUCT: + stored_memory += 2.0 * memory_st["python"] + + # We also have initialied C memory that is retained until the end of the calculation + stored_memory += memory_st["c_init"] + + # Add the two brightness_temperature boxes + stored_memory += 2.0 * memory_bt["python"] + + # Now we have an estimate for the data retained in memory, now we just need to check + # the peak memory usage (which includes the additional C memory that is allocated + # and then freed on a single redshift call) + # First check perturb field + current_memory = stored_memory + memory_pf["c"] # Add the temporary C memory + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Check spin temperature + if flag_options.USE_TS_FLUCT: + current_memory = stored_memory + memory_st["c_per_z"] + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Check ionized box + current_memory = stored_memory + memory_ib["c"] + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + # Check brightness temperature + current_memory = stored_memory + memory_bt["c"] + + # Check if our peak memory usage has been exceeded + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + memory_data.update({"peak_memory": peak_memory}) + + format_output( + memory_data=memory_data, + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + + return memory_data def estimate_memory_lightcone( @@ -127,6 +332,7 @@ def estimate_memory_lightcone( st_c_per_z: Estimate of C allocated memory for a single spin temperature box (in Bytes) bt_python: Estimate of python allocated memory for a single brightness temperature box (in Bytes) bt_c: Estimate of C allocated memory for a single brightness temperature box (in Bytes) + python_lc: Estimate of python allocated memory for all provided lightcones (in Bytes) peak_memory: As estimate of the peak memory usage for running a lightcone (generating all data) (in Bytes) """ From 8505fc0678a1e1bebbb432901ffa1db74c0383f3 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Sat, 10 Apr 2021 18:09:12 +1000 Subject: [PATCH 19/28] Added function for estimating ics and perturb field. Modified function to output information --- src/py21cmfast/__init__.py | 1 + src/py21cmfast/_memory.py | 148 ++++++++++++++++++++++++++++++++----- 2 files changed, 130 insertions(+), 19 deletions(-) diff --git a/src/py21cmfast/__init__.py b/src/py21cmfast/__init__.py index a0a6df2c5..4588a0ef0 100644 --- a/src/py21cmfast/__init__.py +++ b/src/py21cmfast/__init__.py @@ -17,6 +17,7 @@ from . import cache_tools, inputs, outputs, plotting, wrapper from ._cfg import config from ._logging import configure_logging +from ._memory import print_memory_estimate from .cache_tools import query_cache from .outputs import ( Coeval, diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 47e2f4cd7..e112dc219 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -259,7 +259,7 @@ def estimate_memory_coeval( memory_data.update({"peak_memory": peak_memory}) - format_output( + print_memory_estimate( memory_data=memory_data, user_params=user_params, astro_params=astro_params, @@ -538,7 +538,7 @@ def estimate_memory_lightcone( memory_data.update({"peak_memory": peak_memory}) - format_output( + print_memory_estimate( memory_data=memory_data, user_params=user_params, astro_params=astro_params, @@ -548,6 +548,98 @@ def estimate_memory_lightcone( return memory_data +def estimate_memory_ics( + *, + user_params=None, +): + r""" + Compute an estimate of the requisite memory needed just for the initial conditions. + + Note, this is an upper-limit + + Parameters + ---------- + user_params : `~UserParams`, optional + Defines the overall options and parameters of the run. + + Returns + ------- + dict : + ics_python: Estimate of python allocated memory for initial conditions (in Bytes) + ics_c: Estimate of C allocated memory for initial conditions (in Bytes) + peak_memory: As estimate of the peak memory usage for running a lightcone (generating all data) (in Bytes) + """ + # Calculate the memory usage for the initial conditions + memory_ics = mem_initial_conditions(user_params=user_params) + + memory_data = {"ics_%s" % k: memory_ics[k] for k in memory_ics.keys()} + + # Maximum memory while running ICs + peak_memory = memory_ics["c"] + memory_ics["python"] + + memory_data.update({"peak_memory": peak_memory}) + + print_memory_estimate( + memory_data=memory_data, + user_params=user_params, + ) + + return memory_data + + +def estimate_memory_perturb( + *, + user_params=None, +): + r""" + Compute an estimate of the requisite memory needed by the user for a single perturb_field call. + + Note, this is an upper-limit + + Parameters + ---------- + user_params : `~UserParams`, optional + Defines the overall options and parameters of the run. + + Returns + ------- + dict : + ics_python: Estimate of python allocated memory for initial conditions (in Bytes) + ics_c: Estimate of C allocated memory for initial conditions (in Bytes) + pf_python: Estimate of python allocated memory for a single perturb field (in Bytes) + pf_c: Estimate of C allocated memory for a single perturb field (in Bytes) + peak_memory: As estimate of the peak memory usage for running a lightcone (generating all data) (in Bytes) + + """ + # First, calculate the memory usage for the initial conditions + memory_ics = mem_initial_conditions(user_params=user_params) + + memory_data = {"ics_%s" % k: memory_ics[k] for k in memory_ics.keys()} + + # Maximum memory while running ICs + peak_memory = memory_ics["c"] + memory_ics["python"] + + # Now the perturb field + memory_pf = mem_perturb_field(user_params=user_params) + + memory_data.update({"pf_%s" % k: memory_pf[k] for k in memory_pf.keys()}) + + # Stored ICs in python + allocated C and Python memory for perturb_field + current_memory = memory_ics["python"] + memory_pf["python"] + memory_pf["c"] + + # Check if running perturb_field requires more memory than generating ICs + peak_memory = peak_memory if peak_memory > current_memory else current_memory + + memory_data.update({"peak_memory": peak_memory}) + + print_memory_estimate( + memory_data=memory_data, + user_params=user_params, + ) + + return memory_data + + def mem_initial_conditions( *, user_params=None, @@ -1042,7 +1134,7 @@ def mem_perturb_halo( return {"python": 0.0, "c": size_c} -def format_output( +def print_memory_estimate( *, memory_data=None, user_params=None, @@ -1055,12 +1147,18 @@ def format_output( print("") if "python_lc" in memory_data.keys(): print("Memory info for run_lightcone") + elif len(memory_data) == 3: + print("Memory info for initial_conditions") + elif len(memory_data) == 5: + print("Memory info for perturb_field") else: print("Memory info for run_coeval") print("") print("%s" % (user_params)) - print("%s" % (astro_params)) - print("%s" % (flag_options)) + if astro_params is not None: + print("%s" % (astro_params)) + if flag_options is not None: + print("%s" % (flag_options)) print("") print("Peak memory usage: %g (GB)" % (memory_data["peak_memory"] / bytes_in_gb)) print("") @@ -1074,11 +1172,15 @@ def format_output( "Memory for ICs: %g (GB; Python) %g (GB; C)" % (memory_data["ics_python"] / bytes_in_gb, memory_data["ics_c"] / bytes_in_gb) ) - print( - "Memory for single perturbed field: %g (GB; Python) %g (GB; C)" - % (memory_data["pf_python"] / bytes_in_gb, memory_data["pf_c"] / bytes_in_gb) - ) - if flag_options.USE_HALO_FIELD: + if "pf_python" in memory_data.keys(): + print( + "Memory for single perturbed field: %g (GB; Python) %g (GB; C)" + % ( + memory_data["pf_python"] / bytes_in_gb, + memory_data["pf_c"] / bytes_in_gb, + ) + ) + if "hf_python" in memory_data.keys() and flag_options.USE_HALO_FIELD: print( "Note these are approximations as we don't know a priori how many haloes there are (assume 10 per cent of volume)" ) @@ -1096,12 +1198,16 @@ def format_output( memory_data["phf_c"] / bytes_in_gb, ) ) + if "ib_python" in memory_data.keys(): + print( + "Memory for single ionized box: %g (GB; Python) %g (GB; C)" + % ( + memory_data["ib_python"] / bytes_in_gb, + memory_data["ib_c"] / bytes_in_gb, + ) + ) - print( - "Memory for single ionized box: %g (GB; Python) %g (GB; C)" - % (memory_data["ib_python"] / bytes_in_gb, memory_data["ib_c"] / bytes_in_gb) - ) - if flag_options.USE_TS_FLUCT: + if "st_python" in memory_data.keys() and flag_options.USE_TS_FLUCT: print( "Memory for single spin temperature box: %g (GB; Python) %g (GB; C per z) %g (GB; C retained)" % ( @@ -1110,8 +1216,12 @@ def format_output( memory_data["st_c_init"] / bytes_in_gb, ) ) - print( - "Memory for single brightness temperature box: %g (GB; Python) %g (GB; C)" - % (memory_data["bt_python"] / bytes_in_gb, memory_data["bt_c"] / bytes_in_gb) - ) + if "bt_python" in memory_data.keys(): + print( + "Memory for single brightness temperature box: %g (GB; Python) %g (GB; C)" + % ( + memory_data["bt_python"] / bytes_in_gb, + memory_data["bt_c"] / bytes_in_gb, + ) + ) print("") From 4ba2907118d7a06db108bd6da95f1c579541349a Mon Sep 17 00:00:00 2001 From: BradGreig Date: Mon, 12 Apr 2021 11:37:38 +1000 Subject: [PATCH 20/28] Add a buffer to the peak memory usage corresponding to a rough estimate of unaccounted for usage (e.g declared variables) --- src/py21cmfast/_memory.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index e112dc219..c4aff503f 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -257,6 +257,11 @@ def estimate_memory_coeval( # Check if our peak memory usage has been exceeded peak_memory = peak_memory if peak_memory > current_memory else current_memory + # I have been finding that remaining memory (variables etc.) consistently appear + # to contribute about 0.1 Gb. So I'll add this amount. Only important for small + # box sizes. + peak_memory += 0.1 * 1024 ** 3 + memory_data.update({"peak_memory": peak_memory}) print_memory_estimate( @@ -536,6 +541,11 @@ def estimate_memory_lightcone( # Check if our peak memory usage has been exceeded peak_memory = peak_memory if peak_memory > current_memory else current_memory + # I have been finding that remaining memory (variables etc.) consistently appear + # to contribute about 0.1 Gb. So I'll add this amount. Only important for small + # box sizes. + peak_memory += 0.1 * 1024 ** 3 + memory_data.update({"peak_memory": peak_memory}) print_memory_estimate( @@ -577,6 +587,11 @@ def estimate_memory_ics( # Maximum memory while running ICs peak_memory = memory_ics["c"] + memory_ics["python"] + # I have been finding that remaining memory (variables etc.) consistently appear + # to contribute about 0.1 Gb. So I'll add this amount. Only important for small + # box sizes. + peak_memory += 0.1 * 1024 ** 3 + memory_data.update({"peak_memory": peak_memory}) print_memory_estimate( @@ -630,6 +645,11 @@ def estimate_memory_perturb( # Check if running perturb_field requires more memory than generating ICs peak_memory = peak_memory if peak_memory > current_memory else current_memory + # I have been finding that remaining memory (variables etc.) consistently appear + # to contribute about 0.1 Gb. So I'll add this amount. Only important for small + # box sizes. + peak_memory += 0.1 * 1024 ** 3 + memory_data.update({"peak_memory": peak_memory}) print_memory_estimate( From 97c3d8a735f623f680dfd86cfdd324ee0348fe05 Mon Sep 17 00:00:00 2001 From: Steven Murray Date: Thu, 15 Sep 2022 12:16:28 -0700 Subject: [PATCH 21/28] main: add gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7e2649731..8805bdc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ valgrind* c2r* r2c* +.vscode/ \ No newline at end of file From b34b114bf6171b44d16711f74f985c3fafc484b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:17:18 +0000 Subject: [PATCH 22/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/py21cmfast/_memory.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index c4aff503f..7793f2fcd 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -260,7 +260,7 @@ def estimate_memory_coeval( # I have been finding that remaining memory (variables etc.) consistently appear # to contribute about 0.1 Gb. So I'll add this amount. Only important for small # box sizes. - peak_memory += 0.1 * 1024 ** 3 + peak_memory += 0.1 * 1024**3 memory_data.update({"peak_memory": peak_memory}) @@ -544,7 +544,7 @@ def estimate_memory_lightcone( # I have been finding that remaining memory (variables etc.) consistently appear # to contribute about 0.1 Gb. So I'll add this amount. Only important for small # box sizes. - peak_memory += 0.1 * 1024 ** 3 + peak_memory += 0.1 * 1024**3 memory_data.update({"peak_memory": peak_memory}) @@ -590,7 +590,7 @@ def estimate_memory_ics( # I have been finding that remaining memory (variables etc.) consistently appear # to contribute about 0.1 Gb. So I'll add this amount. Only important for small # box sizes. - peak_memory += 0.1 * 1024 ** 3 + peak_memory += 0.1 * 1024**3 memory_data.update({"peak_memory": peak_memory}) @@ -648,7 +648,7 @@ def estimate_memory_perturb( # I have been finding that remaining memory (variables etc.) consistently appear # to contribute about 0.1 Gb. So I'll add this amount. Only important for small # box sizes. - peak_memory += 0.1 * 1024 ** 3 + peak_memory += 0.1 * 1024**3 memory_data.update({"peak_memory": peak_memory}) @@ -1048,12 +1048,12 @@ def mem_spin_temperature( # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels - size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.HII_DIM ** 3.0) + size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.HII_DIM**3.0) size_c_init = ( (np.float32(1.0).nbytes) * num_c_boxes_initialised - * (user_params.HII_DIM ** 3.0) + * (user_params.HII_DIM**3.0) ) # Now, add all the table data (which are kept throughout the calculation) @@ -1162,7 +1162,7 @@ def print_memory_estimate( flag_options=None, ): """Function to output information in a manageable format.""" - bytes_in_gb = 1024 ** 3 + bytes_in_gb = 1024**3 print("") if "python_lc" in memory_data.keys(): From d71546183bd10c65484011b69a3b09a2760115e8 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Wed, 17 May 2023 15:49:53 +1000 Subject: [PATCH 23/28] Update ICs memory function --- src/py21cmfast/_memory.py | 192 ++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 99 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 7793f2fcd..2c26ee7d6 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -5,6 +5,7 @@ from copy import deepcopy from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params +from .outputs import InitialConditions from .wrapper import _logscroll_redshifts, _setup_lightcone logger = logging.getLogger("21cmFAST") @@ -666,33 +667,26 @@ def mem_initial_conditions( ): """A function to estimate total memory usage of an initial_conditions call.""" # Memory usage of Python InitialConditions class. - # All declared HII_DIM boxes - # lowres_density, lowres_vx, lowres_vy, lowres_vz, lowres_vcb - # lowres_vx_2LPT, lowres_vy_2LPT, lowres_vz_2LPT - num_py_boxes_HII_DIM = 8.0 - - # All declared DIM boxes - # hires_density, hires_vx, hires_vy, hires_vz - # hires_vx_2LPT, hires_vy_2LPT, hires_vz_2LPT - num_py_boxes_DIM = 7.0 - - size_py = num_py_boxes_DIM * (user_params.DIM) ** 3 - size_py += num_py_boxes_HII_DIM * (user_params.HII_DIM) ** 3 + init = InitialConditions(user_params=user_params) + size_py = 0 + for key in init._array_structure: + size_py += np.prod(init._array_structure[key]) # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py # Memory usage within GenerateICs - kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 + kspace_num_pixels = ( + float(user_params.DIM) * user_params.NON_CUBIC_FACTOR / 2.0 + 1.0 + ) * (user_params.DIM) ** 2 # All declared DIM boxes # HIRES_box, HIRES_box_saved num_c_boxes = 2 # All declared 2LPT boxes (DIM) - # phi_1 (6 components) - if global_params.SECOND_ORDER_LPT_CORRECTIONS: - num_c_boxes += 6 + if user_params.USE_2LPT: + num_c_boxes += 1 # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * kspace_num_pixels @@ -1162,86 +1156,86 @@ def print_memory_estimate( flag_options=None, ): """Function to output information in a manageable format.""" - bytes_in_gb = 1024**3 - - print("") - if "python_lc" in memory_data.keys(): - print("Memory info for run_lightcone") - elif len(memory_data) == 3: - print("Memory info for initial_conditions") - elif len(memory_data) == 5: - print("Memory info for perturb_field") - else: - print("Memory info for run_coeval") - print("") - print("%s" % (user_params)) - if astro_params is not None: - print("%s" % (astro_params)) - if flag_options is not None: - print("%s" % (flag_options)) - print("") - print("Peak memory usage: %g (GB)" % (memory_data["peak_memory"] / bytes_in_gb)) - print("") - if "python_lc" in memory_data.keys(): - print( - "Memory for stored lightcones: %g (GB)" - % (memory_data["python_lc"] / bytes_in_gb) - ) - """logger.info("Peak memory usage: %g (GB)"%(memory_data['peak_memory']/bytes_in_gb))""" - print( - "Memory for ICs: %g (GB; Python) %g (GB; C)" - % (memory_data["ics_python"] / bytes_in_gb, memory_data["ics_c"] / bytes_in_gb) - ) - if "pf_python" in memory_data.keys(): - print( - "Memory for single perturbed field: %g (GB; Python) %g (GB; C)" - % ( - memory_data["pf_python"] / bytes_in_gb, - memory_data["pf_c"] / bytes_in_gb, - ) - ) - if "hf_python" in memory_data.keys() and flag_options.USE_HALO_FIELD: - print( - "Note these are approximations as we don't know a priori how many haloes there are (assume 10 per cent of volume)" - ) - print( - "Memory for generating halo list: %g (GB; Python) %g (GB; C)" - % ( - memory_data["hf_python"] / bytes_in_gb, - memory_data["hf_c"] / bytes_in_gb, - ) - ) - print( - "Memory for perturbing halo list: %g (GB; Python) %g (GB; C)" - % ( - memory_data["phf_python"] / bytes_in_gb, - memory_data["phf_c"] / bytes_in_gb, - ) - ) - if "ib_python" in memory_data.keys(): - print( - "Memory for single ionized box: %g (GB; Python) %g (GB; C)" - % ( - memory_data["ib_python"] / bytes_in_gb, - memory_data["ib_c"] / bytes_in_gb, - ) - ) - - if "st_python" in memory_data.keys() and flag_options.USE_TS_FLUCT: - print( - "Memory for single spin temperature box: %g (GB; Python) %g (GB; C per z) %g (GB; C retained)" - % ( - memory_data["st_python"] / bytes_in_gb, - memory_data["st_c_per_z"] / bytes_in_gb, - memory_data["st_c_init"] / bytes_in_gb, - ) - ) - if "bt_python" in memory_data.keys(): - print( - "Memory for single brightness temperature box: %g (GB; Python) %g (GB; C)" - % ( - memory_data["bt_python"] / bytes_in_gb, - memory_data["bt_c"] / bytes_in_gb, - ) - ) - print("") + # bytes_in_gb = 1024**3 + + # print("") + # if "python_lc" in memory_data.keys(): + # print("Memory info for run_lightcone") + # elif len(memory_data) == 3: + # print("Memory info for initial_conditions") + # elif len(memory_data) == 5: + # print("Memory info for perturb_field") + # else: + # print("Memory info for run_coeval") + # print("") + # print("%s" % (user_params)) + # if astro_params is not None: + # print("%s" % (astro_params)) + # if flag_options is not None: + # print("%s" % (flag_options)) + # print("") + # print("Peak memory usage: %g (GB)" % (memory_data["peak_memory"] / bytes_in_gb)) + # print("") + # if "python_lc" in memory_data.keys(): + # print( + # "Memory for stored lightcones: %g (GB)" + # % (memory_data["python_lc"] / bytes_in_gb) + # ) + # """logger.info("Peak memory usage: %g (GB)"%(memory_data['peak_memory']/bytes_in_gb))""" + # print( + # "Memory for ICs: %g (GB; Python) %g (GB; C)" + # % (memory_data["ics_python"] / bytes_in_gb, memory_data["ics_c"] / bytes_in_gb) + # ) + # if "pf_python" in memory_data.keys(): + # print( + # "Memory for single perturbed field: %g (GB; Python) %g (GB; C)" + # % ( + # memory_data["pf_python"] / bytes_in_gb, + # memory_data["pf_c"] / bytes_in_gb, + # ) + # ) + # if "hf_python" in memory_data.keys() and flag_options.USE_HALO_FIELD: + # print( + # "Note these are approximations as we don't know a priori how many haloes there are (assume 10 per cent of volume)" + # ) + # print( + # "Memory for generating halo list: %g (GB; Python) %g (GB; C)" + # % ( + # memory_data["hf_python"] / bytes_in_gb, + # memory_data["hf_c"] / bytes_in_gb, + # ) + # ) + # print( + # "Memory for perturbing halo list: %g (GB; Python) %g (GB; C)" + # % ( + # memory_data["phf_python"] / bytes_in_gb, + # memory_data["phf_c"] / bytes_in_gb, + # ) + # ) + # if "ib_python" in memory_data.keys(): + # print( + # "Memory for single ionized box: %g (GB; Python) %g (GB; C)" + # % ( + # memory_data["ib_python"] / bytes_in_gb, + # memory_data["ib_c"] / bytes_in_gb, + # ) + # ) + + # if "st_python" in memory_data.keys() and flag_options.USE_TS_FLUCT: + # print( + # "Memory for single spin temperature box: %g (GB; Python) %g (GB; C per z) %g (GB; C retained)" + # % ( + # memory_data["st_python"] / bytes_in_gb, + # memory_data["st_c_per_z"] / bytes_in_gb, + # memory_data["st_c_init"] / bytes_in_gb, + # ) + # ) + # if "bt_python" in memory_data.keys(): + # print( + # "Memory for single brightness temperature box: %g (GB; Python) %g (GB; C)" + # % ( + # memory_data["bt_python"] / bytes_in_gb, + # memory_data["bt_c"] / bytes_in_gb, + # ) + # ) + # print("") From 303657e3ac1665ccbfd982d422c9f5b16482fb9a Mon Sep 17 00:00:00 2001 From: BradGreig Date: Wed, 17 May 2023 16:13:28 +1000 Subject: [PATCH 24/28] Update the perturb field function --- src/py21cmfast/_memory.py | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 2c26ee7d6..6fa08eaba 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -5,7 +5,7 @@ from copy import deepcopy from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params -from .outputs import InitialConditions +from .outputs import InitialConditions, PerturbedField from .wrapper import _logscroll_redshifts, _setup_lightcone logger = logging.getLogger("21cmFAST") @@ -706,26 +706,38 @@ def mem_initial_conditions( def mem_perturb_field( *, user_params=None, + cosmo_params=None, ): """A function to estimate total memory usage of a perturb_field call.""" - # Memory usage of Python PerturbedField class. - # All declared HII_DIM boxes""" - # density, velocity - num_py_boxes_HII_DIM = 2.0 - - size_py = num_py_boxes_HII_DIM * (user_params.HII_DIM) ** 3 + # Memory usage of Python InitialConditions class. (assume single redshift) + pt_box = PerturbedField( + redshift=9.0, user_params=user_params, cosmo_params=cosmo_params + ) + size_py = 0 + for key in pt_box._array_structure: + size_py += np.prod(pt_box._array_structure[key]) # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py # Memory usage within PerturbField.c - kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 - hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( - user_params.HII_DIM - ) ** 2 - - tot_num_pixels = float(user_params.DIM) ** 3 - hii_tot_num_pixels = float(user_params.HII_DIM) ** 3 + kspace_num_pixels = ( + float(user_params.DIM) * user_params.NON_CUBIC_FACTOR / 2.0 + 1.0 + ) * (user_params.DIM) ** 2 + hii_kspace_num_pixels = ( + float(user_params.HII_DIM) * user_params.NON_CUBIC_FACTOR / 2.0 + 1.0 + ) * (user_params.HII_DIM) ** 2 + + tot_num_pixels = ( + float(user_params.DIM) + * user_params.NON_CUBIC_FACTOR + * float(user_params.DIM) ** 2 + ) + hii_tot_num_pixels = ( + float(user_params.HII_DIM) + * user_params.NON_CUBIC_FACTOR + * float(user_params.HII_DIM) ** 2 + ) # LOWRES_density_perturb, LOWRES_density_perturb_saved num_c_boxes_HII_DIM = 2 From 51c4d24b08d0ba8d1334d66e163a5f1916bc2ebe Mon Sep 17 00:00:00 2001 From: BradGreig Date: Wed, 17 May 2023 17:04:44 +1000 Subject: [PATCH 25/28] Update ionisation box memory --- src/py21cmfast/_memory.py | 70 +++++++++++++++------------------------ 1 file changed, 27 insertions(+), 43 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 6fa08eaba..fc3262be5 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -5,7 +5,7 @@ from copy import deepcopy from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params -from .outputs import InitialConditions, PerturbedField +from .outputs import InitialConditions, IonizedBox, PerturbedField from .wrapper import _logscroll_redshifts, _setup_lightcone logger = logging.getLogger("21cmFAST") @@ -767,52 +767,37 @@ def mem_perturb_field( def mem_ionize_box( *, user_params=None, + cosmo_params=None, astro_params=None, flag_options=None, ): """A function to estimate total memory usage of an ionize_box call.""" - # Implicitly have dealt with INHOMO_RECO earlier - # determine number of filtering scales (for USE_MINI_HALOS) - if flag_options.USE_MINI_HALOS: - n_filtering = ( - int( - np.log( - min(astro_params.R_BUBBLE_MAX, 0.620350491 * user_params.BOX_LEN) - / max( - global_params.R_BUBBLE_MIN, - 0.620350491 - * user_params.BOX_LEN - / np.float64(user_params.HII_DIM), - ) - ) - / np.log(global_params.DELTA_R_HII_FACTOR) - ) - ) + 1 - else: - n_filtering = 1 - # Memory usage of Python IonizedBox class. - - # All declared HII_DIM boxes - # xH_box, Gamma12_box, MFP_box, z_re_box, dNrec_box, temp_kinetic_all_gas - num_py_boxes = 6.0 - - # Fcoll - num_py_boxes += n_filtering - - # Fcoll_MINI - if flag_options.USE_MINI_HALOS: - num_py_boxes += n_filtering - - size_py = num_py_boxes * (user_params.HII_DIM) ** 3 + ibox = IonizedBox( + redshift=9.0, + user_params=user_params, + cosmo_params=cosmo_params, + astro_params=astro_params, + flag_options=flag_options, + ) + size_py = 0 + for key in ibox._array_structure: + if isinstance( + ibox._array_structure[key], dict + ): # It can have embedded dictionaries + for alt_key in ibox._array_structure[key]: + if "shape" in alt_key: + size_py += np.prod(ibox._array_structure[key][alt_key]) + else: + size_py += np.prod(ibox._array_structure[key]) # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py # Memory usage within IonisationBox.c - hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( - user_params.HII_DIM - ) ** 2 + hii_kspace_num_pixels = ( + float(user_params.HII_DIM) * user_params.NON_CUBIC_FACTOR / 2.0 + 1.0 + ) * (user_params.HII_DIM) ** 2 # deltax_unfiltered, delta_unfiltered_original, deltax_filtered num_c_boxes = 3.0 @@ -829,8 +814,6 @@ def mem_ionize_box( # N_rec_unfiltered, N_rec_filtered num_c_boxes += 2.0 - # There are a bunch of 1 and 2D interpolation tables, but ignore those as they are small relative to 3D grids - if flag_options.USE_MASS_DEPENDENT_ZETA and flag_options.USE_MINI_HALOS: # log10_Mturnover_unfiltered, log10_Mturnover_filtered, log10_Mturnover_MINI_unfiltered, log10_Mturnover_MINI_filtered num_c_boxes += 4.0 @@ -839,13 +822,16 @@ def mem_ionize_box( # M_coll_unfiltered, M_coll_filtered num_c_boxes += 2.0 + # Various interpolation tables tables_float = tables_double = 0.0 if flag_options.USE_MASS_DEPENDENT_ZETA: tables_float += 2.0 * (ngl_sfr + 1) # xi_SFR, wi_SFR if user_params.USE_INTERPOLATION_TABLES: - tables_double += nsfr_low # log10_overdense_spline_SFR - tables_float += nsfr_high # Overdense_spline_SFR + tables_double += nsfr_low # log10_overdense_spline_SFR, + tables_float += ( + 2.0 * nsfr_high + nsfr_low + ) # Overdense_spline_SFR, Nion_spline, log10_Nion_spline if flag_options.USE_MINI_HALOS: tables_double += nsfr_low # prev_log10_overdense_spline_SFR @@ -856,8 +842,6 @@ def mem_ionize_box( tables_float += ( 4.0 * nsfr_high * nmturn ) # Nion_spline, Nion_spline_MINI, prev_Nion_spline, prev_Nion_spline_MINI - else: - tables_float += nsfr_high + nsfr_low # log10_Nion_spline, Nion_spline if flag_options.USE_MINI_HALOS: tables_float += 2.0 * nmturn # Mturns, Mturns_MINI From cfe401fde55c8df91dcce08e2554784e88f27d63 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Wed, 17 May 2023 17:50:16 +1000 Subject: [PATCH 26/28] Update the memory function for spin temperature --- src/py21cmfast/_memory.py | 65 +++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index fc3262be5..3b53094a4 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -5,7 +5,7 @@ from copy import deepcopy from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params -from .outputs import InitialConditions, IonizedBox, PerturbedField +from .outputs import InitialConditions, IonizedBox, PerturbedField, TsBox from .wrapper import _logscroll_redshifts, _setup_lightcone logger = logging.getLogger("21cmFAST") @@ -767,7 +767,6 @@ def mem_perturb_field( def mem_ionize_box( *, user_params=None, - cosmo_params=None, astro_params=None, flag_options=None, ): @@ -776,7 +775,6 @@ def mem_ionize_box( ibox = IonizedBox( redshift=9.0, user_params=user_params, - cosmo_params=cosmo_params, astro_params=astro_params, flag_options=flag_options, ) @@ -877,21 +875,24 @@ def mem_spin_temperature( flag_options=None, ): """A function to estimate total memory usage of a spin_temperature call.""" - # Memory usage of Python IonizedBox class. - - # All declared HII_DIM boxes - # Ts_bx, x_e_box, Tk_box, J_21_LW_box - num_py_boxes = 4.0 - - size_py = num_py_boxes * (user_params.HII_DIM) ** 3 + # Memory usage of Python TsBox class. + ts_box = TsBox( + redshift=9.0, + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + size_py = 0 + for key in ts_box._array_structure: + size_py += np.prod(ts_box._array_structure[key]) # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py # Memory usage within SpinTemperatureBox.c - hii_kspace_num_pixels = (float(user_params.HII_DIM) / 2.0 + 1.0) * ( - user_params.HII_DIM - ) ** 2 + hii_kspace_num_pixels = ( + float(user_params.HII_DIM) * user_params.NON_CUBIC_FACTOR / 2.0 + 1.0 + ) * (user_params.HII_DIM) ** 2 # box, unfiltered_box num_c_boxes = 2.0 @@ -919,7 +920,10 @@ def mem_spin_temperature( if flag_options.USE_MASS_DEPENDENT_ZETA: # delNL0 - num_c_boxes_initialised += global_params.NUM_FILTER_STEPS_FOR_Ts + if user_params.MINIMIZE_MEMORY: + num_c_boxes_initialised += 1.0 + else: + num_c_boxes_initialised += global_params.NUM_FILTER_STEPS_FOR_Ts # del_fcoll_Rct, m_xHII_low_box, inverse_val_box num_c_boxes_initialised += ( @@ -928,6 +932,10 @@ def mem_spin_temperature( # dxheat_dt_box, dxion_source_dt_box, dxlya_dt_box, dstarlya_dt_box num_c_boxes_initialised += 2.0 * 4.0 # factor of 2. as these are doubles + if flag_options.USE_LYA_HEATING: + num_c_boxes_initialised += ( + 2.0 * 2.0 + ) # dstarlya_cont_dt_box, dstarlya_inj_dt_box tables_float += global_params.NUM_FILTER_STEPS_FOR_Ts # SFR_timescale_factor tables_double += 2.0 * (ngl_sfr + 1.0) # xi_SFR_Xray, wi_SFR_Xray @@ -950,6 +958,10 @@ def mem_spin_temperature( num_c_boxes_initialised += ( 2.0 * 6.0 ) # factor of 2. taking into account that these are doubles + if flag_options.USE_LYA_HEATING: + num_c_boxes_initialised += ( + 2.0 * 2.0 + ) # dstarlya_cont_dt_box_MINI, dstarlya_inj_dt_box_MINI tables_double += ( global_params.NUM_FILTER_STEPS_FOR_Ts @@ -987,17 +999,27 @@ def mem_spin_temperature( ) # delNL0_bw, delNL0_Offset, delNL0_LL, delNL0_UL, delNL0_ibw, log10delNL0_diff, log10delNL0_diff_UL # dens_grid_int_vals - num_c_boxes_initialised += 0.5 # 0.5 as it is a short + num_c_boxes_initialised += ( + 0.5 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # 0.5 as it is a short + + # dstarlya_dt_prefactor, fcoll_R_array, sigma_Tmin, ST_over_PS, sum_lyn + tables_double += 5.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + if flag_options.USE_LYA_HEATING: + tables_double += ( + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # dstarlya_cont_dt_prefactor,dstarlya_inj_dt_prefactor, sum_ly2, sum_lynto2 - tables_double += global_params.NUM_FILTER_STEPS_FOR_Ts # dstarlya_dt_prefactor if flag_options.USE_MINI_HALOS: tables_double += ( - 3.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + 7.0 * global_params.NUM_FILTER_STEPS_FOR_Ts ) # dstarlya_dt_prefactor_MINI, dstarlyLW_dt_prefactor, dstarlyLW_dt_prefactor_MINI + # ST_over_PS_MINI, sum_lyn_MINI, sum_lyLWn, sum_lyLWn_MINI, + if flag_options.USE_LYA_HEATING: + tables_double += ( + 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + ) # dstarlya_cont_dt_prefactor_MINI,dstarlya_inj_dt_prefactor_MINI,sum_ly2_MINI,sum_lynto2_MINI - tables_double += ( - 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts - ) # ST_over_PS_MINI, sum_lyn_MINI, sum_lyLWn, sum_lyLWn_MINI, tables_float += global_params.NUM_FILTER_STEPS_FOR_Ts # Mcrit_atom_interp_table tables_float += ( @@ -1008,9 +1030,6 @@ def mem_spin_temperature( 6.0 * x_int_nxhii * global_params.NUM_FILTER_STEPS_FOR_Ts ) # freq_int_heat_tbl, freq_int_ion_tbl, freq_int_lya_tbl, freq_int_heat_tbl_diff, freq_int_ion_tbl_diff, freq_int_lya_tbl_diff - tables_double += ( - 4.0 * global_params.NUM_FILTER_STEPS_FOR_Ts - ) # fcoll_R_array, sigma_Tmin, ST_over_PS, sum_lyn tables_float += ( x_int_nxhii + global_params.NUM_FILTER_STEPS_FOR_Ts ) # inverse_diff, zpp_growth From a25394d42b960bd51afca1d717491144687eac3d Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 18 May 2023 12:24:52 +1000 Subject: [PATCH 27/28] Update brightness temp, and halo functions --- src/py21cmfast/_memory.py | 100 ++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 26 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 3b53094a4..1f18a723d 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -5,7 +5,13 @@ from copy import deepcopy from .inputs import AstroParams, CosmoParams, FlagOptions, UserParams, global_params -from .outputs import InitialConditions, IonizedBox, PerturbedField, TsBox +from .outputs import ( + BrightnessTemp, + InitialConditions, + IonizedBox, + PerturbedField, + TsBox, +) from .wrapper import _logscroll_redshifts, _setup_lightcone logger = logging.getLogger("21cmFAST") @@ -1057,12 +1063,20 @@ def mem_spin_temperature( # These are all fftwf complex arrays (thus 2 * size) size_c = (2.0 * (np.float32(1.0).nbytes)) * num_c_boxes * hii_kspace_num_pixels - size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.HII_DIM**3.0) + size_c += ( + (np.float32(1.0).nbytes) + * num_c_boxes_alt + * (user_params.HII_DIM) + * user_params.NON_CUBIC_FACTOR + * (user_params.HII_DIM) ** 2 + ) size_c_init = ( (np.float32(1.0).nbytes) * num_c_boxes_initialised - * (user_params.HII_DIM**3.0) + * (user_params.HII_DIM) + * user_params.NON_CUBIC_FACTOR + * (user_params.HII_DIM) ** 2 ) # Now, add all the table data (which are kept throughout the calculation) @@ -1076,29 +1090,44 @@ def mem_spin_temperature( def mem_brightness_temperature( *, user_params=None, + astro_params=None, + flag_options=None, ): """A function to estimate total memory usage of a brightness_temperature call.""" # Memory usage of Python BrightnessTemp class. - - # All declared HII_DIM boxes - # brightness_temp - num_py_boxes = 1.0 - - size_py = num_py_boxes * (user_params.HII_DIM) ** 3 + ts_box = BrightnessTemp( + redshift=9.0, + user_params=user_params, + astro_params=astro_params, + flag_options=flag_options, + ) + size_py = 0 + for key in ts_box._array_structure: + size_py += np.prod(ts_box._array_structure[key]) # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py # Memory usage within BrightnessTemperatureBox.c hii_tot_fft_num_pixels = ( - 2.0 * (float(user_params.HII_DIM) / 2.0 + 1.0) * (user_params.HII_DIM) ** 2 + 2.0 + * (float(user_params.HII_DIM) * user_params.NON_CUBIC_FACTOR / 2.0 + 1.0) + * (user_params.HII_DIM) ** 2 ) - # box, unfiltered_box + # v, vel_gradient num_c_boxes = 2.0 size_c = (np.float32(1.0).nbytes) * num_c_boxes * hii_tot_fft_num_pixels + # For x_pos, x_pos_offset, delta_T_RSD_LOS as part of the RSDs + size_c += (np.float32(1.0).nbytes) * ( + 2.0 * global_params.NUM_FILTER_STEPS_FOR_Ts + + user_params.N_THREADS + * float(user_params.HII_DIM) + * user_params.NON_CUBIC_FACTOR + ) + return {"python": size_py, "c": size_c} @@ -1107,19 +1136,10 @@ def mem_halo_field( user_params=None, ): """A function to estimate total memory usage of a determine_halo_list call.""" - # Memory usage of Python HaloField class. - - # All declared DIM boxes - # halo_field - num_py_boxes = 1.0 - - size_py = num_py_boxes * (user_params.DIM) ** 3 - - # These are all float arrays - size_py = (np.float32(1.0).nbytes) * size_py - # Memory usage within FindHaloes.c - kspace_num_pixels = (float(user_params.DIM) / 2.0 + 1.0) * (user_params.DIM) ** 2 + kspace_num_pixels = ( + float(user_params.DIM) * user_params.NON_CUBIC_FACTOR / 2.0 + 1.0 + ) * (user_params.DIM) ** 2 # density_field, density_field_saved num_c_boxes = 2.0 @@ -1136,13 +1156,34 @@ def mem_halo_field( # These are fftwf complex arrays (thus 2 * size) size_c = (2.0 * np.float32(1.0).nbytes) * num_c_boxes * kspace_num_pixels - size_c += (np.float32(1.0).nbytes) * num_c_boxes_alt * (user_params.DIM) ** 3 + size_c += ( + (np.float32(1.0).nbytes) + * num_c_boxes_alt + * (user_params.DIM) + * user_params.NON_CUBIC_FACTOR + * (user_params.DIM) ** 2 + ) # We don't know a priori how many haloes that will be found, but we'll estimate the memory usage # at 10 per cent of the total number of pixels (HII_DIM, likely an over estimate) # Below the factor of 4 corresponds to the mass and three spatial locations. It is defined as an # int but I'll leave it as 4 bytes in case - size_c += 0.1 * 4.0 * (np.float32(1.0).nbytes) * (user_params.HII_DIM) ** 3 + size_c += ( + 0.1 + * 4.0 + * (np.float32(1.0).nbytes) + * (user_params.HII_DIM) + * user_params.NON_CUBIC_FACTOR + * (user_params.HII_DIM) ** 2 + ) + size_py = ( + 0.1 + * 4.0 + * (np.float32(1.0).nbytes) + * (user_params.HII_DIM) + * user_params.NON_CUBIC_FACTOR + * (user_params.HII_DIM) ** 2 + ) return {"python": size_py, "c": size_c} @@ -1158,7 +1199,14 @@ def mem_perturb_halo( # at 10 per cent of the total number of pixels (HII_DIM, likely an over estimate) # Below the factor of 4 corresponds to the mass and three spatial locations. It is defined as an # int but I'll leave it as 4 bytes in case - size_c = 0.1 * 4.0 * (np.float32(1.0).nbytes) * (user_params.HII_DIM) ** 3 + size_c = ( + 0.1 + * 4.0 + * (np.float32(1.0).nbytes) + * (user_params.HII_DIM) + * user_params.NON_CUBIC_FACTOR + * (user_params.HII_DIM) ** 2 + ) return {"python": 0.0, "c": size_c} From 8ed443047331586fee64e61d81a1508ef30a0c87 Mon Sep 17 00:00:00 2001 From: BradGreig Date: Thu, 18 May 2023 16:25:55 +1000 Subject: [PATCH 28/28] Some general cleanup --- src/py21cmfast/_memory.py | 316 ++++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 135 deletions(-) diff --git a/src/py21cmfast/_memory.py b/src/py21cmfast/_memory.py index 1f18a723d..de3aa48c1 100644 --- a/src/py21cmfast/_memory.py +++ b/src/py21cmfast/_memory.py @@ -12,9 +12,14 @@ PerturbedField, TsBox, ) -from .wrapper import _logscroll_redshifts, _setup_lightcone +from .wrapper import ( + _get_required_redshifts_coeval, + _logscroll_redshifts, + _setup_lightcone, +) logger = logging.getLogger("21cmFAST") +logger.setLevel(logging.INFO) # Constants defining interpolation table lengths (from C side) # Not ideal to be here, but unlikely to ever change (names in lower case) @@ -40,6 +45,7 @@ def estimate_memory_coeval( cosmo_params=None, astro_params=None, flag_options=None, + write=False, ): r""" Compute an estimate of the requisite memory needed by the user for a run_coeval call. @@ -95,6 +101,9 @@ def estimate_memory_coeval( peak_memory = memory_ics["c"] + memory_ics["python"] # Now the perturb field + # First we purge the ICs memory for boxes we no longer need + memory_ics_purged = mem_initial_conditions(user_params=user_params, purge_memory=1) + memory_pf = mem_perturb_field(user_params=user_params) memory_data.update({"pf_%s" % k: memory_pf[k] for k in memory_pf.keys()}) @@ -123,7 +132,7 @@ def estimate_memory_coeval( # As we iterate through we storing the python memory of two # perturb_field and ionize_boxes plus the C memory of either # of the ionize_box or perturb field as it is being calculated - peak_memory_photoncons = memory_ics[ + peak_memory_photoncons = memory_ics_purged[ "python" ] # We have the initial conditions in memory peak_memory_photoncons += 2 * ( @@ -147,7 +156,9 @@ def estimate_memory_coeval( n_redshifts = len(redshift) - current_memory = memory_ics["python"] # We have the initial conditions in memory + current_memory = memory_ics_purged[ + "python" + ] # We have the initial conditions in memory current_memory += ( n_redshifts * memory_pf["python"] ) # Python memory for all perturb fields @@ -157,6 +168,18 @@ def estimate_memory_coeval( peak_memory = peak_memory if peak_memory > current_memory else current_memory # Now start generating estimates for all other data products + # First, we purge the ICs memory further (only if they exist on disk) + if write is True and user_params.MINIMIZE_MEMORY: + memory_ics_purged = mem_initial_conditions( + user_params=user_params, purge_memory=2 + ) + else: + # Because if we don't store it, we keep it + if flag_options.USE_TS_FLUCT: + scrollz = _get_required_redshifts_coeval(flag_options, redshift) + memory_pf["python"] *= len(scrollz) + else: + memory_pf["python"] *= n_redshifts # Calculate the memory for a determine_halo_list call memory_hf = mem_halo_field(user_params=user_params) @@ -180,7 +203,6 @@ def estimate_memory_coeval( # Calculate the memory for a spin_temperature call memory_st = mem_spin_temperature( user_params=user_params, - astro_params=astro_params, flag_options=flag_options, ) @@ -192,7 +214,9 @@ def estimate_memory_coeval( memory_data.update({"bt_%s" % k: memory_bt[k] for k in memory_bt.keys()}) # All the data kept in memory at this point in Python - stored_memory = memory_ics["python"] # We have the initial conditions in memory + stored_memory = memory_ics_purged[ + "python" + ] # We have the initial conditions in memory stored_memory += ( n_redshifts * memory_pf["python"] ) # Python memory for all perturb fields @@ -290,6 +314,7 @@ def estimate_memory_lightcone( astro_params=None, flag_options=None, lightcone_quantities=("brightness_temp",), + write=False, ): r""" Compute an estimate of the requisite memory needed by the user for a run_lightcone call. @@ -360,12 +385,51 @@ def estimate_memory_lightcone( peak_memory = memory_ics["c"] + memory_ics["python"] # Now the perturb field + # First we purge the ICs memory for boxes we no longer need + memory_ics_purged = mem_initial_conditions(user_params=user_params, purge_memory=1) + memory_pf = mem_perturb_field(user_params=user_params) memory_data.update({"pf_%s" % k: memory_pf[k] for k in memory_pf.keys()}) # Stored ICs in python + allocated C and Python memory for perturb_field - current_memory = memory_ics["python"] + memory_pf["python"] + memory_pf["c"] + current_memory = memory_ics_purged["python"] + memory_pf["python"] + memory_pf["c"] + + # Now need to determine the size of the light-cone and how many types are to be stored in memory. + # Below are taken from the run_lightcone function. + # Determine the maximum redshift (starting point) for the light-cone. + max_redshift = ( + global_params.Z_HEAT_MAX + if ( + flag_options.INHOMO_RECO + or flag_options.USE_TS_FLUCT + or max_redshift is None + ) + else max_redshift + ) + + # Get the redshift through which we scroll and evaluate the ionization field. + scrollz = _logscroll_redshifts( + redshift, global_params.ZPRIME_STEP_FACTOR, max_redshift + ) + + # Obtain the size of the light-cone object (n_lightcone) + d_at_redshift, lc_distances, n_lightcone = _setup_lightcone( + cosmo_params, + max_redshift, + redshift, + scrollz, + user_params, + global_params.ZPRIME_STEP_FACTOR, + ) + + # First, we purge the ICs memory further (only if they exist on disk) + if write is True and user_params.MINIMIZE_MEMORY: + memory_ics_purged = mem_initial_conditions( + user_params=user_params, purge_memory=2 + ) + else: + memory_pf["python"] *= len(scrollz) # Because if we don't store it, we keep it # Check if running perturb_field requires more memory than generating ICs peak_memory = peak_memory if peak_memory > current_memory else current_memory @@ -412,34 +476,6 @@ def estimate_memory_lightcone( else peak_memory_photoncons ) - # Now need to determine the size of the light-cone and how many types are to be stored in memory. - # Below are taken from the run_lightcone function. - # Determine the maximum redshift (starting point) for the light-cone. - max_redshift = ( - global_params.Z_HEAT_MAX - if ( - flag_options.INHOMO_RECO - or flag_options.USE_TS_FLUCT - or max_redshift is None - ) - else max_redshift - ) - - # Get the redshift through which we scroll and evaluate the ionization field. - scrollz = _logscroll_redshifts( - redshift, global_params.ZPRIME_STEP_FACTOR, max_redshift - ) - - # Obtain the size of the light-cone object (n_lightcone) - d_at_redshift, lc_distances, n_lightcone = _setup_lightcone( - cosmo_params, - max_redshift, - redshift, - scrollz, - user_params, - global_params.ZPRIME_STEP_FACTOR, - ) - # Total number of light-cones to be stored in memory num_lightcones = len(lightcone_quantities) @@ -450,7 +486,7 @@ def estimate_memory_lightcone( memory_data.update({"python_lc": size_lightcones}) # All the data kept in memory at this point in Python - current_memory = memory_ics["python"] + memory_pf["python"] + size_lightcones + current_memory = memory_ics_purged["python"] + memory_pf["python"] + size_lightcones # Check if we now exceed the peak memory usage thus far peak_memory = peak_memory if peak_memory > current_memory else current_memory @@ -479,7 +515,6 @@ def estimate_memory_lightcone( # Calculate the memory for a spin_temperature call memory_st = mem_spin_temperature( user_params=user_params, - astro_params=astro_params, flag_options=flag_options, ) @@ -568,6 +603,7 @@ def estimate_memory_lightcone( def estimate_memory_ics( *, user_params=None, + purge_memory=0, ): r""" Compute an estimate of the requisite memory needed just for the initial conditions. @@ -587,7 +623,9 @@ def estimate_memory_ics( peak_memory: As estimate of the peak memory usage for running a lightcone (generating all data) (in Bytes) """ # Calculate the memory usage for the initial conditions - memory_ics = mem_initial_conditions(user_params=user_params) + memory_ics = mem_initial_conditions( + user_params=user_params, purge_memory=purge_memory + ) memory_data = {"ics_%s" % k: memory_ics[k] for k in memory_ics.keys()} @@ -670,13 +708,26 @@ def estimate_memory_perturb( def mem_initial_conditions( *, user_params=None, + purge_memory=0, ): """A function to estimate total memory usage of an initial_conditions call.""" # Memory usage of Python InitialConditions class. init = InitialConditions(user_params=user_params) size_py = 0 - for key in init._array_structure: - size_py += np.prod(init._array_structure[key]) + if purge_memory > 0: + for key in init._array_structure: + if purge_memory == 1: # prepare for perturb + if user_params.PERTURB_ON_HIGH_RES: + if "hires" in key or "lowres_vcb" in key: + size_py += np.prod(init._array_structure[key]) + else: + if "lowres" in key or "hires_density" in key: + size_py += np.prod(init._array_structure[key]) + if purge_memory == 2 and "lowres_vcb" in key: # prepare for spin temp + size_py += np.prod(init._array_structure[key]) + else: + for key in init._array_structure: + size_py += np.prod(init._array_structure[key]) # These are all float arrays size_py = (np.float32(1.0).nbytes) * size_py @@ -712,13 +763,10 @@ def mem_initial_conditions( def mem_perturb_field( *, user_params=None, - cosmo_params=None, ): """A function to estimate total memory usage of a perturb_field call.""" # Memory usage of Python InitialConditions class. (assume single redshift) - pt_box = PerturbedField( - redshift=9.0, user_params=user_params, cosmo_params=cosmo_params - ) + pt_box = PerturbedField(redshift=0.0, user_params=user_params) size_py = 0 for key in pt_box._array_structure: size_py += np.prod(pt_box._array_structure[key]) @@ -779,7 +827,7 @@ def mem_ionize_box( """A function to estimate total memory usage of an ionize_box call.""" # Memory usage of Python IonizedBox class. ibox = IonizedBox( - redshift=9.0, + redshift=0.0, user_params=user_params, astro_params=astro_params, flag_options=flag_options, @@ -877,16 +925,13 @@ def mem_ionize_box( def mem_spin_temperature( *, user_params=None, - astro_params=None, flag_options=None, ): """A function to estimate total memory usage of a spin_temperature call.""" # Memory usage of Python TsBox class. ts_box = TsBox( - redshift=9.0, + redshift=0.0, user_params=user_params, - astro_params=astro_params, - flag_options=flag_options, ) size_py = 0 for key in ts_box._array_structure: @@ -1090,16 +1135,12 @@ def mem_spin_temperature( def mem_brightness_temperature( *, user_params=None, - astro_params=None, - flag_options=None, ): """A function to estimate total memory usage of a brightness_temperature call.""" # Memory usage of Python BrightnessTemp class. ts_box = BrightnessTemp( redshift=9.0, user_params=user_params, - astro_params=astro_params, - flag_options=flag_options, ) size_py = 0 for key in ts_box._array_structure: @@ -1219,86 +1260,91 @@ def print_memory_estimate( flag_options=None, ): """Function to output information in a manageable format.""" - # bytes_in_gb = 1024**3 - - # print("") - # if "python_lc" in memory_data.keys(): - # print("Memory info for run_lightcone") - # elif len(memory_data) == 3: - # print("Memory info for initial_conditions") - # elif len(memory_data) == 5: - # print("Memory info for perturb_field") - # else: - # print("Memory info for run_coeval") - # print("") - # print("%s" % (user_params)) - # if astro_params is not None: - # print("%s" % (astro_params)) - # if flag_options is not None: - # print("%s" % (flag_options)) - # print("") - # print("Peak memory usage: %g (GB)" % (memory_data["peak_memory"] / bytes_in_gb)) - # print("") - # if "python_lc" in memory_data.keys(): - # print( - # "Memory for stored lightcones: %g (GB)" - # % (memory_data["python_lc"] / bytes_in_gb) - # ) - # """logger.info("Peak memory usage: %g (GB)"%(memory_data['peak_memory']/bytes_in_gb))""" - # print( - # "Memory for ICs: %g (GB; Python) %g (GB; C)" - # % (memory_data["ics_python"] / bytes_in_gb, memory_data["ics_c"] / bytes_in_gb) - # ) - # if "pf_python" in memory_data.keys(): - # print( - # "Memory for single perturbed field: %g (GB; Python) %g (GB; C)" - # % ( - # memory_data["pf_python"] / bytes_in_gb, - # memory_data["pf_c"] / bytes_in_gb, - # ) - # ) - # if "hf_python" in memory_data.keys() and flag_options.USE_HALO_FIELD: - # print( - # "Note these are approximations as we don't know a priori how many haloes there are (assume 10 per cent of volume)" - # ) - # print( - # "Memory for generating halo list: %g (GB; Python) %g (GB; C)" - # % ( - # memory_data["hf_python"] / bytes_in_gb, - # memory_data["hf_c"] / bytes_in_gb, - # ) - # ) - # print( - # "Memory for perturbing halo list: %g (GB; Python) %g (GB; C)" - # % ( - # memory_data["phf_python"] / bytes_in_gb, - # memory_data["phf_c"] / bytes_in_gb, - # ) - # ) - # if "ib_python" in memory_data.keys(): - # print( - # "Memory for single ionized box: %g (GB; Python) %g (GB; C)" - # % ( - # memory_data["ib_python"] / bytes_in_gb, - # memory_data["ib_c"] / bytes_in_gb, - # ) - # ) - - # if "st_python" in memory_data.keys() and flag_options.USE_TS_FLUCT: - # print( - # "Memory for single spin temperature box: %g (GB; Python) %g (GB; C per z) %g (GB; C retained)" - # % ( - # memory_data["st_python"] / bytes_in_gb, - # memory_data["st_c_per_z"] / bytes_in_gb, - # memory_data["st_c_init"] / bytes_in_gb, - # ) - # ) - # if "bt_python" in memory_data.keys(): - # print( - # "Memory for single brightness temperature box: %g (GB; Python) %g (GB; C)" - # % ( - # memory_data["bt_python"] / bytes_in_gb, - # memory_data["bt_c"] / bytes_in_gb, - # ) - # ) - # print("") + bytes_in_gb = 1024**3 + + logger.info("") + if "python_lc" in memory_data.keys(): + logger.info("Memory info for run_lightcone") + elif len(memory_data) == 3: + logger.info("Memory info for initial_conditions") + elif len(memory_data) == 5: + logger.info("Memory info for perturb_field") + else: + logger.info("Memory info for run_coeval") + logger.info("") + logger.info("%s" % (user_params)) + if astro_params is not None: + logger.info("%s" % (astro_params)) + if flag_options is not None: + logger.info("%s" % (flag_options)) + logger.info("") + logger.info( + "Peak memory usage: %g (GB)" % (memory_data["peak_memory"] / bytes_in_gb) + ) + logger.info( + "(Note these are purely indicative and may differ from actual performance)" + ) + logger.info("") + if "python_lc" in memory_data.keys(): + logger.info( + "Memory for stored lightcones: %g (GB)" + % (memory_data["python_lc"] / bytes_in_gb) + ) + """logger.info("Peak memory usage: %g (GB)"%(memory_data['peak_memory']/bytes_in_gb))""" + logger.info( + "Memory for ICs: %g (GB; Python) %g (GB; C)" + % (memory_data["ics_python"] / bytes_in_gb, memory_data["ics_c"] / bytes_in_gb) + ) + if "pf_python" in memory_data.keys(): + logger.info( + "Memory for single perturbed field: %g (GB; Python) %g (GB; C)" + % ( + memory_data["pf_python"] / bytes_in_gb, + memory_data["pf_c"] / bytes_in_gb, + ) + ) + if "hf_python" in memory_data.keys() and flag_options.USE_HALO_FIELD: + logger.info( + "Note these are approximations as we don't know a priori how many haloes there are (assume 10 per cent of volume)" + ) + logger.info( + "Memory for generating halo list: %g (GB; Python) %g (GB; C)" + % ( + memory_data["hf_python"] / bytes_in_gb, + memory_data["hf_c"] / bytes_in_gb, + ) + ) + logger.info( + "Memory for perturbing halo list: %g (GB; Python) %g (GB; C)" + % ( + memory_data["phf_python"] / bytes_in_gb, + memory_data["phf_c"] / bytes_in_gb, + ) + ) + if "ib_python" in memory_data.keys(): + logger.info( + "Memory for single ionized box: %g (GB; Python) %g (GB; C)" + % ( + memory_data["ib_python"] / bytes_in_gb, + memory_data["ib_c"] / bytes_in_gb, + ) + ) + + if "st_python" in memory_data.keys() and flag_options.USE_TS_FLUCT: + logger.info( + "Memory for single spin temperature box: %g (GB; Python) %g (GB; C per z) %g (GB; C retained)" + % ( + memory_data["st_python"] / bytes_in_gb, + memory_data["st_c_per_z"] / bytes_in_gb, + memory_data["st_c_init"] / bytes_in_gb, + ) + ) + if "bt_python" in memory_data.keys(): + logger.info( + "Memory for single brightness temperature box: %g (GB; Python) %g (GB; C)" + % ( + memory_data["bt_python"] / bytes_in_gb, + memory_data["bt_c"] / bytes_in_gb, + ) + ) + logger.info("")