From 7dd606a0be34fa58fd1717ace23a95f0bb7d14d5 Mon Sep 17 00:00:00 2001 From: Erik P G Johansson Date: Mon, 19 Feb 2024 10:10:24 +0100 Subject: [PATCH 1/7] solo.qli.quicklooks_main: Cleanup --- .../+solo/+qli/quicklooks_main.m | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/mission/solar_orbiter/+solo/+qli/quicklooks_main.m b/mission/solar_orbiter/+solo/+qli/quicklooks_main.m index 9a6d47b27..b58438e3d 100644 --- a/mission/solar_orbiter/+solo/+qli/quicklooks_main.m +++ b/mission/solar_orbiter/+solo/+qli/quicklooks_main.m @@ -14,7 +14,7 @@ % NOTE: Creates subdirectories to the output directory if not pre-existing. % NOTE: 7-day plots will only cover that largest sub-time interval that is % composed of full 7-day periods. Note: 7-day periods begin with a -% hardcoded weekday (Wednesday as of 2023-05-09). +% hardcoded weekday (Wednesday as of 2023-07-24). % NOTE: Overwrites pre-existing plot files without warning. % % @@ -36,6 +36,9 @@ % Useful for testing and not re-running unnecessary time-consuming plots. % utcBegin, utcEnd : Strings. % Defines time interval for which quicklooks should be generated. +% NOTE: Weekly plots will only be produced for those weeks which are +% contained entirely inside the specified time interval (?). Weekly plots +% always (as of 2023-07-24) begin on a Wednesday(!) at 00:00:00. % % % Initially created ~<2021-03-11, based on code by Konrad Steinvall, IRF, @@ -112,10 +115,12 @@ function quicklooks_main(... % IMPLEMENTATION NOTE: Disabling B (use empty; pretend there is no B data) % speeds up solo.qli.quicklooks_24_6_2_h() greatly. Useful for some debugging. % Should be enabled by default. -ENABLE_B = 1; -% Whether to catch plotting exceptions (and continue) in order to produce as -% many plots as possible. Should be enabled by default. -CATCH_PLOT_EXCEPTIONS_ENABLED = 1; +ENABLE_B = 1; % 0 or 1. +% Whether to catch plotting exceptions, continue plotting other days/weeks, and +% then re-raise the last caught exception at the very end. This produces as many +% plots as possible when one or some plots fail. +% Should be enabled by default. +CATCH_PLOT_EXCEPTIONS_ENABLED = 1; % 0 or 1. @@ -181,13 +186,13 @@ function quicklooks_main(... if runNonweeklyPlots % Daily time-intervals Time1DayStepsArray = make_time_array(TimeIntervalNonWeeks, 1); - + % Load data % This is the .mat file containing RPW speeds at 1h resolution. % The file should be in the current path. This file can be found in % brain:/solo/data/data_yuri/. vht1h = load(fullfile(vhtDataDir, VHT_1H_DATA_FILENAME)); - + for iTint=1:length(Time1DayStepsArray)-1 % Select time interval. Tint = irf.tint(Time1DayStepsArray(iTint), Time1DayStepsArray(iTint+1)); @@ -206,15 +211,15 @@ function quicklooks_main(... % Run the code for weekly overviews %=================================== if runWeeklyPlots - + % Weekly time-intervals Time7DayStepsArray = make_time_array(TimeIntervalWeeks, 7); - + % Load data % This is the .mat file containing RPW speeds at 6h resolution. % The file should be in the same folder as this script (quicklook_main). vht6h = load(fullfile(vhtDataDir, VHT_6H_DATA_FILENAME)); - + for iTint=1:length(Time7DayStepsArray)-1 % Select time interval. Tint = irf.tint(Time7DayStepsArray(iTint), Time7DayStepsArray(iTint+1)); @@ -277,7 +282,7 @@ function log_plot_function_time_interval(Tint) function handle_plot_exception(catchExceptionEnabled, Exc) if catchExceptionEnabled % Print stack trace without rethrowing exception. - % One wants that in log. + % One wants that in the log. % NOTE: fprintf(FID=2) => stderr fprintf(2, 'Caught plotting error without rethrowing it.\n') fprintf(2, 'Plot error/exception: "%s"\n', Exc.message) @@ -431,7 +436,7 @@ function quicklooks_7days_local(Tint, vht6h, Paths, logoPath) if ~isempty(spiceEarthPos) [E_radius, E_lon, E_lat] = cspice_reclat(spiceEarthPos); earthPos = [E_radius', E_lon', E_lat']; - + Tlength = Tint(end)-Tint(1); dTimes = 0:dt:Tlength; Times = Tint(1)+dTimes; @@ -484,11 +489,9 @@ function quicklooks_7days_local(Tint, vht6h, Paths, logoPath) % Wrapper around solo.db_get_ts() that normalizes the output to a TSeries. % -% NOTE: solo.db_get_ts() has been observed to return cell array of TSeries -% (instead of TSeries) for Npas, Tpas and Vpas for -% solo.qli.quicklooks_main('2020-10-21T00:00:00', '2020-10-28T00:00:00', '/data/solo/data_yuri', ...) -% There might be other cases but those are as of yet unknown. -% /Erik P G Johansson 2021-03-22 +% NOTE: solo.db_get_ts() may sometimes return cell array of TSeries instead of a +% single TSeries when the underlying code thinks that the underlying CDFs do not +% have consistent metadata. See solo.db_get_ts(). % function Ts = db_get_ts(varargin) @@ -504,7 +507,7 @@ function quicklooks_7days_local(Tint, vht6h, Paths, logoPath) -% Takes a cell-array of TSeries and merges them to one TSeries. +% Take a cell array of TSeries and merges them into one TSeries. function OutputTs = cell_array_TS_to_TS(InputTs) assert(iscell(InputTs)) From 5969f5d7c0d7348bde7d5d4da71f288b24703d47 Mon Sep 17 00:00:00 2001 From: Erik P G Johansson Date: Mon, 19 Feb 2024 13:22:35 +0100 Subject: [PATCH 2/7] solo.qli: Cleanup modified: +qli/quicklooks_24_6_2_h.m modified: +qli/quicklooks_main.m modified: +qli/quicklooks_main_cron.m modified: read_TNR.m --- .../+solo/+qli/quicklooks_24_6_2_h.m | 21 ++++++++ .../+solo/+qli/quicklooks_main.m | 48 ++++++++++--------- .../+solo/+qli/quicklooks_main_cron.m | 9 ++-- mission/solar_orbiter/+solo/read_TNR.m | 8 ++-- 4 files changed, 57 insertions(+), 29 deletions(-) diff --git a/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m b/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m index 20a0d92ea..aa391dccd 100644 --- a/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m +++ b/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m @@ -4,6 +4,12 @@ function quicklooks_24_6_2_h(data,paths,Tint_24h,logoPath) % solo.qli.quicklooks_main). Computes spectrum of B, so takes a while to run. % Tint_24h should be a 24hour time interval, e.g. % irf.tint('2020-06-01T00:00:00.00Z','2020-06-02T00:00:00.00Z'); +% +% NOTE: Function uses solo.read_TNR() indirectly which in turns relies on a +% hardcoded path to "/data/solo/remote/data/L2/thr/" and selected +% subdirectories. + + % BUG?: Panel 2/density/abs(B): Sometimes has no left-hand ticks (for density?). % /EJ 2023-05-10 @@ -33,6 +39,21 @@ function quicklooks_24_6_2_h(data,paths,Tint_24h,logoPath) % TODO-NI Panel 10 (log) is hardcoded to YLim~[10, 100] (because that is what % it used to be). This does not cover the entire interval of data % (there is more data at lower y). Should it be that way? +% +% PROPOSAL: Make function not directly call solo.read_TNR() +% PRO: Makes function testable. +% CON: Must understand the solo.read_TNR() return value. +% CON: Seems feasible. +% case 0: +% out = 0; +% case 1: +% out = struct('t', time_.epochUnix, 'f', freq_tnr, 'p',vp.^10); +% out.p_label={'dB'}; +% +% PROPOSAL: Only call solo.read_TNR() via dependency injection. +% CON: Overkill. +% PROPOSAL: Submit the return value of solo.read_TNR() as argument instead of +% calling it. diff --git a/mission/solar_orbiter/+solo/+qli/quicklooks_main.m b/mission/solar_orbiter/+solo/+qli/quicklooks_main.m index b58438e3d..c55817b52 100644 --- a/mission/solar_orbiter/+solo/+qli/quicklooks_main.m +++ b/mission/solar_orbiter/+solo/+qli/quicklooks_main.m @@ -5,17 +5,20 @@ % Intended for batch processing, e.g. being called from bash script, e.g. cron % job via a MATLAB wrapper script. % -% NOTE: Requires solo.db_init() to have been properly used to initialize dataset -% lookup. -% NOTE: Uses SPICE implicitly, and therefore relies on some path convention. Not -% sure which, but presumably it does at least find /data/solo/SPICE/. -% NOTE: Uses solo.read_TNR() indirectly which in turns relies on a hardcoded -% path to "/data/solo/remote/data/L2/thr/" and selected subdirectories. -% NOTE: Creates subdirectories to the output directory if not pre-existing. -% NOTE: 7-day plots will only cover that largest sub-time interval that is -% composed of full 7-day periods. Note: 7-day periods begin with a -% hardcoded weekday (Wednesday as of 2023-07-24). -% NOTE: Overwrites pre-existing plot files without warning. +% +% NOTES +% ===== +% * Requires solo.db_init() to have been properly used to initialize dataset +% lookup. +% * Uses SPICE implicitly, and therefore relies on some path convention. Not +% sure which, but presumably it does at least find /data/solo/SPICE/. +% * Uses solo.read_TNR() indirectly which in turns relies on a hardcoded +% path to "/data/solo/remote/data/L2/thr/" and selected subdirectories. +% * Creates subdirectories to the output directory if not pre-existing. +% * 7-day plots will only cover that largest sub-time interval that is +% composed of full 7-day periods. Note: 7-day periods begin with a +% hardcoded weekday (Wednesday as of 2023-07-24). +% * Overwrites pre-existing plot files without warning. % % % ARGUMENTS @@ -124,7 +127,7 @@ function quicklooks_main(... -% NOTE: Usually found on solo/data_yuri. +% NOTE: Usually found at /data/solo/data_yuri/. VHT_1H_DATA_FILENAME = 'V_RPW_1h.mat'; VHT_6H_DATA_FILENAME = 'V_RPW.mat'; @@ -186,13 +189,13 @@ function quicklooks_main(... if runNonweeklyPlots % Daily time-intervals Time1DayStepsArray = make_time_array(TimeIntervalNonWeeks, 1); - + % Load data % This is the .mat file containing RPW speeds at 1h resolution. % The file should be in the current path. This file can be found in % brain:/solo/data/data_yuri/. vht1h = load(fullfile(vhtDataDir, VHT_1H_DATA_FILENAME)); - + for iTint=1:length(Time1DayStepsArray)-1 % Select time interval. Tint = irf.tint(Time1DayStepsArray(iTint), Time1DayStepsArray(iTint+1)); @@ -211,15 +214,15 @@ function quicklooks_main(... % Run the code for weekly overviews %=================================== if runWeeklyPlots - + % Weekly time-intervals Time7DayStepsArray = make_time_array(TimeIntervalWeeks, 7); - + % Load data % This is the .mat file containing RPW speeds at 6h resolution. % The file should be in the same folder as this script (quicklook_main). vht6h = load(fullfile(vhtDataDir, VHT_6H_DATA_FILENAME)); - + for iTint=1:length(Time7DayStepsArray)-1 % Select time interval. Tint = irf.tint(Time7DayStepsArray(iTint), Time7DayStepsArray(iTint+1)); @@ -436,7 +439,7 @@ function quicklooks_7days_local(Tint, vht6h, Paths, logoPath) if ~isempty(spiceEarthPos) [E_radius, E_lon, E_lat] = cspice_reclat(spiceEarthPos); earthPos = [E_radius', E_lon', E_lat']; - + Tlength = Tint(end)-Tint(1); dTimes = 0:dt:Tlength; Times = Tint(1)+dTimes; @@ -487,11 +490,12 @@ function quicklooks_7days_local(Tint, vht6h, Paths, logoPath) -% Wrapper around solo.db_get_ts() that normalizes the output to a TSeries. +% Wrapper around solo.db_get_ts() which normalizes the output to always return +% one TSeries object. % -% NOTE: solo.db_get_ts() may sometimes return cell array of TSeries instead of a -% single TSeries when the underlying code thinks that the underlying CDFs do not -% have consistent metadata. See solo.db_get_ts(). +% NOTE: solo.db_get_ts() returns a cell array of TSeries instead of a single +% TSeries when the underlying code thinks that the underlying CDFs do not have +% consistent metadata. See solo.db_get_ts(). % function Ts = db_get_ts(varargin) diff --git a/mission/solar_orbiter/+solo/+qli/quicklooks_main_cron.m b/mission/solar_orbiter/+solo/+qli/quicklooks_main_cron.m index 39f9c0f68..985b61ed4 100644 --- a/mission/solar_orbiter/+solo/+qli/quicklooks_main_cron.m +++ b/mission/solar_orbiter/+solo/+qli/quicklooks_main_cron.m @@ -93,9 +93,9 @@ function quicklooks_main_cron(... % bash. irf -%================================= -% Configure Solar Orbiter database -%================================= +%=============================================================== +% Configure Solar Orbiter database from which data will be used +%=============================================================== % NOTE: System-dependent configuration! solo.db_init('local_file_db', '/data/solo/'); solo.db_init('local_file_db', '/data/solo/data_irfu'); @@ -103,6 +103,9 @@ function quicklooks_main_cron(... solo.db_init('db_cache_size_max', 4096) solo.db_cache('on', 'save') +%====== +% Plot +%====== solo.qli.quicklooks_main(... logoPath, vhtDataDir, outputDir, ... runNonweeklyPlots, runWeeklyPlots, utcBegin, utcEnd) diff --git a/mission/solar_orbiter/+solo/read_TNR.m b/mission/solar_orbiter/+solo/read_TNR.m index c85bd5e78..e924651af 100644 --- a/mission/solar_orbiter/+solo/read_TNR.m +++ b/mission/solar_orbiter/+solo/read_TNR.m @@ -1,4 +1,4 @@ -function out = read_TNR(tint) +function out = read_TNR(tint) % Read L2 data from TNR. % % @author: Louis Richard @@ -29,7 +29,7 @@ % % Returns % ------- -% out : strcut +% out : struct % Spectrum of the measured signals. % % Notes @@ -173,7 +173,7 @@ %select frequencies lower than 100 kHz -freq_tnr=freq_tnr(freq_tnr<100000); +freq_tnr = freq_tnr(freq_tnr<100000); f100_ind = length(freq_tnr); vp = v_(1:f100_ind, 2:end)'; @@ -198,7 +198,7 @@ out.p_label={'dB'}; -%For ploting +%For plotting % h(10)=irf_panel('tnr'); % irf_spectrogram(h(10),out,'log','donotfitcolorbarlabel') % %fpe_sc.units = 'kHz'; From 36a6cf9ad3d5e0f7cb5ee314f8bacaf5b01d7580 Mon Sep 17 00:00:00 2001 From: Erik P G Johansson Date: Mon, 19 Feb 2024 13:22:35 +0100 Subject: [PATCH 3/7] solo.qli: Cleanup modified: +qli/README.TXT modified: +qli/quicklooks_24_6_2_h.m modified: +qli/quicklooks_main.m modified: +qli/quicklooks_main_cron.m modified: read_TNR.m --- mission/solar_orbiter/+solo/+qli/README.TXT | 11 ++++++++++- .../solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m | 1 + mission/solar_orbiter/+solo/+qli/quicklooks_main.m | 6 ++++-- mission/solar_orbiter/+solo/read_TNR.m | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/mission/solar_orbiter/+solo/+qli/README.TXT b/mission/solar_orbiter/+solo/+qli/README.TXT index 5a6360edb..3d1c6204f 100644 --- a/mission/solar_orbiter/+solo/+qli/README.TXT +++ b/mission/solar_orbiter/+solo/+qli/README.TXT @@ -3,5 +3,14 @@ QLI = QuickLooks IRFU ("IRFU quicklooks") SolO quicklooks generated at IRFU. -NOTE: Uses NOT only SolO/RPW data. +NOTE: Uses SolO/RPW data AND data from instruments. NOTE: Not to be confused with LESIA's summary plots or BIA_QL3. + + + +If you want to edit the plots, check files + quicklooks_24_6_2_h.m + quicklooks_7days.m + +If you want to produce plots, call + quicklooks_main.m diff --git a/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m b/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m index aa391dccd..e845f4b7d 100644 --- a/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m +++ b/mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m @@ -1,4 +1,5 @@ function quicklooks_24_6_2_h(data,paths,Tint_24h,logoPath) +% % Given data in the struct 'data' (see solo.qli.quicklooks_main), generates % plots and saves them in the paths specified in the struct 'paths' (see % solo.qli.quicklooks_main). Computes spectrum of B, so takes a while to run. diff --git a/mission/solar_orbiter/+solo/+qli/quicklooks_main.m b/mission/solar_orbiter/+solo/+qli/quicklooks_main.m index c55817b52..a56e66c2a 100644 --- a/mission/solar_orbiter/+solo/+qli/quicklooks_main.m +++ b/mission/solar_orbiter/+solo/+qli/quicklooks_main.m @@ -37,8 +37,8 @@ % Whether to run the resp. groups of plots. % NOTE: Permits chars "0" and "1" for when calling from bash. % Useful for testing and not re-running unnecessary time-consuming plots. -% utcBegin, utcEnd : Strings. -% Defines time interval for which quicklooks should be generated. +% utcBegin, utcEnd +% Strings. Defines time interval for which quicklooks should be generated. % NOTE: Weekly plots will only be produced for those weeks which are % contained entirely inside the specified time interval (?). Weekly plots % always (as of 2023-07-24) begin on a Wednesday(!) at 00:00:00. @@ -111,6 +111,8 @@ function quicklooks_main(... % 2022-03-12T07:22:03.090373000Z -- 2022-03-13T00 % Missing data.Tpas +% ############################################################################## + %============ % ~Constants diff --git a/mission/solar_orbiter/+solo/read_TNR.m b/mission/solar_orbiter/+solo/read_TNR.m index e924651af..7514900bf 100644 --- a/mission/solar_orbiter/+solo/read_TNR.m +++ b/mission/solar_orbiter/+solo/read_TNR.m @@ -58,7 +58,7 @@ %TNR_BAND_FREQ from the TNR cdf file, therefore the dataobj(x) function %is used instead, which requires giving the full path of the file. %The solo.get_db_ts function seems to fail to create the TSeries object -%because the DEPEND_0 field is of diferent size from the data. +%because the DEPEND_0 field is of different size from the data. path = ['/data/solo/remote/data/L2/thr/' yyyy '/' mm '/solo_L2_rpw-tnr-surv-cdag_' yyyy mm dd '_V*.cdf']; data_l2 = rcdf(path, tint); From 59de4ecbc95e392655ab71d2695ae8ba57285ba0 Mon Sep 17 00:00:00 2001 From: Erik P G Johansson Date: Thu, 22 Feb 2024 21:28:43 +0100 Subject: [PATCH 4/7] solo.adm: Updated: bIsDatasetArray, require column arrays solo.adm.paths_to_DSMD_array(), parse_dataset_filename_many(): New additional return value bIsDatasetArray. Require column array argument. NOTE: Technically mildly backward-incompatible. --- .../+adm/assert_no_time_overlap___UTEST.m | 4 +-- ...MD_CURRENT_largest_time_coverage___UTEST.m | 10 +++--- .../+solo/+adm/parse_dataset_filename_many.m | 24 ++++++++++--- .../+solo/+adm/paths_to_DSMD_array.m | 12 ++++--- .../+solo/+adm/paths_to_DSMD_array___UTEST.m | 36 ++++++++++++------- 5 files changed, 59 insertions(+), 27 deletions(-) diff --git a/mission/solar_orbiter/+solo/+adm/assert_no_time_overlap___UTEST.m b/mission/solar_orbiter/+solo/+adm/assert_no_time_overlap___UTEST.m index 1d19ef3a8..86842bf6d 100644 --- a/mission/solar_orbiter/+solo/+adm/assert_no_time_overlap___UTEST.m +++ b/mission/solar_orbiter/+solo/+adm/assert_no_time_overlap___UTEST.m @@ -19,14 +19,14 @@ function test0(testCase) function test(filePathCa) - DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa); + DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa(:)); % Test that does not raise exception. solo.adm.assert_no_time_overlap(DsmdArray); end function test_exc(filePathCa) - DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa); + DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa(:)); testCase.verifyError(... @() solo.adm.assert_no_time_overlap(DsmdArray), ... ?MException) diff --git a/mission/solar_orbiter/+solo/+adm/filter_DSMD_CURRENT_largest_time_coverage___UTEST.m b/mission/solar_orbiter/+solo/+adm/filter_DSMD_CURRENT_largest_time_coverage___UTEST.m index a8779e72a..f08b6cccf 100644 --- a/mission/solar_orbiter/+solo/+adm/filter_DSMD_CURRENT_largest_time_coverage___UTEST.m +++ b/mission/solar_orbiter/+solo/+adm/filter_DSMD_CURRENT_largest_time_coverage___UTEST.m @@ -18,12 +18,12 @@ function test0(testCase) - function test(inputFileList, expFileList) - inputDsmdArray = solo.adm.paths_to_DSMD_array( inputFileList); - expDsmdArray = solo.adm.paths_to_DSMD_array(expFileList); + function test(inputFileCa, expFileCa) + InputDsmdArray = solo.adm.paths_to_DSMD_array(inputFileCa(:)); + ExpDsmdArray = solo.adm.paths_to_DSMD_array(expFileCa(:)); - actDsmdArray = solo.adm.filter_DSMD_CURRENT_largest_time_coverage(inputDsmdArray); - testCase.assertEqual(actDsmdArray, expDsmdArray) + ActDsmdArray = solo.adm.filter_DSMD_CURRENT_largest_time_coverage(InputDsmdArray); + testCase.assertEqual(ActDsmdArray, ExpDsmdArray) end X1 = 'solo_L2_rpw-lfr-surv-swf-e-cdag_20200314_V01.cdf'; diff --git a/mission/solar_orbiter/+solo/+adm/parse_dataset_filename_many.m b/mission/solar_orbiter/+solo/+adm/parse_dataset_filename_many.m index 4e810ac2c..d86c70ec9 100644 --- a/mission/solar_orbiter/+solo/+adm/parse_dataset_filename_many.m +++ b/mission/solar_orbiter/+solo/+adm/parse_dataset_filename_many.m @@ -10,6 +10,12 @@ % wrong that function name mentions filenames, not paths. % % +% ARGUMENT +% ======== +% filePathCa +% Cell column array of paths to files. Can be both datasets and not. +% +% % RETURN VALUE % ============ % fiCa @@ -18,12 +24,15 @@ % solo.adm.parse_dataset_filename() plus extra field % below: % .path : Path in filePathList{iFile}. +% bIsDatasetArray +% Logical column array. Same size as argument. True iff the corresponding +% input path was interpreted as a dataset (was translated into a DSMD). % % % Author: Erik P G Johansson, IRF, Uppsala, Sweden % First created 2020-04-25. % -function fiCa = parse_dataset_filename_many(filePathCa) +function [fiCa, bIsDatasetArray] = parse_dataset_filename_many(filePathCa) % PROPOSAL: Change name % PROPOSAL: parse_dataset_filenames_many ("FILENAMES" in plural) % PROPOSAL: parse_dataset_filename_many_paths @@ -38,10 +47,15 @@ % TODO-DEC: Different policy for suffix .cdf and not? % PROPOSAL: Assertion for parsable *.cdf filenames. Ignore filenames without % suffix ".cdf". +% +% NOTE: Has no separate test code. Is indirectly tested by +% solo.adm.paths_to_DSMD_array___UTEST. -assert(iscell(filePathCa), 'filePathList is not a cell array.') +assert(iscell(filePathCa), 'filePathCa is not a cell array.') +assert(iscolumn(filePathCa), 'filePathCa is not a column array.') -fiCa = cell(0,1); +fiCa = cell(0, 1); +bIsDatasetArray = false(numel(filePathCa), 1); for iFile = 1:numel(filePathCa) filename = irf.fs.get_name(filePathCa{iFile}); @@ -55,12 +69,14 @@ Fi = R; Fi.path = filePathCa{iFile}; - fiCa{end+1, 1} = Fi; + fiCa{ end+1, 1} = Fi; + bIsDatasetArray(iFile, 1) = true; else % CASE: Can not identify as dataset filename. % Do nothing. (Silently ignore files that can not be identified as % datasets.) + bIsDatasetArray(iFile, 1) = false; end end end diff --git a/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array.m b/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array.m index 92a9bfef9..2b20ed4d6 100644 --- a/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array.m +++ b/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array.m @@ -17,13 +17,16 @@ % RETURN VALUE % ============ % DsmdArray -% Array of DSMD objects. +% Column array of DSMD objects. +% bIsDatasetArray +% Logical column array. Same size as argument. True iff the corresponding +% input path was interpreted as a dataset (was translated into a DSMD). % % % Author: Erik P G Johansson, IRF, Uppsala, Sweden % First created 2020-05-08. % -function DsmdArray = paths_to_DSMD_array(filePathCa) +function [DsmdArray, bIsDatasetArray] = paths_to_DSMD_array(filePathCa) % PROPOSAL: Rename % PROPOSAL: DSMDs_from_paths() % PROPOSAL: paths_to_DSMDs(). @@ -47,9 +50,10 @@ % from parsable filename: ignore, warning, error. % FI = File Info -fiCa = solo.adm.parse_dataset_filename_many(filePathCa); +[fiCa, bIsDatasetArray] = solo.adm.parse_dataset_filename_many(filePathCa); + +DsmdArray = solo.adm.DSMD.empty(0, 1); -DsmdArray = solo.adm.DSMD.empty(0,1); for i = 1:numel(fiCa) Fi = fiCa{i}; diff --git a/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array___UTEST.m b/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array___UTEST.m index e6261b320..d76548d26 100644 --- a/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array___UTEST.m +++ b/mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array___UTEST.m @@ -26,9 +26,10 @@ function test0(testCase) % solo_L1_rpw-bia-sweep-cdag_20200307T053018-20200307T053330_V01.cdf % solo_L1_rpw-bia-current-cdag_20200401T000000-20200421T000000_V01.cdf - function test(filePathList, expDsmdArray) - actDsmdArray = solo.adm.paths_to_DSMD_array(filePathList); - testCase.assertEqual(expDsmdArray, actDsmdArray) + function test(filePathList, expDsmdArray, expBIsDatasetArray) + [actDsmdArray, actBIsDatasetArray] = solo.adm.paths_to_DSMD_array(filePathList(:)); + testCase.assertEqual(actDsmdArray, expDsmdArray) + testCase.assertEqual(actBIsDatasetArray, expBIsDatasetArray) end function dt = dtu(varargin) @@ -60,26 +61,37 @@ function test(filePathList, expDsmdArray) % Empty argument (no files) - test({}, solo.adm.DSMD.empty(0,1)); + test(... + {}, ... + solo.adm.DSMD.empty(0,1), ... + false(0,1)); - test({FILE_CDF_IGNORE}, solo.adm.DSMD.empty(0,1)); - test({FILE_NONCDF_IGNORE}, solo.adm.DSMD.empty(0,1)); + test(... + {FILE_CDF_IGNORE}, ... + solo.adm.DSMD.empty(0,1), ... + false(1,1)); + test(... + {FILE_NONCDF_IGNORE}, ... + solo.adm.DSMD.empty(0,1), ... + false(1,1)); test(... {DSMD_1.path}, ... - [DSMD_1]); - + [DSMD_1], ... + true(1,1)); test(... {DSMD_2.path}, ... - [DSMD_2]); - + [DSMD_2], ... + true(1,1)); test(... {DSMD_3.path}, ... - [DSMD_3]); + [DSMD_3], ... + true(1,1)); test(... {DSMD_1.path; FILE_NONCDF_IGNORE; DSMD_2.path; FILE_CDF_IGNORE; DSMD_3.path}, ... - [DSMD_1; DSMD_2; DSMD_3]); + [DSMD_1; DSMD_2; DSMD_3], ... + logical([1; 0; 1; 0; 1])); end From 833863ce127da77b64311916f097d796fe1b5a91 Mon Sep 17 00:00:00 2001 From: Erik P G Johansson Date: Tue, 20 Feb 2024 14:07:41 +0100 Subject: [PATCH 5/7] solo: Cleanup (minor) modified: +solo/+adm/find_overlapping_DSMD_groups.m modified: +solo/psp2ne.m modified: +solo/vdccal.m --- .../+solo/+adm/find_overlapping_DSMD_groups.m | 3 ++- mission/solar_orbiter/+solo/psp2ne.m | 6 +++--- mission/solar_orbiter/+solo/vdccal.m | 8 ++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mission/solar_orbiter/+solo/+adm/find_overlapping_DSMD_groups.m b/mission/solar_orbiter/+solo/+adm/find_overlapping_DSMD_groups.m index fe8b00ff1..ed5977e06 100644 --- a/mission/solar_orbiter/+solo/+adm/find_overlapping_DSMD_groups.m +++ b/mission/solar_orbiter/+solo/+adm/find_overlapping_DSMD_groups.m @@ -6,7 +6,8 @@ % NOTE: Overlap must have non-zero length. % % NOTE: Algorithm only works under the assumption that datasets with the same -% DATASET_ID do not overlap (in time) (assertion). +% DATASET_ID do not overlap (in time) (assertion). Can thus not handle +% SOLO_L2_RPW-LFR-SBM1/2-CWF-E. % % % ARGUMENTS diff --git a/mission/solar_orbiter/+solo/psp2ne.m b/mission/solar_orbiter/+solo/psp2ne.m index 034fe3f59..5532b1b90 100644 --- a/mission/solar_orbiter/+solo/psp2ne.m +++ b/mission/solar_orbiter/+solo/psp2ne.m @@ -162,9 +162,9 @@ % not yet used by BICAS (2023-08-10). % NOTE: Overwrite every value with zero in order to also overwrite Nan which % may otherwise inherited from NeScp. -%NOTE: Density from TNR plasma line used to calibrate NeScp only measues up -%to 122 cc, everything above that value is uncertain, therefore is flagged. -%Low values of NeScp i.e <2 cc are also uncertain +% NOTE: Density from TNR plasma line used to calibrate NeScp only measures up +% to 122 cc, everything above that value is uncertain, therefore is flagged. +% Low values of NeScp i.e <2 cc are also uncertain. NeScpQualityBit = TSeries(NeScp.time, ones(size(NeScp.data))); NeScpQualityBit.data(NeScp.data<=122 & NeScp.data>=2) = 0; NeScpQualityBit.data(isnan(NeScpQualityBit.data)) = 0; diff --git a/mission/solar_orbiter/+solo/vdccal.m b/mission/solar_orbiter/+solo/vdccal.m index 6f5e03a3c..6811775a0 100644 --- a/mission/solar_orbiter/+solo/vdccal.m +++ b/mission/solar_orbiter/+solo/vdccal.m @@ -153,7 +153,7 @@ % ======================= VDC = VDC_inp.tlim(subTint); - % Indices/samples for which which should be treated as single probe. + % Indices/samples for which data should be treated as single probe. bSingleProbe = isnan(VDC.y.data) & isnan(VDC.z.data); bSingleProbe = bSingleProbe | (VDC.time > TIME_PSP_BEGIN_SINGLE_PROBE); @@ -183,7 +183,9 @@ PSP.units = 'V'; PSP_out = PSP_out.combine(PSP); - PLASMA_POT = 1.5; SHORT_FACTOR = 2.5; % XXX: these are just ad hoc numbers. + % XXX: these are just ad hoc numbers. + PLASMA_POT = 1.5; + SHORT_FACTOR = 2.5; ScPot = irf.ts_scalar(VDC.time, -(PSP.data-PLASMA_POT)*SHORT_FACTOR); ScPot.units = PSP.units; @@ -201,6 +203,8 @@ % in Steinvall et al., 2021. Ez_SRF = (V23_scaled - V1)*1e3/11.2; + % NOTE: Ey_SRF may contain NaN. Therefore Ey_SRF*0 != zeros(size(Ey_SRF)). + % (Bug?!) DCE_SRF = irf.ts_vec_xyz(VDC.time, [Ey_SRF*0 Ey_SRF Ez_SRF]); DCE_SRF.units = 'mV/m'; DCE_SRF.coordinateSystem = 'SRF'; From eca22ab2dd5b5c53e36f7ba3b259638281ca735f Mon Sep 17 00:00:00 2001 From: Erik P G Johansson Date: Mon, 26 Feb 2024 10:14:07 +0100 Subject: [PATCH 6/7] BICAS: Add bicas.tools.batch package (on devel) --- irf/+irf/+fs/create_empty_file.m | 26 ++ .../+tools/+batch/BPCI_to_BICAS_call_args.m | 31 ++ .../+batch/BicasProcessingAccessAbstract.m | 30 ++ .../+tools/+batch/BicasProcessingAccessImpl.m | 35 ++ .../+tools/+batch/BicasProcessingAccessTest.m | 130 ++++++ .../+tools/+batch/BicasProcessingCallInfo.m | 96 +++++ .../+batch/BicasProcessingCallSummary.m | 49 +++ .../src/+bicas/+tools/+batch/BpciInput.m | 45 ++ .../src/+bicas/+tools/+batch/BpciOutput.m | 59 +++ .../+bicas/+tools/+batch/DSMDs_to_filenames.m | 29 ++ .../bicas/src/+bicas/+tools/+batch/README.TXT | 11 + .../+bicas/+tools/+batch/TestSwmProcessing.m | 57 +++ .../+tools/+batch/autocreate_input_BPCIs.m | 74 ++++ .../autocreate_input_BPCIs2___speedTest.m | 102 +++++ .../+batch/autocreate_input_BPCIs___UTEST.m | 273 ++++++++++++ .../+tools/+batch/autocreate_many_BPCIs.m | 45 ++ .../+tools/+batch/autocreate_one_SWM_BPCI.m | 97 +++++ .../+batch/default_get_BPCI_output_filename.m | 131 ++++++ .../+tools/+batch/filter_BPCIs_to_run.m | 54 +++ .../+batch/filter_BPCIs_to_run___UTEST.m | 139 ++++++ .../+tools/+batch/get_BPCI_output_path2.m | 212 +++++++++ .../+batch/get_BPCI_output_path2___UTEST.m | 180 ++++++++ .../+tools/+batch/get_directory_DSMDs.m | 56 +++ .../+batch/get_directory_DSMDs___UTEST.m | 96 +++++ .../bicas/src/+bicas/+tools/+batch/main.m | 391 +++++++++++++++++ .../src/+bicas/+tools/+batch/main_bash.m | 36 ++ .../+tools/+batch/run_BICAS_all_passes.m | 225 ++++++++++ .../+batch/run_BICAS_all_passes___UTEST.m | 406 ++++++++++++++++++ .../+tools/+batch/try_run_BICAS_for_BPCIs.m | 119 +++++ .../+batch/try_run_BICAS_for_BPCIs___UTEST.m | 285 ++++++++++++ 30 files changed, 3519 insertions(+) create mode 100644 irf/+irf/+fs/create_empty_file.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BPCI_to_BICAS_call_args.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessAbstract.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessImpl.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessTest.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallInfo.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallSummary.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciInput.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciOutput.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/DSMDs_to_filenames.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/README.TXT create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/TestSwmProcessing.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs2___speedTest.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs___UTEST.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_many_BPCIs.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_one_SWM_BPCI.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/default_get_BPCI_output_filename.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run___UTEST.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2___UTEST.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs___UTEST.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main_bash.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes___UTEST.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs.m create mode 100644 mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs___UTEST.m diff --git a/irf/+irf/+fs/create_empty_file.m b/irf/+irf/+fs/create_empty_file.m new file mode 100644 index 000000000..44b09d236 --- /dev/null +++ b/irf/+irf/+fs/create_empty_file.m @@ -0,0 +1,26 @@ +% +% Create empty file. This is useful for debugging purposes sometimes, e.g. +% if batch code selects the correct files to be created. +% +% +% ARGUMENTS +% ========= +% path +% Path to file. +% NOTE: Parent directory has to pre-exist. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function create_empty_file(path) + % PROPOSAL: Error if fails to create file. + % PROPOSAL: Policy arguments + % (1) Whether to permit path not available: + % Permit overwrite + % Assert no overwrite + % (2) What happens when createing, writing file: + % Assert write success (does not assert no overwrite) + % Permit write failure (does not assert no overwrite), return ~error code/boolean + + fclose(fopen(path, 'w')); +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BPCI_to_BICAS_call_args.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BPCI_to_BICAS_call_args.m new file mode 100644 index 000000000..9b734231c --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BPCI_to_BICAS_call_args.m @@ -0,0 +1,31 @@ +% +% Convert a BPCI into a sequence of arguments that can be used for calling +% BICAS. +% +% +% Initially created 2020-03-02 by Erik P G Johansson, IRF, Uppsala, Sweden. +% +function argsCa = BPCI_to_BICAS_call_args(Bpci) +% PROPOSAL: Refactor into BPCI method. +% PROPOSAL: No hardcoded CLI_OPTION_PREFIX +% PROPOSAL: CLI_OPTION_PREFIX as argument. +% PROPOSAL: CLI_OPTION_PREFIX using BICAS constant. + + CLI_OPTION_PREFIX = '--'; + + assert(isa(Bpci, 'bicas.tools.batch.BicasProcessingCallInfo')) + + argsCa = {Bpci.swmCliOption}; + + for i = 1:numel(Bpci.inputsArray) + In = Bpci.inputsArray(i); + argsCa{end+1} = [CLI_OPTION_PREFIX, In.cohb]; + argsCa{end+1} = In.path; + end + + for i = 1:numel(Bpci.outputsArray) + Out = Bpci.outputsArray(i); + argsCa{end+1} = [CLI_OPTION_PREFIX, Out.cohb]; + argsCa{end+1} = Out.path; + end +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessAbstract.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessAbstract.m new file mode 100644 index 000000000..46724e5e8 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessAbstract.m @@ -0,0 +1,30 @@ +% +% Abstract class for handling the communication with BICAS for the purpose of +% processing data. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef(Abstract) BicasProcessingAccessAbstract < handle + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Abstract) + + + + % Call bicas.main() with the exact same arguments and return value(s). + [varargout] = bicas_main(obj, varargin); + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessImpl.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessImpl.m new file mode 100644 index 000000000..1a96850f3 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessImpl.m @@ -0,0 +1,35 @@ +% +% Implementation of abstract class for nominal use. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef BicasProcessingAccessImpl < bicas.tools.batch.BicasProcessingAccessAbstract + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Access=public) + + + + % OVERRIDE + function [varargout] = bicas_main(obj, varargin) + for i = 1:numel(varargin) + assert(ischar(varargin{i})) + end + + [varargout{1:nargout}] = bicas.main(varargin{:}); + end + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessTest.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessTest.m new file mode 100644 index 000000000..f553bbefe --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingAccessTest.m @@ -0,0 +1,130 @@ +% +% Implementation for automatic tests. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef BicasProcessingAccessTest < bicas.tools.batch.BicasProcessingAccessAbstract + % PROPOSAL: Support returning non-zero error code. + % PROPOSAL: Support raising exception. + + + + %##################### + %##################### + % INSTANCE PROPERTIES + %##################### + %##################### + properties(SetAccess=immutable, GetAccess=private) + SwmArray + callNonZeroErrorArray + end % properties(SetAccess=immutable) + properties(Access=private) + % The number of times method "bicas_main" has been called. + nCalls + end + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Access=public) + + + + % callNonZeroErrorArray + % Array of numbers. Call numbers for when method bicas_main() + % should return non-zero error code and simulate failure. + % 1=First call. + % + function obj = BicasProcessingAccessTest(SwmArray, callNonZeroErrorArray) + assert(isa(SwmArray, 'bicas.swm.SoftwareMode') & iscolumn(SwmArray)) + assert(isnumeric(callNonZeroErrorArray)) + assert(iscolumn(callNonZeroErrorArray) || isempty (callNonZeroErrorArray)) + + obj.SwmArray = SwmArray; + obj.callNonZeroErrorArray = callNonZeroErrorArray; + obj.nCalls = 0; + end + + + + % OVERRIDE + function [varargout] = bicas_main(obj, varargin) + + obj.nCalls = obj.nCalls + 1; + iCall = obj.nCalls; + if ismember(iCall, obj.callNonZeroErrorArray) + %============================= + % CASE: Return non-zero error + %============================= + + % Do no processing (generate no output files) + % ------------------------------------------- + % NOTE: One could imagine simulating an error after or between + % output datasets are generated but that should be overkill. + + [varargout{1}] = 1; + else + %======================================= + % CASE: Error code zero. Do processing. + %======================================= + + for i = 1:numel(varargin) + assert(ischar(varargin{i})) + end + + swmCliOption = varargin{1}; + + iSwm = find(strcmp(swmCliOption, {obj.SwmArray(:).cliOption})); + assert(isscalar(iSwm), 'Did not find exactly one SWM which matches CLI arg. "%s".', swmCliOption) + Swm = obj.SwmArray(iSwm); + + + + %========================================= + % Input datasets: Assert that files exist + %========================================= + for iInput = 1:numel(Swm.inputsList) + cohb = Swm.inputsList(iInput).cliOptionHeaderBody; + iCoh = find(strcmp(['--', cohb], varargin)); + assert(isscalar(iCoh), ... + 'Can not identify exactly one BICAS argument with cohb="%s"', cohb) + inputPath = varargin{iCoh+1}; + + % ASSERTION: Input dataset exists. + irf.assert.file_exists(inputPath) + end + + %=============================================== + % Output datasets: Create empty output datasets + %=============================================== + for iOutput = 1:numel(Swm.outputsList) + cohb = Swm.outputsList(iOutput).cliOptionHeaderBody; + iCoh = find(strcmp(['--', cohb], varargin)); + outputFilename = varargin{iCoh+1}; + + % NOTE: Can not assert that output datasets do not pre-exist, + % since bicas.batch.main permits overwriting files. + + % CREATE EMPTY OUTPUT DATASET. + irf.fs.create_empty_file(outputFilename); + end + + [varargout{1}] = 0; + end + + + + end + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallInfo.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallInfo.m new file mode 100644 index 000000000..f8727868d --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallInfo.m @@ -0,0 +1,96 @@ +% +% Info required for processing datasets using one single BICAS call, but not the +% argument list as such. +% +% +% Author: Erik P G Johansson, Uppsala, Sweden +% First created 2020-05-27. +% +classdef BicasProcessingCallInfo + % PROPOSAL: Better name. "info" is too generic. + % NOTE: Compare bicas.tools.batch.BicasProcessingCallSummary. + % NOTE: Only includes information on SWM + input + output, but excludes + % custom settings. Excludes --version, --swdescriptor, --config etc. + % PROPOSAL: Only imply specifying datasets (paths) in & out, not + % specifying the entire call. + % -- + % BICAS + % Datasets + % Paths + % Input/output (datasets) + % I/O + % Processing (as opposed to non-processing) + % Data, Info, Record + % -- + % BicasCallPathsIO, BicasCallDatasetsIO + % + % PROPOSAL: Be able to generate BPCI from SWM for testing purposes. + % See bicas.tools.batch.autocreate_one_SWM_BPCI(). + % CON: Still needs to be able to associate COBHs with paths and filenames. + + + + %##################### + %##################### + % INSTANCE PROPERTIES + %##################### + %##################### + properties(SetAccess=immutable) + swmCliOption + inputsArray + outputsArray + end + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Access=public) + + + + function obj = BicasProcessingCallInfo(swmCliOption, inputsArray, outputsArray) + assert(ischar(swmCliOption)) + assert(iscolumn(inputsArray) & isa(inputsArray, 'bicas.tools.batch.BpciInput')) + assert(iscolumn(outputsArray) & isa(outputsArray, 'bicas.tools.batch.BpciOutput')) + + obj.swmCliOption = swmCliOption; + obj.inputsArray = inputsArray; + obj.outputsArray = outputsArray; + end + + + + function filenameCa = get_output_filenames(obj) + filenameCa = cellfun(... + @irf.fs.get_name, {obj.outputsArray.path}', ... + 'UniformOutput', false); + filenameCa = filenameCa(:); + end + + + + % Create modified copy of object. Prepend all output paths with a + % specified path. + function obj = prepend_output_directory(obj, outputDir) + outputsArray = bicas.tools.batch.BpciOutput.empty(0, 1); + for i = 1:numel(obj.outputsArray) + o = obj.outputsArray(i); + outputsArray(i, 1) = bicas.tools.batch.BpciOutput(... + o.cohb, o.dsi, fullfile(outputDir, o.path)); + end + + obj = bicas.tools.batch.BicasProcessingCallInfo(... + obj.swmCliOption, obj.inputsArray, outputsArray); + end + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallSummary.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallSummary.m new file mode 100644 index 000000000..22508cd24 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BicasProcessingCallSummary.m @@ -0,0 +1,49 @@ +% +% Each instance summarizes a completed BICAS call, including error code. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef BicasProcessingCallSummary + + + + %##################### + %##################### + % INSTANCE PROPERTIES + %##################### + %##################### + properties(SetAccess=immutable) + Bpci + + % BICAS error code + errorCode + end + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Access=public) + + + + function obj = BicasProcessingCallSummary(Bpci, errorCode) + + assert(isa(Bpci, 'bicas.tools.batch.BicasProcessingCallInfo')) + assert(isnumeric(errorCode)) + + obj.Bpci = Bpci; + obj.errorCode = errorCode; + end + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciInput.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciInput.m new file mode 100644 index 000000000..66414988f --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciInput.m @@ -0,0 +1,45 @@ +% +% Helper class to bicas.tools.batch.BicasProcessingCallInfo. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef BpciInput + + + + %##################### + %##################### + % INSTANCE PROPERTIES + %##################### + %##################### + properties(SetAccess=immutable) + cohb + dsi + path + end + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Access=public) + + + + function obj = BpciInput(cohb, dsi, path) + obj.cohb = cohb; + obj.dsi = dsi; + obj.path = path; + end + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciOutput.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciOutput.m new file mode 100644 index 000000000..9653a812a --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/BpciOutput.m @@ -0,0 +1,59 @@ +% +% Helper class to bicas.tools.batch.BicasProcessingCallInfo. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef BpciOutput + % PROPOSAL: Replace property "filename" with "path". -- IMPLEMENTED + % PRO: Analogous with bicas.tools.batch.BpciInput. + % PRO: Can maybe use object in BPTD (instead of this class converted to + % struct which is then modified). + % PRO: A path is actually sent to BICAS. More accurate representation of + % a call to BICAS. + % NOTE: Autocreation of BPCIs needs way of setting paths (not just + % filenames) of output datasets. + % Ex: bicas.tools.batch.autocreate_one_SWM_BPCI(). + % PROPOSAL: Submit output directory to those functions. + % PROPOSAL: Redefine ~createOutputFilenameFh to set entire path. + % PRO: Most generic. + % NOTE/PROBLEM: Class becomes identical to + % bicas.tools.batch.BpciInput! + + + + %##################### + %##################### + % INSTANCE PROPERTIES + %##################### + %##################### + properties(SetAccess=immutable) + cohb + dsi + path + end + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Access=public) + + + + function obj = BpciOutput(cohb, dsi, path) + obj.cohb = cohb; + obj.dsi = dsi; + obj.path = path; + end + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/DSMDs_to_filenames.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/DSMDs_to_filenames.m new file mode 100644 index 000000000..051fee06f --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/DSMDs_to_filenames.m @@ -0,0 +1,29 @@ +% +% Convert multiple DSMDs to filenames (not paths). +% +% +% ARGUMENTS +% ========= +% DsmdArray +% Array of solo.adm.DSMD. +% +% +% RETURN VALUES +% ============= +% filenamesCa +% Cell array of same size as DsmdArray. Filenames (not paths) in +% DsmdArray. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function filenamesCa = DSMDs_to_filenames(DsmdArray) + % PROPOSAL: Automatic test code. + % PROPOSAL: Move to DSMD class. + + assert(isa(DsmdArray, 'solo.adm.DSMD')) + + filenamesCa = cellfun(... + @irf.fs.get_name, {DsmdArray.path}, ... + 'UniformOutput', false); +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/README.TXT b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/README.TXT new file mode 100644 index 000000000..a066a662d --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/README.TXT @@ -0,0 +1,11 @@ +Package for batch-processing datasets using BICAS. + + +ABBREVIATIONS +============= +BPA + Class bicas.tools.batch.BicasProcessingAccess (and + subclasses). +BPCS + Class bicas.tools.batch.BicasProcessingCallSummary. + NOTE: Not to be confused with BPCI! diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/TestSwmProcessing.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/TestSwmProcessing.m new file mode 100644 index 000000000..59bcfda8d --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/TestSwmProcessing.m @@ -0,0 +1,57 @@ +% +% Implementation for being able to build SWMs for tests. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef TestSwmProcessing < bicas.proc.SwmProcessing + + + + %##################### + %##################### + % INSTANCE PROPERTIES + %##################### + %##################### + properties(SetAccess=immutable) + end % properties(SetAccess=immutable) + + + + %######################### + %######################### + % PUBLIC INSTANCE METHODS + %######################### + %######################### + methods(Access=public) + + + + % ARGUMENTS + % ========= + % InputsMap + % containers.Map with + % : String defining a name of an input ("prodFuncInputKey" in + % bicas.swm.SoftwareModeList). + % : A struct with data corresponding to a CDF file + % (zVariables+global attributes). + % OutputsMap + % containers.Map with + % : String defining a name of an output ("prodFuncOutputKey" in + % bicas.swm.SoftwareModeList). + % : A struct with data corresponding to a CDF file (zVariables). + % + % OVERRIDE + function OutputDatasetsMap = production_function(obj, ... + InputDatasetsMap, rctDir, NsoTable, Bso, L) + + OutputDatasetsMap = containers.Map(); + end + + + + end % methods(Access=public) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs.m new file mode 100644 index 000000000..16cd76afe --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs.m @@ -0,0 +1,74 @@ +% +% Autocreate all BPCIs which can be derived from specified paths. +% +% NOTE: If the reference directory is simultaneously specified as an input path +% (optional), then that is included here too. +% +% +% ARGUMENTS +% ========= +% get_BPCI_output_path_fh +% Function handle. +% path = @(outputDsi, BpciInputDsmdArray, cohbCa). +% Determines file paths for BPCI output datasets. +% +% +% RETURN VALUES +% ============= +% BpciArray +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function BpciArray = autocreate_input_BPCIs(... + InputDsmdArray, get_BPCI_output_path_fh, SwmArray, currentDatasetExtensionDays) + + % IMPLEMENTATION NOTE: Setting will probably eventually be abolished by the + % function solo.adm.group_sort_DSMD_versions() that uses it, but it is kept + % here as a global constant (instead of an argument) for clarity. + SETTINGS_sortWrtFormerVersionsDir = false; + + + + %===================================== + % Find all DSMDs in INPUT directories + %===================================== + t = tic(); + fprintf('INPUT paths: #Datasets, all versions (DSMDs): %i\n', numel(InputDsmdArray)) + fprintf('SPEED: bicas.tools.batch.autocreate_input_BPCIs(): Time to obtain input DSMDs: %.1f [s] wall time\n', toc(t)); + + + + %============================ + % Filter out latest versions + %============================ + + % Must do BEFORE filtering latest version. + InputDsmdArray = solo.adm.convert_DSMD_DATASET_ID_to_SOLO(InputDsmdArray); + + % NOTE: Might be slow. + t = tic(); + InputDsmdArray = solo.adm.group_sort_DSMD_versions(... + InputDsmdArray, 'latest', ... + 'sortWrtFormerVersionsDir', SETTINGS_sortWrtFormerVersionsDir); + fprintf('SPEED: bicas.tools.batch.autocreate_input_BPCIs(): solo.adm.group_sort_DSMD_versions(): %.1f [s] wall time\n', toc(t)); + + % NOTE: Must do AFTER filtering latest version. The function extends the + % DSMD time intervals which would make solo.adm.group_sort_DSMD_versions() + % fail to group the datasets correctly. + InputDsmdArray = solo.adm.extend_last_CURRENT_DSMD(... + InputDsmdArray, currentDatasetExtensionDays); + fprintf('INPUT paths: #Datasets (DSMDs), latest versions: %i\n', numel(InputDsmdArray)) + + + + %============================================= + % Find all BPCIs, based on available datasets + %============================================= + t = tic(); + BpciArray = bicas.tools.batch.autocreate_many_BPCIs(... + InputDsmdArray, SwmArray, ... + get_BPCI_output_path_fh); + fprintf('SPEED: bicas.tools.batch.autocreate_input_BPCIs(): autocreate_many_BPCIs(): %.1f [s] wall time\n', toc(t)); + fprintf('#BPCIs, found total: %3i\n', numel(BpciArray)) +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs2___speedTest.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs2___speedTest.m new file mode 100644 index 000000000..9cec383bc --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs2___speedTest.m @@ -0,0 +1,102 @@ +% +% Test performance of +% bicas.tools.batch.autocreate_input_BPCIs(). +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function autocreate_input_BPCIs2___speedTest() + + % profile clear; profile on; + + nArray = []; + wallTimeSecArray = []; + + N_CALLS = 5; + + for n = round(logspace(log10(20), log10(1000), N_CALLS)) + % NOTE: test() prints log messages. + wallTimeSec = test(n); + + % Log + n + wallTimeSec + + nArray(end+1) = n; + wallTimeSecArray(end+1) = wallTimeSec; + end + + close all + figure('WindowState', 'maximized') + plot(nArray, wallTimeSecArray, 'o-') + grid on + + % profile off; profile viewer +end + + + +function wallTimeSec = test(n) + DSI_1 = 'SOLO_L1R_RPW-LFR-SURV-CWF-E'; + DSI_2 = 'SOLO_L2_RPW-LFR-SURV-CWF-E'; + + SWMP = bicas.tools.batch.TestSwmProcessing(); + + SWM_1 = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_1', 'SWD purpose', ... + bicas.swm.InputDataset( 'cli_in', DSI_1, 'IN_cdf'), ... + bicas.swm.OutputDataset('cli_out', DSI_2, 'OUT_cdf', 'SWD ', 'SWD ', '02') ... + ); + + path1Ca = get_dataset_paths(n, 'solo_L1R_rpw-lfr-surv-cwf-e_%4i%02i%02i_V02.cdf'); + path2Ca = get_dataset_paths(n, 'solo_L2_rpw-lfr-surv-cwf-e_%4i%02i%02i_V01.cdf'); + + InputDsmdArray = solo.adm.paths_to_DSMD_array(path1Ca); + preexistingOutputFilenamesCa = path2Ca; % Collision with every output file. + + OUTPUT_DIR = '/tmp/nonexisting'; % Should be irrelevant. + OUTPUT_ISCDAG = false; % Must match preexistingOutputFilenamesCa. + CURRENT_DATASET_EXTENSION_DAYS = 0; + + PreexistingOutputLvDsmdArray = solo.adm.paths_to_DSMD_array(preexistingOutputFilenamesCa); + + get_BPCI_output_path_fh = ... + @(outputDsi, BpciInputDsmdArray) ( ... + bicas.tools.batch.get_BPCI_output_path2( ... + BpciInputDsmdArray, PreexistingOutputLvDsmdArray, ... + outputDsi, ... + 'HIGHEST_USED', ... + OUTPUT_DIR, OUTPUT_ISCDAG ... + ) ... + ); + + % =============== + % RUN TESTED CODE + % =============== + t = tic(); + BpciInputArray = bicas.tools.batch.autocreate_input_BPCIs(... + InputDsmdArray, get_BPCI_output_path_fh, [SWM_1], ... + CURRENT_DATASET_EXTENSION_DAYS); + wallTimeSec = toc(t); + + assert(numel(BpciInputArray) == numel(InputDsmdArray)) + +end + + + +% Generate n dataset paths. +% +% NOTE: It is important that code generates correct dataset filenames which can be +% recognized as datasets. Otherwise the tested code will ignore them, and +% presumable run faster. +function filenameCa = get_dataset_paths(n, patternStr) + dt = datetime('2020-01-01') + caldays([1:n]-1)'; + + filenameCa = cell(n, 1); + for i = 1:n + assert(dt(i).Year < 10000) % Year must be four digits. + + filenameCa{i} = sprintf(patternStr, dt(i).Year, dt(i).Month, dt(i).Day); + end +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs___UTEST.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs___UTEST.m new file mode 100644 index 000000000..4bfd3ea26 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_input_BPCIs___UTEST.m @@ -0,0 +1,273 @@ +% +% matlab.unittest automatic test code for +% bicas.tools.batch.autocreate_input_BPCIs(). +% +% NOTE: Not (yet) testing +% indirect use of solo.adm.convert_DSMD_DATASET_ID_to_SOLO() +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef autocreate_input_BPCIs___UTEST < matlab.unittest.TestCase + + + + %############## + %############## + % TEST METHODS + %############## + %############## + methods(Test) + + + + function test_zero_one_BPCIs(testCase) + + function test(inputDatasetsPathsCa, SwmArray, ExpBpciArray) + bicas.tools.batch.autocreate_input_BPCIs___UTEST.test(... + testCase, inputDatasetsPathsCa, SwmArray, 0, ExpBpciArray) + end + + + + %================== + % Define constants + %================== + + FILE_1 = 'L1R/solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240101_V01.cdf'; + FILE_NONMATCHING = 'HK_/solo_HK_rpw-bia_20240101_V05.cdf'; + + % SWM that matches one file. + SWMP = bicas.tools.batch.TestSwmProcessing(); + SWM_1 = bicas.swm.SoftwareMode(... + SWMP, 'CLI-TEST-1', 'Human readable purpose', ... + bicas.swm.InputDataset(... + 'cli_input', 'SOLO_L1R_RPW-LFR-SURV-CWF-E', 'IN_cdf' ... + ), ... + bicas.swm.OutputDataset(... + 'cli_output', 'SOLO_L2_RPW-LFR-SURV-CWF-E', 'OUT_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + ); + % SWM that does not match any file. + SWM_NONMATCHING = bicas.swm.SoftwareMode(... + SWMP, 'CLI-TEST-NONMATCHING', 'Human readable purpose', ... + bicas.swm.InputDataset(... + 'cli_input', 'SOLO_L1_RPW-BIA-CURRENT', 'IN_cdf' ... + ), ... + bicas.swm.OutputDataset(... + 'cli_output', 'SOLO_L2_RPW-LFR-SURV-CWF-E', 'OUT_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + ); + + EXP_BPCI = bicas.tools.batch.BicasProcessingCallInfo(... + SWM_1.cliOption, ... + bicas.tools.batch.BpciInput(... + SWM_1.inputsList.cliOptionHeaderBody, ... + SWM_1.inputsList.dsi, FILE_1), ... + bicas.tools.batch.BpciOutput(... + SWM_1.outputsList.cliOptionHeaderBody, ... + SWM_1.outputsList.dsi, ... + 'SOLO_L2_RPW-LFR-SURV-CWF-E___output_dataset.cdf')); + + + + ZERO_BPCI = bicas.tools.batch.BicasProcessingCallInfo.empty(0, 1); + + %============ + % Zero BPCIs + %============ + if 1 + test(... + {}, ... + bicas.swm.SoftwareMode.empty(0, 1), ... + ZERO_BPCI) + test(... + {}, ... + [SWM_1, SWM_NONMATCHING], ... + ZERO_BPCI) + + test(... + {FILE_NONMATCHING}, ... + [SWM_1], ... + ZERO_BPCI) + test(... + {FILE_NONMATCHING}, ... + [SWM_1, SWM_NONMATCHING], ... + ZERO_BPCI) + test(... + {FILE_1}, ... + [SWM_NONMATCHING], ... + ZERO_BPCI) + end + + %================ + % Non-zero BPCIs + %================ + test(... + {FILE_1}, ... + [SWM_1], ... + EXP_BPCI) + test(... + {FILE_1, FILE_NONMATCHING}, ... + [SWM_1, SWM_NONMATCHING], ... + EXP_BPCI) + end + + + + % Test + % (1) can find BPCIs only for datasets that overlap in time, and + % (2) that currentDatasetExtensionDays works. + function test_time_overlap_currentDatasetExtensionDays(testCase) + + % NOTE: Files DO NOT overlap in time. + FILE_1 = 'L1R/solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240101_V01.cdf'; + FILE_CURRENT = 'BIA/solo_L1_rpw-bia-current-cdag_20231201-20231231_V05.cdf'; + ZERO_BPCI = bicas.tools.batch.BicasProcessingCallInfo.empty(0, 1); + + SWMP = bicas.tools.batch.TestSwmProcessing(); + SWM_SCI_CURRENT = bicas.swm.SoftwareMode(... + SWMP, 'CLI-TEST-SCI-CURRENT', 'Human readable purpose', ... + [... + bicas.swm.InputDataset('cli_sci', 'SOLO_L1R_RPW-LFR-SURV-CWF-E', 'SCI_cdf'); ... + bicas.swm.InputDataset('cli_current', 'SOLO_L1_RPW-BIA-CURRENT', 'CUR_cdf') ... + ], ... + bicas.swm.OutputDataset(... + 'cli_output', 'SOLO_L2_RPW-LFR-SURV-CWF-E', 'OUT_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + ); + + EXP_BPCI = bicas.tools.batch.BicasProcessingCallInfo(... + SWM_SCI_CURRENT.cliOption, ... + [... + bicas.tools.batch.BpciInput(... + SWM_SCI_CURRENT.inputsList(1).cliOptionHeaderBody, ... + SWM_SCI_CURRENT.inputsList(1).dsi, ... + FILE_1); ... + bicas.tools.batch.BpciInput(... + SWM_SCI_CURRENT.inputsList(2).cliOptionHeaderBody, ... + SWM_SCI_CURRENT.inputsList(2).dsi, ... + FILE_CURRENT), ... + ], ... + bicas.tools.batch.BpciOutput(... + SWM_SCI_CURRENT.outputsList(1).cliOptionHeaderBody, ... + SWM_SCI_CURRENT.outputsList(1).dsi, ... + 'SOLO_L2_RPW-LFR-SURV-CWF-E___output_dataset.cdf')); + + % No BPCI + bicas.tools.batch.autocreate_input_BPCIs___UTEST.test(... + testCase, ... + {FILE_1; FILE_CURRENT}, ... + [SWM_SCI_CURRENT], 0, ... + ZERO_BPCI) + + % One BPCI + bicas.tools.batch.autocreate_input_BPCIs___UTEST.test(... + testCase, ... + {FILE_1; FILE_CURRENT}, ... + [SWM_SCI_CURRENT], 1, ... + EXP_BPCI) + end + + + + % Check that function only uses latest version of input datasets. + function test_latest_version(testCase) + + function test(inputDatasetsPathsCa, SwmArray, ExpBpciArray) + bicas.tools.batch.autocreate_input_BPCIs___UTEST.test(... + testCase, inputDatasetsPathsCa, SwmArray, 0, ExpBpciArray) + end + + FILE_1_V1 = 'L1R/solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240101_V01.cdf'; + FILE_1_V2 = 'L1R/solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240101_V02.cdf'; + FILE_2_V1 = 'L1R/solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240102_V01.cdf'; + FILE_3_V2 = 'L1R/solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240103_V02.cdf'; + + SWMP = bicas.tools.batch.TestSwmProcessing(); + SWM_1 = bicas.swm.SoftwareMode(... + SWMP, 'CLI-TEST-1', 'Human readable purpose', ... + bicas.swm.InputDataset(... + 'cli_input', 'SOLO_L1R_RPW-LFR-SURV-CWF-E', 'IN_cdf' ... + ), ... + bicas.swm.OutputDataset(... + 'cli_output', 'SOLO_L2_RPW-LFR-SURV-CWF-E', 'OUT_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + ); + + function ExpBpci = exp_BPCI(outputPath) + ExpBpci = bicas.tools.batch.BicasProcessingCallInfo(... + SWM_1.cliOption, ... + bicas.tools.batch.BpciInput(... + SWM_1.inputsList.cliOptionHeaderBody, ... + SWM_1.inputsList.dsi, ... + outputPath), ... + bicas.tools.batch.BpciOutput(... + SWM_1.outputsList.cliOptionHeaderBody, ... + SWM_1.outputsList.dsi, ... + 'SOLO_L2_RPW-LFR-SURV-CWF-E___output_dataset.cdf')); + end + + % NOTE: ORDER IS NOT GUARANTEED BUT TEST ASSUMES ORDER FOR + % SIMPLICITY. + test(... + {FILE_1_V1, FILE_1_V2, FILE_2_V1, FILE_3_V2}, ... + SWM_1, ... + [exp_BPCI(FILE_1_V2); ... + exp_BPCI(FILE_2_V1); ... + exp_BPCI(FILE_3_V2)]... + ) + + end + + + + end % methods(Test) + + + + %######################## + %######################## + % PRIVATE STATIC METHODS + %######################## + %######################## + methods(Static, Access=private) + + + + function test(testCase, inputDatasetsPathsCa, SwmArray, currentDatasetExtensionDays, ExpBpciArray) + + function filename = get_BPCI_output_path(... + outputDsi, BpciInputDsmdArray) + + filename = sprintf('%s___output_dataset.cdf', ... + outputDsi); + end + + + + InputDsmdArray = solo.adm.paths_to_DSMD_array(inputDatasetsPathsCa(:)); + + % Assert that all paths could be converted to DSMDs (e.g. that + % filenames conformed to filenaming conventions). + assert(numel(InputDsmdArray) == numel(inputDatasetsPathsCa)) + + % CALL TESTED FUNCTION + ActBpciArray = bicas.tools.batch.autocreate_input_BPCIs(... + InputDsmdArray, @get_BPCI_output_path, ... + SwmArray, currentDatasetExtensionDays); + + testCase.assertEqual(ActBpciArray, ExpBpciArray) + end + + + + end % methods(Static, Access=private) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_many_BPCIs.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_many_BPCIs.m new file mode 100644 index 000000000..af4739372 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_many_BPCIs.m @@ -0,0 +1,45 @@ +% +% Find all possible BPCIs for a given set of DSMDs and SWMs. +% +% NOTE: One can limit the number of SWMs by trimming SwmArray. +% NOTE: Requires DSMDs to not overlap in time for each DSI separately. +% Therefore typically wants to only supply the latest versions of +% datasets. +% Can therefore not handle SOLO_L2_RPW-LFR-SBM1-CWF-E. +% +% +% ARGUMENTS +% ========= +% See bicas.tools.batch.autocreate_one_SWM_BPCI(). +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% First created 2020-01-23. +% +function BpciArray = autocreate_many_BPCIs(... + DsmdArray, SwmArray, createOutputPathFh) + + % ASSERTIONS + solo.adm.assert_no_time_overlap(DsmdArray) + assert(isa(SwmArray, 'bicas.swm.SoftwareMode')) + + BpciArray = bicas.tools.batch.BicasProcessingCallInfo.empty(0,1); + for iSwm = 1:numel(SwmArray) + + Swm = SwmArray(iSwm); + dsiList = {Swm.inputsList.dsi}; + dsmdGroupsCa = solo.adm.find_overlapping_DSMD_groups(DsmdArray, dsiList); + + nGroups = numel(dsmdGroupsCa); + for iGrp = 1:nGroups + % t2 = tic(); + Bpci = bicas.tools.batch.autocreate_one_SWM_BPCI(... + Swm, ... + dsmdGroupsCa{iGrp}, ... + createOutputPathFh); + % fprintf('SPEED: autocreate_many_BPCIs(): autocreate_one_SWM_BPCI(): %.1f [s] wall time\n', toc(t2)) + + BpciArray = [BpciArray; Bpci]; + end + end +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_one_SWM_BPCI.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_one_SWM_BPCI.m new file mode 100644 index 000000000..c9bf93e24 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/autocreate_one_SWM_BPCI.m @@ -0,0 +1,97 @@ +% +% Autocreate ONE BPCI from DSMDs, including default output filenames while only +% considering one SWM. +% +% +% NOTE: Not obvious how to derive output file time range automatically. +% ** If one uses the UNION of ALL input dataset time intervals, then the +% interval becomes very large when current datasets are used. +% ** If one uses the INTERSECTION of ALL input dataset time intervals, then +% the interval can become negative if the CURRENT DSMD was derived using +% zVar Epoch and begins after the other datasets. +% +% +% ARGUMENTS +% ========= +% Swm +% bicas.swm.SoftwareMode object. +% DsmdArray +% DSMD array. Must contain exactly one DSI for each input DSI required by +% the SWM. +% createOutputPathFh +% Function for obtaining the path to one output dataset. +% path = func(outputDsi, InputDsmdArray) +% Arguments describe the SWM inputs (all) & output (one). +% NOTE: bicas.tools.batch.default_get_BPCI_output_filename() is +% meant to be used for helping to construct such functions by default. +% That function does however only construct the filename (not specify the +% parent directory). +% +% +% RETURN VALUE +% ============ +% Bpci +% bicas.tools.batch.BicasProcessingCallInfo object. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% First created 2020-05-27. +% +function Bpci = autocreate_one_SWM_BPCI(Swm, DsmdArray, createOutputPathFh) + + % ASSERTIONS + assert(isa(Swm, 'bicas.swm.SoftwareMode') && isscalar(Swm)) + irf.assert.castring_set({Swm.inputsList.dsi }) + irf.assert.castring_set({Swm.outputsList.dsi}) + assert(isa(createOutputPathFh, 'function_handle')) + assert(numel(Swm.inputsList) == numel(DsmdArray)) + + % t = tic(); + + % cohbCa = cell(0, 1); + inputsArray = bicas.tools.batch.BpciInput.empty(0, 1); + for i = 1:numel(Swm.inputsList) + % Iterate over inputs + % =================== + % NOTE: Better to iterate over SWM inputs than DSMDs, since + % (1) Easier to permit irrelevant DSMDs (not implemented) + % (2) Check that there is exactly one matching DSMD (by DSI). + + dsiToSearchFor = Swm.inputsList(i).dsi; + iDsmd = find(strcmp(dsiToSearchFor, {DsmdArray.datasetId})); + + % ASSERTIONS + if numel(iDsmd) == 0 + error('Can not find any dataset with DSI="%s".', dsiToSearchFor) + elseif numel(iDsmd) >= 2 + error('Found more than one dataset with DSI="%s".', dsiToSearchFor) + end + + inputsArray(i, 1) = bicas.tools.batch.BpciInput(... + Swm.inputsList(i).cliOptionHeaderBody, ... + Swm.inputsList(i).dsi, ... + DsmdArray(iDsmd).path); + + % Prepare for calling output filenaming function. + % cohbCa{i} = Swm.inputsList(iDsmd).cliOptionHeaderBody; + iDsmdArray(i) = iDsmd; + end + + % Re-arrange order so that it is equal to that of cohbCa. + % DsmdArray = DsmdArray(iDsmdArray); + + %fprintf('SPEED: autocreate_one_SWM_BPCI(): t=%.1f [s]\n', toc(t)); + + outputsArray = bicas.tools.batch.BpciOutput.empty(0, 1); + for i = 1:numel(Swm.outputsList) + outputsArray(i, 1) = bicas.tools.batch.BpciOutput(... + Swm.outputsList(i).cliOptionHeaderBody, ... + Swm.outputsList(i).dsi, ... + createOutputPathFh(... + Swm.outputsList(i).dsi, DsmdArray)... + ); + end + %fprintf('SPEED: autocreate_one_SWM_BPCI(): t=%.1f [s]\n', toc(t)); + + Bpci = bicas.tools.batch.BicasProcessingCallInfo(Swm.cliOption, inputsArray, outputsArray); +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/default_get_BPCI_output_filename.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/default_get_BPCI_output_filename.m new file mode 100644 index 000000000..46c60950d --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/default_get_BPCI_output_filename.m @@ -0,0 +1,131 @@ +% +% Default algorithm for selecting output dataset filename for BPCIs. Meant to be +% used as an argument to bicas.tools.batch.autocreate_one_SWM_BPCI(), +% unless the caller wants to use their own customized function. +% +% NOTE: Not obvious whether to use CDAG, which version. +% NOTE: A wrapper function can set settings by adding them to the end of the +% argument list. Migt not be able to do this using an anonymous function though. +% +% +% ARGUMENTS +% ========= +% BpciInputDsmdArray +% Column array of solo.adm.DSMD. DSMDs for BPCI input datasets. These are +% used for setting the time the output dataset filename. +% varargin +% Arguments as interpreted by +% irf.utils.interpret_settings_args(). +% -- +% Determined by argument "createOutputFilenameFunc" to function +% bicas.tools.batch.autocreate_one_SWM_BPCI(). +% +% +% RETURN VALUES +% ============= +% filename +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% First created 2020-06-15. +% +function filename = default_get_BPCI_output_filename(... + outputDsi, BpciInputDsmdArray, ... + versionStr, varargin) + + % PROPOSAL: Automatic test code. + % + % PROPOSAL: INPUT_DSI_FOR_OUTPUT_TIME --> argument + % PROPOSAL: Only use first matching DSI in INPUT_DSI_FOR_OUTPUT_TIME. + % PRO: More general. Less constraint on SWMs. + % + % PROPOSAL: Add argument for parent directory. + % CON: Setting parent directory is a different task. Different from + % defining filenaming conventions. + % PRO: Common use case. + + % List of DSIs. Only matching input datasets are used for deriving + % time range in output filename. May contain unused DSIs. + INPUT_DSI_FOR_OUTPUT_TIME = { ... + 'SOLO_L1_RPW-LFR-SBM1-CWF', ... + 'SOLO_L1_RPW-LFR-SBM2-CWF', ... + 'SOLO_L1_RPW-LFR-SURV-CWF', ... + 'SOLO_L1_RPW-LFR-SURV-SWF', ... + 'SOLO_L1_RPW-TDS-LFM-CWF', ... + 'SOLO_L1_RPW-TDS-LFM-RSWF', ... + ... + 'SOLO_L1R_RPW-LFR-SBM1-CWF-E', ... + 'SOLO_L1R_RPW-LFR-SBM2-CWF-E', ... + 'SOLO_L1R_RPW-LFR-SURV-CWF-E', ... + 'SOLO_L1R_RPW-LFR-SURV-SWF-E', ... + 'SOLO_L1R_RPW-TDS-LFM-CWF-E', ... + 'SOLO_L1R_RPW-TDS-LFM-RSWF-E', ... + ... + 'SOLO_L2_RPW-LFR-SURV-CWF-E'}; + + DEFAULT_SETTINGS.unoffBasenameExtension = []; % No extension. + DEFAULT_SETTINGS.isCdagPolicy = false; % true/false/ + + Settings = irf.utils.interpret_settings_args(DEFAULT_SETTINGS, varargin); + irf.assert.struct(Settings, fieldnames(Settings), {}) + + + + % ASSERTIONS + assert(ischar(outputDsi)) + assert(isa(BpciInputDsmdArray, 'solo.adm.DSMD')) + %assert(iscell(cohbCa)) + assert(ischar(versionStr)) + %assert(numel(BpciInputDsmdArray) == numel(cohbCa)) + assert(islogical(Settings.isCdagPolicy) || isnumeric(Settings.isCdagPolicy), ... + 'Illegal Settings.isCdagPolicy.') + + + + % Identify exactly one BPCI INPUT DSMD which shall be used for determining + % time interval for OUTPUT dataset. + [~, iDsmd] = intersect({BpciInputDsmdArray.datasetId}, INPUT_DSI_FOR_OUTPUT_TIME); + assert(isscalar(iDsmd), ... + 'Can not determine exactly one input DSI to use for determining output filename time interval.') + InputDsmd = BpciInputDsmdArray(iDsmd); + + + + %========================================== + % Set variables in output dataset filename + %========================================== + R = struct(); + R.isCdag = logical(Settings.isCdagPolicy); + R.datasetId = outputDsi; + R.versionStr = versionStr; + R.dsicdagCase = 'lower'; + R.unoffExtension = Settings.unoffBasenameExtension; + + % Set date vector(s), depending on time range, effectively selecting filename + % format for the dataset. + dt1 = InputDsmd.dt1; + dt2 = InputDsmd.dt2; + if dt2 <= (dt1 + caldays(1)) + % CASE: (dt1,dt2) covers less than or equal to a calendar day. + % ==> Use filenaming format yymmdd (no begin-end; just the calendar day). + + % IMPLEMENTATION NOTE: Using center of day so that can handle DSMD time + % boundaries that are slightly on the wrong side of midnight, e.g. if + % deriving DSMD from file content. + % Ex: LFR-SURV-CWF test file + % solo_L1R_rpw-lfr-surv-cwf-e-cdag_20200213_V07.cdf begins + % at Epoch = 2020-02-12T23:59:53.305345024 + dtMiddle = dt1 + (dt2-dt1)/2; + R.dateVec = [dtMiddle.Year, dtMiddle.Month, dtMiddle.Day]; + else + % CASE: (dt1,dt2) covers more than one day. + % ==> use filenaming format yymmddThhmmss-yymmddThhmmss. + R.dateVec1 = datevec(dt1); + R.dateVec2 = datevec(dt2); + end + + %================================ + % Create output dataset filename + %================================ + filename = solo.adm.create_dataset_filename(R); +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run.m new file mode 100644 index 000000000..8afe79f2a --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run.m @@ -0,0 +1,54 @@ +% +% Keep only those BPCIs which should actually be run. Always keep +% BPCI if any output file is missing in both reference directory (unless empty) +% and tpdFilenameCa. +% +% NOTE: Uses exact output filenames in argument to determine which BPCIs should +% not be run. Therefore output filenames have to match for +% (1) CDAG/non-CDAG, +% (2) version number +% (3) unofficial basename suffix, if used. +% +% +% ARGUMENTS +% ========= +% doNotNeedToGenerateFilenamesCa +% Cell array of dataset filenames for datasets. +% +% +% RETURN VALUE +% ============ +% BpciArray +% Input argument BpciArray but only that subset of elements where at least +% one of the output datasets has the same filename as on in +% generateIfMissingFilenamesCa. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function BpciArray = filter_BPCIs_to_run(BpciArray, doNotNeedToGenerateFilenamesCa) + % PROPOSAL: Better name. + % Something more generic with filtering. + % filter, keep, remove + % output datasets + % BPCI + + % ASSERTIONS + assert(isa(BpciArray, 'bicas.tools.batch.BicasProcessingCallInfo')) + assert(iscolumn(BpciArray)) + assert(iscell(doNotNeedToGenerateFilenamesCa)) + + %============================================================ + % Filter BpciArray: + % Only keep BPCIs for which at least one datasets is missing + %============================================================ + bKeep = false(size(BpciArray)); + for iBpci = 1:numel(BpciArray) + outputFilenameCa = BpciArray(iBpci).get_output_filenames(); + + % Keep BPCI if at least one of its output datasets is missing. + bKeep(iBpci) = ~all(ismember(outputFilenameCa, doNotNeedToGenerateFilenamesCa)); + end + + BpciArray = BpciArray(bKeep, 1); +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run___UTEST.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run___UTEST.m new file mode 100644 index 000000000..68a583193 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/filter_BPCIs_to_run___UTEST.m @@ -0,0 +1,139 @@ +% +% matlab.unittest automatic test code for +% bicas.tools.batch.filter_BPCIs_to_run(). +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef filter_BPCIs_to_run___UTEST < matlab.unittest.TestCase + + + + %############## + %############## + % TEST METHODS + %############## + %############## + methods(Test) + + + + function test0(testCase) + + function test(BpciArray, doNotNeedToGenerateFilenamesCa, ExpBpciArray) + % CALL TESTED FUNCTION + ActBpciArray = bicas.tools.batch.filter_BPCIs_to_run(... + BpciArray, doNotNeedToGenerateFilenamesCa); + + testCase.assertEqual(... + ActBpciArray, ... + ExpBpciArray) + end + + % ================================================================== + + ZERO_BPCI = bicas.tools.batch.BicasProcessingCallInfo.empty(0, 1); + + test(ZERO_BPCI, {}, ZERO_BPCI); + + INPUT_FILE_1 = 'input_dataset_1.cdf'; + OUTPUT_FILE_1 = 'output_dataset_1.cdf'; + + INPUT_FILE_2 = 'input_dataset_2.cdf'; + OUTPUT_FILE_2a = 'output_dataset_2a.cdf'; + OUTPUT_FILE_2b = 'output_dataset_2b.cdf'; + + BPCI_1 = bicas.tools.batch.BicasProcessingCallInfo(... + 'CLI_SWM_1', ... + bicas.tools.batch.BpciInput(... + 'cli_input', 'DSI_1', INPUT_FILE_1), ... + bicas.tools.batch.BpciOutput(... + 'cli_output', 'DSI_2', OUTPUT_FILE_1)); + + BPCI_2 = bicas.tools.batch.BicasProcessingCallInfo(... + 'CLI_SWM_2', ... + bicas.tools.batch.BpciInput(... + 'cli_input', 'DSI_1', INPUT_FILE_2), ... + [... + bicas.tools.batch.BpciOutput(... + 'cli_output', 'DSI_2', OUTPUT_FILE_2a); ... + bicas.tools.batch.BpciOutput(... + 'cli_output', 'DSI_3', OUTPUT_FILE_2b) ... + ]... + ); + + %================== + % No matching file + %================== + if 1 + test(... + BPCI_1, ... + {}, ... + BPCI_1) + test(... + [BPCI_1; BPCI_2], ... + {}, ... + [BPCI_1; BPCI_2]) + test(... + BPCI_1, ... + {INPUT_FILE_1}, ... % Not matching + BPCI_1) + test(... + [BPCI_1; BPCI_2], ... + {INPUT_FILE_2}, ... % Not matching + [BPCI_1; BPCI_2]) + end + + %========== + % One BPCI + %========== + test(... + BPCI_1, ... + {OUTPUT_FILE_1}, ... + ZERO_BPCI) + test(... + BPCI_2, ... + {OUTPUT_FILE_2a}, ... + BPCI_2) + test(... + BPCI_2, ... + {OUTPUT_FILE_2b}, ... + BPCI_2) + test(... + BPCI_2, ... + {OUTPUT_FILE_2b, OUTPUT_FILE_2a}, ... + ZERO_BPCI) + + %=========== + % Two BPCIs + %=========== + test(... + [BPCI_1; BPCI_2], ... + {}, ... + [BPCI_1; BPCI_2]) + test(... + [BPCI_1; BPCI_2], ... + {OUTPUT_FILE_2b, OUTPUT_FILE_2a}, ... + BPCI_1) + test(... + [BPCI_1; BPCI_2], ... + {OUTPUT_FILE_1}, ... + BPCI_2) + test(... + [BPCI_1; BPCI_2], ... + {OUTPUT_FILE_1, OUTPUT_FILE_2b, OUTPUT_FILE_2a}, ... + ZERO_BPCI) + + test(... + [BPCI_1; BPCI_2], ... + {OUTPUT_FILE_1, OUTPUT_FILE_2a}, ... + [BPCI_2]) + end + + + + end % methods(Test) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2.m new file mode 100644 index 000000000..efdbb9a4e --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2.m @@ -0,0 +1,212 @@ +% +% YK 2020-10-15: Do not use any unofficial basename extension for the IRF +% pipeline. This is necessary for irfu-matlab's automatic zVar & dataset +% finding. +% +% +% ARGUMENTS +% ========= +% BpciInputDsmdArray +% DSMD column array for INPUT datasets for the (one) relevant BPCI. +% PreexistingOutputLvDsmdArray +% DSMD column array containing only the latest versions (LV) +% of preexisting datasets which shall be used for determining output +% dataset version number. +% fnVerAlgorithm +% String constant. Represents selected filename (FN) version algorithm. +% +% +% RETURN VALUES +% ============= +% filePath +% Path to output file, including filename. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function filePath = get_BPCI_output_path2(... + BpciInputDsmdArray, PreexistingOutputLvDsmdArray, ... + outputDsi, fnVerAlgorithm, outputDir, outputIsCdag) + + % PROPOSAL: Test for day with leap second. + % + % PROPOSAL: Rename PreexistingOutputLvDsmdArray + % dataset version + % consideration + % preexisting + % basis (for) + % competitors + % + % PROPOSAL: Not set output directory. The caller should do that. + % + % PROBLEM: SBM1/2 has L1R datasets on a different filenaming (time) format + % Ex: + % solo_L1R_rpw-lfr-sbm1-cwf-e-cdag_20240201T025448-20240201T030848_V02.cdf + % ==> + % solo_L2_rpw-lfr-sbm1-cwf-e_20240201_V01.cdf (incorrect) + % solo_L2_rpw-lfr-sbm1-cwf-e_20240201T025448-20240201T030848_V01.cdf (correct) + % PROBLEM: DSMDs do not store the filenaming format. + % PROPOSAL: Separate list of input DSIs which should yield this format. + % PROPOSAL: Use length of time interval. -- IMPLEMENTED + % + % PROPOSAL: Separate function for converting a selected reference input + % dataset filename into output dataset filename. -- IMPLEMENTED + + assert(isa(BpciInputDsmdArray, 'solo.adm.DSMD') && iscolumn(BpciInputDsmdArray)) + assert(isa(PreexistingOutputLvDsmdArray, 'solo.adm.DSMD') && iscolumn(PreexistingOutputLvDsmdArray)) + assert(ischar(outputDsi)) + assert(ischar(fnVerAlgorithm)) + assert(islogical(outputIsCdag)) + + INPUT_DSI_FOR_OUTPUT_TIME = { ... + 'SOLO_L1_RPW-LFR-SBM1-CWF', ... + 'SOLO_L1_RPW-LFR-SBM2-CWF', ... + 'SOLO_L1_RPW-LFR-SURV-CWF', ... + 'SOLO_L1_RPW-LFR-SURV-SWF', ... + 'SOLO_L1_RPW-TDS-LFM-CWF', ... + 'SOLO_L1_RPW-TDS-LFM-RSWF', ... + ... + 'SOLO_L1R_RPW-LFR-SBM1-CWF-E', ... + 'SOLO_L1R_RPW-LFR-SBM2-CWF-E', ... + 'SOLO_L1R_RPW-LFR-SURV-CWF-E', ... + 'SOLO_L1R_RPW-LFR-SURV-SWF-E', ... + 'SOLO_L1R_RPW-TDS-LFM-CWF-E', ... + 'SOLO_L1R_RPW-TDS-LFM-RSWF-E', ... + ... + 'SOLO_L2_RPW-LFR-SURV-CWF-E'}; + + + + % Identify exactly one BPCI INPUT DSMD which shall be used for determining + % time interval for OUTPUT dataset. + [~, iRefDsmd] = intersect({BpciInputDsmdArray.datasetId}, INPUT_DSI_FOR_OUTPUT_TIME); + assert(isscalar(iRefDsmd), ... + 'Can not determine exactly one input DSI to use for determining output filename time interval.') + InputRefDsmd = BpciInputDsmdArray(iRefDsmd); + + dt1 = InputRefDsmd.dt1; + dt2 = InputRefDsmd.dt2; + + versionNbr = get_output_version(... + dt1, dt2, ... + outputDsi, fnVerAlgorithm, PreexistingOutputLvDsmdArray); + + fileName = get_BPCI_output_filename2(... + dt1, dt2, ... + outputDsi, outputIsCdag, versionNbr); + + filePath = fullfile(outputDir, fileName); +end + + + +function versionNbr = get_output_version(... + dt1, dt2, outputDsi, fnVerAlgorithm, PreexistingOutputLvDsmdArray) + + %================================================================= + % Identify highest version number of pre-existing output datasets + %================================================================= + PreexistingOutputLvDsmdArray = solo.adm.filter_DSMD_DATASET_ID(... + PreexistingOutputLvDsmdArray, {outputDsi}); + + if ~isempty(PreexistingOutputLvDsmdArray) + % NOTE: Command only works for non-empty array. + bKeep1 = ([PreexistingOutputLvDsmdArray.dt1] == dt1); + bKeep2 = ([PreexistingOutputLvDsmdArray.dt2] == dt2); + bKeep = bKeep1 & bKeep2; + else + bKeep = zeros(0, 1); + end + PreexistingOutputLvDsmd = PreexistingOutputLvDsmdArray(bKeep); % Empty or scalar. + + if isscalar(PreexistingOutputLvDsmd) + versionNbr = PreexistingOutputLvDsmd.versionNbr; + elseif isempty(PreexistingOutputLvDsmd) + versionNbr = NaN; + else + error('Could not find zero or one matching DSMD in PreexistingOutputLvDsmdArray.') + end + + %============================================ + % Determine version number of output dataset + %============================================ + switch(fnVerAlgorithm) + + case 'ABOVE_HIGHEST_USED' + if isnan(versionNbr) + versionNbr = 1; + else + versionNbr = versionNbr + 1; + end + + case 'HIGHEST_USED' + if isnan(versionNbr) + versionNbr = 1; + else + versionNbr = versionNbr; % Nonsense command for clarity. + end + + otherwise + error('Illegal fnVerAlgorithm="%s".', fnVerAlgorithm) + end +end + + + +function outputFileName = get_BPCI_output_filename2(... + dt1, dt2, outputDsi, outputIsCdag, versionNbr) + + UNOFF_BASENAME_EXTENSION = []; + + %========================================== + % Set variables in output dataset filename + %========================================== + R = []; + R.datasetId = outputDsi; + R.versionStr = sprintf('%02i', versionNbr); + R.isCdag = outputIsCdag; + R.dsicdagCase = 'lower'; + R.unoffExtension = UNOFF_BASENAME_EXTENSION; + + % Set date vector(s), depending on time range, effectively selecting filename + % format for the dataset. + + % if dt2 <= (dt1 + caldays(1)) + if is_midnight(dt1) && is_midnight(dt2) && (dt2 == dt1 + caldays(1)) + % CASE: (dt1,dt2) covers exactly one calender day. + % ==> Use filenaming format yymmdd (no begin-end; just the calendar day). + % + % NOTE/BUG: This is technically somewhat imperfect since input dataset + % would + % solo_L1R_rpw-lfr-sbm1-cwf-e-cdag_20240201T000000-20240202T000000_V02.cdf + % would yield an output dataset filename on the YYYYMMDD format. This + % should be unlikely. + + % IMPLEMENTATION NOTE: Using center of day so that can handle DSMD time + % boundaries that are slightly on the wrong side of midnight, e.g. if + % deriving DSMD from file content. + % Ex: LFR-SURV-CWF test file + % solo_L1R_rpw-lfr-surv-cwf-e-cdag_20200213_V07.cdf begins + % at Epoch = 2020-02-12T23:59:53.305345024 + dtMiddle = dt1 + (dt2-dt1)/2; + R.dateVec = [dtMiddle.Year, dtMiddle.Month, dtMiddle.Day]; + else + % CASE: (dt1,dt2) does not exactly cover one day one day. + % ==> use filenaming format yymmddThhmmss-yymmddThhmmss. + R.dateVec1 = datevec(dt1); + R.dateVec2 = datevec(dt2); + end + + + + %================================ + % Create output dataset filename + %================================ + outputFileName = solo.adm.create_dataset_filename(R); +end + + + +function isMidnight = is_midnight(dt) + isMidnight = dateshift(dt, 'start', 'day') == dt; +end \ No newline at end of file diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2___UTEST.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2___UTEST.m new file mode 100644 index 000000000..20d767d12 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_BPCI_output_path2___UTEST.m @@ -0,0 +1,180 @@ +% +% matlab.unittest automatic test code for +% bicas.tools.batch.get_BPCI_output_path2(). +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef get_BPCI_output_path2___UTEST < matlab.unittest.TestCase + + + + %################# + %################# + % TEST PARAMETERS + %################# + %################# + properties(TestParameter) + FILENAME_VERSION_ALGORITHM = {'HIGHEST_USED', 'ABOVE_HIGHEST_USED'} + + % YMD = Year-Month-Day + BPCI_INPUT_YMD_FILES = { + { + 'solo_L1R_rpw-lfr-surv-cwf-e_20220101_V03.cdf', ... % No CDAG + }, ... + { ... + 'solo_L1R_rpw-lfr-surv-cwf-e-cdag_20220101_V03.cdf' ... % CDAG + }, ... + { + % Different time intervals. + 'solo_L1_rpw-bia-current-cdag_20220101-20220131_V15.cdf', ... + 'solo_L1R_rpw-lfr-surv-cwf-e_20220101_V03.cdf', ... % No CDAG + }, ... + } + + % YMDHMS = Year-Month-Day-Hour-Minute-Second + % No CDAG + BPCI_INPUT_YMDHMS_FILES = { + { + 'solo_L1R_rpw-lfr-sbm1-cwf-e_20220101T012345-20220101T123456_V02.cdf', ... + }, ... + { + % Different time intervals. + 'solo_L1_rpw-bia-current_20220101-20220131_V15.cdf', ... + 'solo_L1R_rpw-lfr-sbm1-cwf-e_20220101T012345-20220101T123456_V02.cdf', ... + }, ... + } + end + + + + %############## + %############## + % TEST METHODS + %############## + %############## + methods(Test) + + + + function test_YMD_CDAG_output_dir_no_preexisting(testCase, FILENAME_VERSION_ALGORITHM, BPCI_INPUT_YMD_FILES) + % CDAG output filename + % Non-empty-string output directory. + % No pre-existing datasets. + bicas.tools.batch.get_BPCI_output_path2___UTEST.test(... + testCase, ... + BPCI_INPUT_YMD_FILES, ... + {... + }, ... + 'SOLO_L2_RPW-LFR-SURV-CWF-E', FILENAME_VERSION_ALGORITHM, 'out', true, ... + fullfile('out', 'solo_L2_rpw-lfr-surv-cwf-e-cdag_20220101_V01.cdf') ... + ) + end + + + + function test_YMD_preexisting_output(testCase, BPCI_INPUT_YMD_FILES, FILENAME_VERSION_ALGORITHM) + + switch(FILENAME_VERSION_ALGORITHM) + case 'HIGHEST_USED' + expOutputFilename = 'solo_L2_rpw-lfr-surv-cwf-e_20220101_V04.cdf'; + case 'ABOVE_HIGHEST_USED' + expOutputFilename = 'solo_L2_rpw-lfr-surv-cwf-e_20220101_V05.cdf'; + end + + bicas.tools.batch.get_BPCI_output_path2___UTEST.test(... + testCase, ... + BPCI_INPUT_YMD_FILES, ... + {... + fullfile('ref', 'solo_L2_rpw-lfr-surv-cwf-e-cdag_20211231_V01.cdf'), ... % Irrelevant adjacent date. + fullfile('ref', 'solo_L2_rpw-lfr-surv-cwf-e-cdag_20220101_V04.cdf'), ... + fullfile('ref', 'solo_L2_rpw-lfr-surv-cwf-e-cdag_20220102_V02.cdf'), ... % Irrelevant adjacent date. + }, ... + 'SOLO_L2_RPW-LFR-SURV-CWF-E', FILENAME_VERSION_ALGORITHM, '', false, ... + expOutputFilename ... + ) + end + + + + function test_YMDHMS_output_dir_no_preexisting(testCase, FILENAME_VERSION_ALGORITHM, BPCI_INPUT_YMDHMS_FILES) + % CDAG output filename + % Non-empty-string output directory. + % No pre-existing datasets. + bicas.tools.batch.get_BPCI_output_path2___UTEST.test(... + testCase, ... + BPCI_INPUT_YMDHMS_FILES, ... + {... + }, ... + 'SOLO_L2_RPW-LFR-SBM1-CWF-E', FILENAME_VERSION_ALGORITHM, ... + 'out', true, ... + fullfile('out', 'solo_L2_rpw-lfr-sbm1-cwf-e-cdag_20220101T012345-20220101T123456_V01.cdf') ... + ) + end + + + + function test_YMDHMS_preexisting_output(testCase, BPCI_INPUT_YMDHMS_FILES, FILENAME_VERSION_ALGORITHM) + + switch(FILENAME_VERSION_ALGORITHM) + case 'HIGHEST_USED' + expOutputFilename = 'solo_L2_rpw-lfr-sbm1-cwf-e_20220101T012345-20220101T123456_V04.cdf'; + case 'ABOVE_HIGHEST_USED' + expOutputFilename = 'solo_L2_rpw-lfr-sbm1-cwf-e_20220101T012345-20220101T123456_V05.cdf'; + end + + bicas.tools.batch.get_BPCI_output_path2___UTEST.test(... + testCase, ... + BPCI_INPUT_YMDHMS_FILES, ... + {... + 'solo_L2_rpw-lfr-sbm1-cwf-e_20220101_V01.cdf', ... % Irrelevant time interval. Unexpected time format. + 'solo_L2_rpw-lfr-sbm1-cwf-e_20220101T012345-20220101T123450_V01.cdf', ... % Different but similar time interval. + 'solo_L2_rpw-lfr-sbm1-cwf-e_20220101T012345-20220101T123456_V04.cdf', ... + 'solo_L2_rpw-lfr-sbm1-cwf-e_20220101T012340-20220101T123456_V01.cdf', ... % Different but similar time interval. + }, ... + 'SOLO_L2_RPW-LFR-SBM1-CWF-E', FILENAME_VERSION_ALGORITHM, ... + '', false, ... + expOutputFilename ... + ) + end + + + + end % methods(Test) + + + + %######################## + %######################## + % PRIVATE STATIC METHODS + %######################## + %######################## + methods(Static, Access=private) + + + + function test(... + testCase, ... + bpciInputPathCa, ... % Converted to DSMD. + preexistingOutputLvCa, ... % Converted to DSMD. + outputDsi, fnVerAlgorithm, outputDir, outputIsCdag, expFilePath) + + BpciInputDsmdArray = solo.adm.paths_to_DSMD_array(bpciInputPathCa(:)); + PreexistingOutputLvDsmdArray = solo.adm.paths_to_DSMD_array(preexistingOutputLvCa(:)); + + % CALL TESTED CODE + actFilePath = bicas.tools.batch.get_BPCI_output_path2(... + BpciInputDsmdArray, PreexistingOutputLvDsmdArray, ... + outputDsi, fnVerAlgorithm, outputDir, outputIsCdag ... + ); + + testCase.assertEqual(actFilePath, expFilePath) + end + + + + end % methods(Static, Access=private) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m new file mode 100644 index 000000000..241134697 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m @@ -0,0 +1,56 @@ +% +% Given a path to the reference directory, return DSMD array for it. +% +% +% ARGUMENTS +% ========= +% dirPathsCa +% Cell array of paths to directories. +% +% +% RETURN VALUES +% ============= +% DsmdArray +% 1D array of DSMDs to datasets under the specified directories, +% recursively. +% OiArray +% The combined result of the underlying calls to dir(), but only for the +% found datasets. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function [DsmdArray, OiArray] = get_directory_DSMDs(dirPathsCa) +% PROPOSAL: Make "generic": Move to solo.adm. + + t = tic(); + + % Create empty struct array of the right size. + % IMPLEMENTATION NOTE: Calling dir() just to make sure that the code uses + % the exact struct which it produces. This should make the code more + % future-proof. + OiArray = dir('~'); + OiArray = OiArray([], 1); % Column array. + + DsmdArray = solo.adm.DSMD.empty(0,1); + for i = 1:numel(dirPathsCa) + DirOiArray = dir(fullfile(dirPathsCa{i}, '**')); + DirOiArray = DirOiArray(~[DirOiArray.isdir]); + dirFilesPathsCa = arrayfun(@(Oi) (fullfile(Oi.folder, Oi.name)), DirOiArray, 'UniformOutput', false); + % CASE: DirOiArray is a column array. + + [DirDsmdArray, bIsDataSetArray] = solo.adm.paths_to_DSMD_array(dirFilesPathsCa); + DirOiArray = DirOiArray(bIsDataSetArray); + + OiArray = [... + OiArray; ... + DirOiArray]; + DsmdArray = [... + DsmdArray; ... + DirDsmdArray]; + end + + wallTimeSec = toc(t); + fprintf('SPEED: Wall time to obtain DSMDs for %g paths: %.1f [s]\n', ... + numel(dirPathsCa), wallTimeSec); +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs___UTEST.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs___UTEST.m new file mode 100644 index 000000000..cc075d352 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs___UTEST.m @@ -0,0 +1,96 @@ +% +% matlab.unittest automatic test code for +% bicas.tools.batch.get_directory_DSMDs(). +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef get_directory_DSMDs___UTEST < matlab.unittest.TestCase + + + + %############## + %############## + % TEST METHODS + %############## + %############## + methods(Test) + + + + function test_zero_direcctories(testCase) + [ActDsmdArray, ActOiArray] = bicas.tools.batch.get_directory_DSMDs({}); + + ExpOiArray = dir('~'); + ExpOiArray = ExpOiArray([], 1); + + testCase.assertEqual(ActDsmdArray, solo.adm.DSMD.empty(0, 1)) + testCase.assertEqual(ActOiArray, ExpOiArray) + end + + + + function test_subdirectories(testCase) + function subdirPath = create_subdir(parentDir, varargin) + % PROPOSAL: Make into generic function. + + for i = 1:numel(varargin) + mkdir(parentDir, varargin{i}) + parentDir = fullfile(parentDir, varargin{i}); + end + + subdirPath = parentDir; + end + + function test(dirPathsCa, expDatasetPathsCa) + [ActDsmdArray, ActOiArray] = bicas.tools.batch.get_directory_DSMDs(dirPathsCa); + actPathCa = {ActDsmdArray.path}'; + + testCase.assertEqual(... + sort(actPathCa(:)), ... + sort(expDatasetPathsCa(:))) + testCase.assertEqual(... + size(ActOiArray), size(ActDsmdArray)) + end + + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture) + testDir = pwd; + cd('~') % Move to any OTHER unrelated directory. + + junk = create_subdir(testDir, 'subdir1', 'subdir11'); + dir2 = create_subdir(testDir, 'subdir2', 'subdir21'); + dir3 = create_subdir(testDir, 'subdir3', 'subdir31'); + + % NOTE: No file in subdir1. + file2 = fullfile(dir2, 'solo_L1R_rpw-lfr-surv-cwf-e_20240101_V02.cdf'); + file3a = fullfile(dir3, 'solo_L1R_rpw-lfr-surv-cwf-e_20240102_V03.cdf'); + file3b = fullfile(dir3, 'solo_L1R_rpw-lfr-surv-cwf-e_20240103_V04.cdf'); + file3c = fullfile(dir3, 'NOT_DATASET.DAT'); + irf.fs.create_empty_file(file2) + irf.fs.create_empty_file(file3a) + irf.fs.create_empty_file(file3b) + irf.fs.create_empty_file(file3c) + + test(... + {testDir}, ... + {file2, file3a, file3b}) + + test(... + {fullfile(testDir, 'subdir1')}, ... + {}) + test(... + {fullfile(testDir, 'subdir2')}, ... + {file2}) + + test(... + {fullfile(testDir, 'subdir3')}, ... + {file3a, file3b}) + end + + + + end % methods(Test) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main.m new file mode 100644 index 000000000..890894e4a --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main.m @@ -0,0 +1,391 @@ +% +% Code for batch processing of datasets with BICAS. +% +% NOTE: There is a separate wrapper script for calling this function from a bash +% script. +% +% +% NOTES +% ===== +% IMPORTANT NOTE: Function intentionally continues on BICAS error (on-zero error +% code). Will then not try the same BICAS call again. Will only call BICAS if +% all input datasets exist immediately prior to the call. +% -- +% NOTE: Will try to only use latest versions of input datasets. +% NOTE: Output dataset filenames are determined by +% bicas.tools.batch.default_get_BPCI_output_filename(). +% NOTE: Will produce one log file per BICAS run. +% NOTE: Will itself not configure settings to enable extra SWMs. Use the BICAS +% config file to enable/disable SWMs as far as BICAS permits it. +% +% NOTE: Relies on BICAS code for +% (1) processing +% (2) obtaining SWM definitions (and potentially uses BICAS SETTINGS to obtain +% this not yet implemented). +% NOTE: Therefore effectively relies on an unofficial BICAS interface! +% -- +% IMPLEMENTATION NOTE: If one wants to specify +% ENV_VAR_OVERRIDE.ROC_RCS_CAL_PATH, +% ENV_VAR_OVERRIDE.ROC_RCS_MASTER_PATH, +% then one needs a system-customized BICAS config file. Therefore, the +% implementation REQUIRES the caller to specify a config file. +% -- +% NOTE: The time needed to identify DSMDs and BPCIs can be slow if applied to +% many datasets. This problem will grow as the mission collects more and more +% data. Therefore logs time consumption for these parts in anticipation of havng +% to speed this up. +% -- +% NOTE: If one uses the output directory as a source directory, then the +% datasets in the output directory will also be used as input for BICAS in +% subsequent rounds of dataset searching+processing. This makes it possible to +% process L1R-->L2-->L3 with a single call. +% -- +% YK 2020-10-15: Do not use any unofficial basename extension for the IRF +% pipeline. This is necessary for irfu-matlab's automatic zVar & dataset +% finding. +% +% +% ALGORITHM FOR WHICH DATASETS THAT WILL BE PRODUCED +% ================================================== +% DEFINITION: ONE PASS +% Let BICAS produce all datasets which can be produced from the datasets in +% the given input paths (given all existing BICAS SWMs), as long as there is +% no corresponding output dataset (any version) in the reference directory. +% Different combinations of referenceDir and modeStr therefore determine which +% datasets are produced and which versions the output datasets will have. +% NOTE: The implementation uses filename collisions between output datasets +% and reference directory datasets in a way which is not what one would expect +% from the description above, but the resulting behaviour is the same. +% -- +% Run pass after pass in a loop, until no new datasets can be produced. +% NOTE: Incrementing dataset versions are always done relative to reference +% directory. +% +% +% ALGORITHM FOR HOW TO IDENTIFY BPCIs +% =================================== +% See bicas.tools.batch.autocreate_many_BPCIs(). +% NOTE: Algorithm requires input datasets to not overlap in time for each +% DSI separately. Can therefore not handle SOLO_L2_RPW-LFR-SBM1-CWF-E +% +% +% HOW TO PROCESS: EXAMPLES +% ======================== +% PROCESS EVERYTHING (potentially overwrite datasets) +% referenceDir = [] +% modeStr = V01 (or NEW_VERSIONS) +% PROCESS EVERYTHING, INCREMENT VERSIONS +% referenceDir = directory with older datasets (can be output directory) +% modeStr = NEW_VERSION +% ONLY PROCESS NON-PREEXISTING DATASETS +% referenceDir = directory with older datasets (can be output directory) +% modeStr = V01 +% PROCESS L1/L1R+HK --> L2 --> L3 IN ONE CALL +% referenceDir = directory with older datasets (can be output directory) +% modeStr = V01 +% inputPathsCa : Includes referenceDir ## IMPORTANT +% +% +% ARGUMENTS +% ========= +% bicasConfigFile +% Path to BICAS' config file. +% outputIsCdag +% Logical. Whether output datasets should be CDAG. +% modeStr : String constant. +% 'V01' +% Only generate datasets not already represented in +% referenceDir (any version). Output datasets always have version V01. +% 'NEW_VERSION' +% Output datasets are set to having the first unused version number +% over the highest pre-existing ones among the corresponding output +% datasets in the reference directory. +% NOTE: Primarily intended to be used for L3 deliveries to ROC. +% outputDir +% Directory to create new datasets in. +% referenceDir +% Directory path. Directory used for determining which datasets have +% already been produced. Used by argument "modeStr". Will be searched +% recursively. +% -- +% SPECIAL CASE: Empty <=> Empty reference directory. +% inputPathsCa : Cell array +% Arbitrary number of paths (arguments) to individual datasets and/or +% directories. Datasets are searched for (recursively) under directories. +% Other files are ignored. The found datasets are used for finding +% combinations of datasets which can be used to generate datasets, based +% on filenaming conventions. +% varargin +% Settings as interpreted by +% irf.utils.interpret_settings_args(). +% +% +% INTERNAL NAMING CONVENTIONS +% =========================== +% BEC : BICAS Error Code +% HTPD : Have or Tried Produce Dataset(s). +% TPD : Tried Produce Dataset(s). Output dataset which the code actually tried +% to produce by calling BICAS (successfully or not), as opposed to output +% datasets that could not be produced due to removed input datasets +% detected before calling BICAS. +% -- +% NOTE: Code also uses a small number of abbreviations defined in BICAS. +% +% +% Initially created 2020-03-02 by Erik P G Johansson, IRF, Uppsala, Sweden. +% +function main(... + bicasConfigFile, outputIsCdag, modeStr, ... + outputDir, referenceDir, inputPathsCa, varargin) + % PROPOSAL: Better name + % ~bicas + % CON: Clashes with package name + % ~batch + % CON: Clashes with package name + % CON: Is sort of in line with Python's convention of main module + % in a package sharing name with the (leaf-most) package name. + % ~main + % NOTE: Cf. bicas.main (irfu-matlab) + % + % PROPOSAL: Move to irfu-matlab: ~bicas.tools.batch + % NOTE: Requires dataset filenaming code, code for identifying datasets + % that can be produced, BPCI, DSMD. + % NOTE: Some dependencies have already been moved to irfu-matlab: + % solo.adm. + % NOTE: Some dependencies are duplicated in irfu-matlab: + % TODO: Identify such dependencies. + % 2024-02-23: No such dependencies. Have been replaced with solo.adm + % counterparts, or code has been moved to bicas.batch. + % -- + % PRO: More natural to use bicas.Logger, bicas.utils.Timekeeper. + % + % PROPOSAL: Be able to optionally limit to specified SWM (several?). + % PROPOSAL: Use settings. + % CON: How handle from bash? + % TODO-DEC: How specify set of SWMs? + % PROPOSAL: Specify set of SWMs to permit (include) + % PROPOSAL: Specify set of SWMs ignore (exclude) + % TODO-DEC: Permit non-existent SWMs? + % NOTE: The SWMs which BICAS sees depends on BICAS settings. + % PROPOSAL: Not permit + % PRO: More rigorous. Less change of misspelling. + % CON: Can not write wrappers, aliases with permanent lists of + % SWMs. + % + % PROPOSAL: Filter DSIs (only keep those needed for SWMs) before sorting + % by version and before searching for BPCIs. + % PRO: Likely faster. + % + % PROPOSAL: Rename modeStr --> outputVersionModeStr + % CON: modeStr does not only determine the output version algorithm. V01 + % only generates datasets not already in reference directory + % whereas NEW_VERSION always generates new datasets. + % + % PROPOSAL: Redefine algorithm. + % NOTE: Technically, the reference directory is used for two different + % things: + % (1) Determine output dataset versions. + % (2) Determine whether to output datasets. + % NOTE: modeStr=NEW_VERSION will always always produce output + % dataset candidates which never collide with reference datasets. + % -- + % PROPOSAL: Argument/flag for whether to permit overwriting datasets. + % PROPOSAL: Not argument visible in outside interface. Could be a + % function of "mode" for at least the short term. + % TODO-DEC: Would need policy on how to handle SWMs/BPCIs which + % simultaneously output multiple files, but for which there is a + % collision only for some. + % NOTE: Always write all files or no file. + Overwrite disallowed. + % ==> Any collision implies writing no file. + % PRO: Clarity. + % PRO: Flexibility. + % CON: More to test. + % PROPOSAL: + % Generate BPCIs with output dataset versions using reference + % directory + versioning algorithm (as now). + % Run every BPCI which does not only produce datasets pre-existing in + % reference directory. + % + % NOTE/~BUG/~PROBLEM: Algorithm requires input datasets to not overlap in + % time for each unique DSI separately. Can therefore not handle + % SOLO_L2_RPW-LFR-SBM1-CWF-E. + % NOTE: Implemented via + % bicas.tools.batch.autocreate_input_BPCIs() + % calling + % bicas.tools.batch.autocreate_many_BPCIs(). + % PROPOSAL: Define (hardcoded) list of DSIs "main" input datasets. + % There is one main DSI in every SWM (one main input dataset in evey + % BPCI). Separately for every SWM, iterate over all main datasets. For + % every main dataset, find those other datasets (with input DSIs in + % the SWM) which overlap with the main dataset in time. + % -- + % PROPOSAL: Same list as INPUT_DSI_FOR_OUTPUT_TIME in functions + % bicas.tools.batch.get_BPCI_output_path2() + % bicas.tools.batch.default_get_BPCI_output_filename() + % (latter to be phased out). + % PROPOSAL: Given a set of input DSIs specified by a SWM, find any group + % of datasets with some shared overlap. + % NOTE: Datasets may be used in multiple BPCIs (for same SWM). This is + % legitimately expected for + % (1) CURRENT datasets, + % (2) non-SBM datasets in SBM SWMs. + % NOTE: Since L1R SBM1s overlap in time, there should legitimately + % be L2 SBM1s overlapping in time. + % PRO: Symmetric w.r.t. datasets. Does not need to define any "main" + % dataset for every SWM. + % CON: Having two DSIs with time overlapping datasets in the same SWM + % leads to processing four different combinations of datasets for + % producing data for the same time interval. + % Ex: Extreme but clear case: + % Datasets 1a and 1b: Almost same time interval. Same DSI_1. + % Datasets 2a and 2b: Almost same time interval. Same DSI_2. + % ==> Output dataset(s) + % 1a+2a ==> 3a + % 1a+2b ==> 3b + % 1b+2a ==> 3c + % 1b+2b ==> 3d + % If DSI_1 determines the output filename, then + % 3a and 3b will have the same filename (except version), + % 3c and 3d will have the same filename (except version). + % CON: This should never happen with real datasets. SBM1 is the + % only dataset known to be overlapping with itself. CURRENT + % datasets overlap with many other datasets because they are + % long, not because they overlap with themselves. + % CON: Unclear what the algorithm should be. + + + % IMPLEMENTATION NOTE: bicas.tools.batch.main() calls + % bicas.create_default_BSO() directly which in turn uses irfu-matlab + % code (+irf/). This happens before BICAS is called, which itself + % initializes irfu-matlab paths using the same command, but then it is too + % late. + irf('check_path') + + %========== + % Settings + %========== + DEFAULT_SETTINGS = []; + % How many days to extend the time coverage of CURRENT datasets after last + % timestamp (bias current setting). + DEFAULT_SETTINGS.currentDatasetExtensionDays = 0; + %------------------------------------------------------------------------ + % BICAS settings + % -------------- + % NOTE: These settings are always set in this code, thus overwriting any + % setting in config file. + % NOTE: Capitalized since that is the BICAS naming convention for + % BICAS-internal settings. + %------------------------------------------------------------------------ + %DEFAULT_SETTINGS.bicasSetting_SWM_L1_L2_ENABLED = 1; + %DEFAULT_SETTINGS.bicasSetting_SWM_L2_L3_ENABLED = 1; + % + Settings = irf.utils.interpret_settings_args(DEFAULT_SETTINGS, varargin); + irf.assert.struct(Settings, fieldnames(DEFAULT_SETTINGS), {}) + clear DEFAULT_SETTINGS + assert(isnumeric(Settings.currentDatasetExtensionDays)) + + + + %====================================== + % ASSERTIONS: Arguments (non-settings) + %====================================== + % Useful to check existence of config file first since a faulty path will + % oterwise be found first much later. + irf.assert.file_exists(bicasConfigFile) + assert(isscalar(outputIsCdag) & islogical(outputIsCdag)) + irf.assert.dir_exists(outputDir) + assert(iscell(inputPathsCa)) + + + + SwmArray = get_SWMs(bicasConfigFile); + + bicasSettingsArgsCa = {}; +% bicasSettingsArgsCa(end+1:end+3) = {'--set', 'SWM.L1-L2_ENABLED', sprintf('%i', Settings.bicasSetting_SWM_L1_L2_ENABLED)}; +% bicasSettingsArgsCa(end+1:end+3) = {'--set', 'SWM.L2-L3_ENABLED', sprintf('%i', Settings.bicasSetting_SWM_L2_L3_ENABLED)}; + + + + switch(modeStr) + case 'V01' + fnVerAlgorithm = 'HIGHEST_USED'; + + % IMPLEMENTATION NOTE: The filename version algorithm is more + % generic. This code only creates datasets for which there is no + % pre-existing dataset (ref.dir.). ==> All created datasets are V01, + % when using this filename version algorithm. ==> Name "V01". + + case 'NEW_VERSION' + fnVerAlgorithm = 'ABOVE_HIGHEST_USED'; + + otherwise + error('Illegal argument modeStr="%s".', modeStr) + end + + + + % 31 = Format: 2021-03-19 20:08:34 + fprintf('%s: Starting BICAS passes.\n', datestr(now, 31)) + t = tic(); + + %==================== + % CALL MAIN FUNCTION + %==================== + Bpa = bicas.tools.batch.BicasProcessingAccessImpl(); + + BpcsArray = bicas.tools.batch.run_BICAS_all_passes(... + Bpa, bicasSettingsArgsCa, ... + bicasConfigFile, outputDir, referenceDir, inputPathsCa, ... + fnVerAlgorithm, outputIsCdag, SwmArray, Settings); + + %======================= + % Log BICAS error codes + %======================= + becArray = [BpcsArray.errorCode]; + nNonError = nnz(becArray == 0); + nError = nnz(becArray ~= 0); + fprintf('#BICAS calls, no error: %i\n', nNonError); + fprintf(' error: %i\n', nError); + + %================ + % Log time usage + %================ + nTpd = sum(arrayfun(@(Bpcs) numel(Bpcs.Bpci.outputsArray), BpcsArray)); + wallTimeSec = toc(t); + fprintf('%s: Finished BICAS passes.\n', datestr(now, 31)); + fprintf('SPEED: Wall time: %.3f [h] = %.0f [s]\n', wallTimeSec / 3600, wallTimeSec); + fprintf('SPEED: %.2f [s/TPD]\n', wallTimeSec / nTpd); + + + + if nError > 0 + error('%i of %i BICAS calls returned error.', ... + nError, numel(becArray)) + end +end + + + +function SwmArray = get_SWMs(bicasConfigFile) + BSO = bicas.create_default_BSO(); + + % ID string used to inform BICAS SETTINGS of who set the setting. Only + % relevant for inspecting logs. + % NOTE: Exact string not really important. + bicasSettingsSource = mfilename('fullpath'); + +% BSO.override_value('SWM.L1-L2_ENABLED', ... +% Settings.bicasSetting_SWM_L1_L2_ENABLED, ... +% bicasSettingsSource) +% BSO.override_value('SWM.L2-L3_ENABLED', ... +% Settings.bicasSetting_SWM_L2_L3_ENABLED, ... +% bicasSettingsSource) + + bicas.override_settings_from_config_file(... + bicasConfigFile, BSO, bicas.Logger('none', false)); + + BSO.make_read_only(); + + % NOTE: Converting SWML to array of SWMs. + SwmArray = bicas.swm.get_SWML(BSO).List; +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main_bash.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main_bash.m new file mode 100644 index 000000000..3b5bf9827 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/main_bash.m @@ -0,0 +1,36 @@ +% +% Wrapper around bicas.tools.batch.main() intended to be called from bash (as +% opposed to the wrapped function). +% +% +% ARGUMENTS +% ========= +% NOTE: See implementation and bicas.tools.batch.main() for details. +% -- +% varargin : Paths to input datasets, or directories with datasets (recursive). +% input paths +% +% +% RETURN VALUES +% ============= +% (none) +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% First created 2021-03-19. +% +function main_bash(bicasConfigFile, isCdagOption, modeStr, outputDir, referenceDir, varargin) + + switch(isCdagOption) + case '-n' % N = normal/no CDAG + outputIsCdag = false; + case '-c' % C = CDAG + outputIsCdag = true; + otherwise + error('Illegal isCdagOption="%s"', isCdagOption) + end + + bicas.tools.batch.main(... + bicasConfigFile, outputIsCdag, modeStr, outputDir, ... + referenceDir, varargin) +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes.m new file mode 100644 index 000000000..b9a2accfd --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes.m @@ -0,0 +1,225 @@ +% +% Run multiple passes over input datasets, making it potentially possible to +% process the input of previous passes. +% +% NOTE: See bicas.tools.batch.main() for detailed documentation on +% the exact "algorithm" etc. +% +% +% ARGUMENTS +% ========= +% SwmArray +% SWM array. +% NOTE: Use for identifying input datasets and setting output datasets +% etc. +% +% +% RETURN VALUES +% ============= +% BpcsAllArray +% Array of BPCSs. Metadata on all calls made to BICAS, in cronological +% order. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function [BpcsAllArray] = run_BICAS_all_passes(... + Bpa, bicasSettingsArgsCa, configFile, ... + outputDir, referenceDir, ... + inputPathsCa, fnVerAlgorithm, outputIsCdag, ... + SwmArray, Settings) + + % NOTE: Algorithm documentation in bicas.tools.batch.main(). + % + % PROPOSAL: Better name. + % + % PROPOSAL: Convert iteration to function. ~run_BICAS_one_pass(). + % PRO: Can convert get_BPCI_output_path_fh to regular function. + % PRO: Size of code can be allowed to be larger. + % + % PROPOSAL: Never convert ref.dir. --> DSDMs. + % PRO: Only filenames are used. + % CON: Must filter out the datasets from other files though. + % CON: Not really since they are only used for matching against output + % dataset filenames. + % CON: Inefficient. + % + % PROPOSAL: tpdFilenamesCa should contain paths, not filenames. + % PRO: DSMDs store paths. + % CON: TPD datasets always refer to output dir. paths which need to + % compared with (for set ops. diff+union) ref. dir. paths. Those + % operations must be done on filenames anyway. + % PROPOSAL: DSMDs for TPD using only filenames. + % + % PROPOSAL: Log (print) input files, output files. + % CON: Could be many. + % CON-PROPOSAL: Optional for debugging. + % CON: Useful for logging in general. + % CON: BICAS own log messages include files in & out. + + DEBUG_ENABLED = false; +% DEBUG_ENABLED = true; + + assert(isa(Bpa, 'bicas.tools.batch.BicasProcessingAccessAbstract')) + + % List of output datasets which BICAS has TRIED to create, successfully or + % not. If BICAS failed to produce them, then the corresponding files do not + % exist, but the code should still not try to generate them again since + % BICAS will likely just fail again and for every future attempt forever. + tpdFilenamesCa = cell(0,1); + % BPCSs for all passes so far. + BpcsAllArray = bicas.tools.batch.BicasProcessingCallSummary.empty(0, 1); + + + + iPass = 1; + while true + + fprintf('########################\n') + fprintf('BEGIN BICAS BATCH PASS %i\n', iPass) + fprintf('########################\n') + + %===================== + % Get reference DSMDs + %===================== + if ~isempty(referenceDir) + [RefDsmdArray, ~] = bicas.tools.batch.get_directory_DSMDs({referenceDir}); + else + RefDsmdArray = solo.adm.DSMD.empty(0, 1); + end + if DEBUG_ENABLED + log_DSMD_array('Reference directory (only recognized datasets)', RefDsmdArray) + end + + %===================================================================== + % Function handle to function which creates OUTPUT DATASET PATHS, + % including filenames, given relevant BPCI, datasets to consider when + % determining output data versions etc. + %===================================================================== + % IMPLEMENTATION NOTE: Removing TPD file names to make sure that any + % previously generated file would be re-generated by re-running (what is + % effectively) the same BPCI. + preexistingOutputFilenamesCa = setdiff(... + bicas.tools.batch.DSMDs_to_filenames(RefDsmdArray), ... + tpdFilenamesCa); + + % NEW ALGORITHM: FASTER + % --------------------- + % IMPLEMENTATION NOTE: Converts filenames to DSMDs to be able to + % efficiently extract only the latest versions (LV). Should possibly + % convert preexistingOutputFilenamesCa to DSMDs or paths to make + % this natural. + PreexistingOutputDsmdArray = solo.adm.paths_to_DSMD_array(preexistingOutputFilenamesCa(:)); + PreexistingOutputLvDsmdArray = solo.adm.group_sort_DSMD_versions(... + PreexistingOutputDsmdArray, ... + 'latest', 'sortWrtFormerVersionsDir', false); + + get_BPCI_output_path_fh = ... + @(outputDsi, BpciInputDsmdArray) ( ... + bicas.tools.batch.get_BPCI_output_path2(... + BpciInputDsmdArray, PreexistingOutputLvDsmdArray, ... + outputDsi, fnVerAlgorithm, ... + outputDir, outputIsCdag)); + + %==================================================== + % Autocreate all possible BPCIs based on input paths + %==================================================== + [InputDsmdArray, ~] = bicas.tools.batch.get_directory_DSMDs(inputPathsCa); + if DEBUG_ENABLED + log_DSMD_array('Input directories (only recognized datasets)', InputDsmdArray) + end + + BpciInputArray = bicas.tools.batch.autocreate_input_BPCIs(... + InputDsmdArray, get_BPCI_output_path_fh, SwmArray, ... + Settings.currentDatasetExtensionDays); + fprintf('Number of possible BPCIs: %3i\n', numel(BpciInputArray)) + + %======================================================================= + % Find out which subset of BPCIs that should actually be run + % ---------------------------------------------------------- + % NOTE: Takes reference directory into account but behaviour depends on + % fnVerAlgorithm in get_BPCI_output_path_fh. + % fnVerAlgorithm = 'HIGHEST_USED': + % Output dataset filenames have same version as highest + % counterpart in ref. dir., if there is one. ==> Filename + % collision. ==> Excluded + % fnVerAlgorithm = 'ABOVE_HIGHEST_USED': + % Output dataset filenames have a higher version than highest + % version counterpart in ref. dir., if there is one. + % ==> Never filename collision. ==> Included/kept. + %======================================================================= + doNotNeedToGenerateFilenamesCa = union(... + bicas.tools.batch.DSMDs_to_filenames(RefDsmdArray), ... + tpdFilenamesCa); + BpciRunArray = bicas.tools.batch.filter_BPCIs_to_run(... + BpciInputArray, doNotNeedToGenerateFilenamesCa); + fprintf('Number of BPCIs that will be run: %3i\n', numel(BpciRunArray)) + + if isempty(BpciRunArray) + % CASE: NO MORE BPCIs TO RUN + break + end + + %=============================================================== + % Try run BICAS for selected BPCIs + % -------------------------------- + % Skip BPCIs for which not all input datasets exist just before + % execution. + %=============================================================== + %error('DELIBERATELY EXITING BEFORE EXECUTING BICAS.') % DEBUG + BpcsPassArray = bicas.tools.batch.try_run_BICAS_for_BPCIs(... + Bpa, BpciRunArray, configFile, bicasSettingsArgsCa); + + BpcsAllArray = [BpcsAllArray; BpcsPassArray]; + + + + % BpcsAllArray --> Output dataset filenames = tpdFilenamesCa + % (= TPD filenames for all passes so far.) + tpdFilenamesCa = get_BPCSs_output_filenames(BpcsAllArray); + + iPass = iPass + 1; + end + + + + % Assign return values. + becArray = [BpcsAllArray.errorCode]; + nTpd = numel(tpdFilenamesCa); +end + + + +% Function for clarifying the code. +function filenamesCa = get_BPCSs_output_filenames(BpcsArray) + filenamesCaCa = arrayfun(... + @(Bpci) (Bpci.get_output_filenames()), ... + [BpcsArray(:).Bpci]', ... + 'UniformOutput', false); + + filenamesCa = cat(1, filenamesCaCa{:}); +end + + + +% Log DSMD array by listing paths, adding header/title and empty rows before and +% after. +% +% Can be used for debugging, if not for logging permanently. +function log_DSMD_array(titleStr, DsmdArray) + fprintf('\n') + fprintf('\n%s\n', titleStr) + fprintf('%s\n', repmat('=', 1, numel(titleStr))) + + nDsmd = numel(DsmdArray); + for i = 1:nDsmd + Dsmd = DsmdArray(i); + fprintf(' %s\n', Dsmd.path) + end + % IMPLEMENTATION NOTE: If there are zero DSMDs, then it is really important + % to print something instead of zero rows of paths since that is confusing + % (looks like a potential bug to the user). + fprintf('Total: %i\n', nDsmd) + + fprintf('\n') +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes___UTEST.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes___UTEST.m new file mode 100644 index 000000000..bee1ec3d1 --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/run_BICAS_all_passes___UTEST.m @@ -0,0 +1,406 @@ +% +% matlab.unittest automatic test code for +% bicas.tools.batch.run_BICAS_all_passes(). +% +% NOTE: Some tests use both possible values for FN_VER_ALGO since the result +% should be the same meaning it is easy to implement both tests. This slows down +% the testing while only adding little value. Could change this to speed up +% tests. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef run_BICAS_all_passes___UTEST < matlab.unittest.TestCase + % PROPOSAL: Replace by tests on bicas.tools.batch.main() + % NOTE: Do not want tests on both since much would be duplicated (if being thorough). + % PRO: More complete test. + % CON: Must change interface of bicas.tools.batch.main() to have BPA object. + % PRO: Makes it less convenient as manual command-line command. + % + % PROPOSAL: Qualitatively different cases: + % fnVerAlgorithm = 'ABOVE_HIGHEST_USED'; + % fnVerAlgorithm = 'HIGHEST_USED'; + % Match/non-match in reference directory. + % In the presence/absence of V01. + % BICAS reads latest version of input datasets. + % In the presence/absence of V01. + % Specifying SWMs to include/exclude + % (functionality not yet implemented 2024-01-12) + % Using output datasets to create new datasets. + % BICAS returns non-zero error code. + % TODO: BICAS raises exception. + % + % PROBLEM: Can not test for input files being added/deleted during/between passes. + % PROPOSAL: Convert iteration inside + % bicas.tools.batch.run_BICAS_all_passes() into separate + % function. Test code for it can call it multiple times to emulate + % calling bicas.tools.batch.run_BICAS_all_passes(), but + % add/delete files between calls. + % CON: There is communication & "processing" between iterations. + % Ex: BpcsAllArray is accumulated. + % Next pass does not use this information. Is just a sum over + % iteration results. + % Ex: tpdFilenamesCa is accumulated. + % + % PROBLEM: Can not test for using the correct input files (latest version). + % PROPOSAL: Have function return BPCSs or BPCIs. + + + + %################# + %################# + % TEST PARAMETERS + %################# + %################# + properties(TestParameter) + FN_VER_ALGO = {'HIGHEST_USED', 'ABOVE_HIGHEST_USED'} + end + + + + %############## + %############## + % TEST METHODS + %############## + %############## + methods(Test) + + + + % 0 (relevant) input files + function test1_0_input(testCase) + [~, P] = bicas.tools.batch.run_BICAS_all_passes___UTEST.get_test_dirs(testCase, {'in', 'out'}); + + irf.fs.create_empty_file(fullfile(P.in, 'NOT_DATASET.cdf')); + irf.fs.create_empty_file(fullfile(P.in, 'solo_L1_rpw-bia-current_20240101-20240131_V02.cdf')); + + bicas.tools.batch.run_BICAS_all_passes___UTEST.test1(... + {P.in}, '', P.out, 'HIGHEST_USED'); + end + + + + % 1 LV input file + % 1 NLV input file + % No ref. dir. + % (There is no V01 input file.) + % --> 1 output file + function test1_1LV_1NLV_to_1(testCase, FN_VER_ALGO) + [~, P] = bicas.tools.batch.run_BICAS_all_passes___UTEST.get_test_dirs(testCase, {'in', 'out'}); + + irf.fs.create_empty_file(fullfile(P.in, 'solo_L1R_rpw-lfr-surv-cwf-e_20240101_V02.cdf')); + INPUT_2 = fullfile(P.in, 'solo_L1R_rpw-lfr-surv-cwf-e_20240101_V03.cdf'); + irf.fs.create_empty_file(INPUT_2); + + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes___UTEST.test1(... + {P.in}, '', P.out, FN_VER_ALGO); + + assert(numel(ActBpcsArray) == 1) + % Assert used correct version of input file. + assert(strcmp(ActBpcsArray(1).Bpci.inputsArray.path, INPUT_2)) + irf.assert.file_exists(fullfile(P.out, 'solo_L2_rpw-lfr-surv-cwf-e_20240101_V01.cdf')) + end + + + + % 1 input file + % Ref. dir. file collision depending on output filename versioning. + % HIGHEST_USED ==> No output + % ABOVE_HIGHEST_USED ==> Increment version number + % + % NOTE: Ref. dir. file. is not V01! Still blocks output. + function test1_1_to_01_ref_collision(testCase, FN_VER_ALGO) + [testDir, P] = bicas.tools.batch.run_BICAS_all_passes___UTEST.get_test_dirs(testCase, {'in', 'ref', 'out'}); + irf.fs.create_empty_file(fullfile(P.in, 'solo_L1R_rpw-lfr-surv-cwf-e_20240101_V02.cdf')); + irf.fs.create_empty_file(fullfile(P.ref, 'solo_L2_rpw-lfr-surv-cwf-e_20240101_V05.cdf')); % Not V01. + + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes___UTEST.test1(... + {P.in}, P.ref, P.out, FN_VER_ALGO); + + % bicas.tools.batch.run_BICAS_all_passes___UTEST.disp_dir_tree(testDir) % DEBUG + + switch(FN_VER_ALGO) + case 'HIGHEST_USED' + % NOTE: No output file. + assert(numel(ActBpcsArray) == 0) + case 'ABOVE_HIGHEST_USED' + assert(numel(ActBpcsArray) == 1) + irf.assert.file_exists(fullfile(P.out, 'solo_L2_rpw-lfr-surv-cwf-e_20240101_V06.cdf')) + otherwise + error('') + end + end + + + + % 1x L1 + % --> 1x L2 + % --> 1x L3 + % + % NOTE: Output directory is also input directory. + function test1_1_to_1_to_1(testCase, FN_VER_ALGO) + [~, P] = bicas.tools.batch.run_BICAS_all_passes___UTEST.get_test_dirs(testCase, {'in', 'out'}); + irf.fs.create_empty_file(fullfile(P.in, 'solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240101_V02.cdf')); + + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes___UTEST.test1(... + {P.in, P.out}, '', P.out, FN_VER_ALGO); + + assert(numel(ActBpcsArray) == 2) + irf.assert.file_exists(fullfile(P.out, 'solo_L2_rpw-lfr-surv-cwf-e_20240101_V01.cdf')) + irf.assert.file_exists(fullfile(P.out, 'solo_L3_rpw-bia-density_20240101_V01.cdf')) + end + + + + + % 2x L1 + % --> 1x L2 + 1 non-zero error + % --> 1x L3 (2x L3 without error) + % + % IMPLEMENTATION NOTE: Want to test that code continues after BICAS + % returning non-zero error code. Therefore good to do this when the code + % is guaranteed to run one more pass after the pass where BICAS failed. + % This a failsafe against the code executing BPCIs in any order (within + % a given pass). + function test1_2_to_1_and_crash_to_1(testCase, FN_VER_ALGO) + [testDir, P] = bicas.tools.batch.run_BICAS_all_passes___UTEST.get_test_dirs(testCase, {'in', 'out'}); + + INPUT_FILE_1 = fullfile(P.in, 'solo_L1R_rpw-lfr-surv-cwf-e_20240101_V02.cdf'); % Crashes + irf.fs.create_empty_file(INPUT_FILE_1); + irf.fs.create_empty_file(fullfile(P.in, 'solo_L1R_rpw-lfr-surv-cwf-e_20240102_V02.cdf')); + + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes___UTEST.test1(... + {P.in, P.out}, '', P.out, FN_VER_ALGO, [1]); + + assert(numel(ActBpcsArray) == 3) + irf.assert.file_exists(fullfile(P.out, 'solo_L2_rpw-lfr-surv-cwf-e_20240102_V01.cdf')) + irf.assert.file_exists(fullfile(P.out, 'solo_L3_rpw-bia-density_20240102_V01.cdf')) + + % Assert one error, one non-error (without assuming order). + errorCodeArray = [ActBpcsArray.errorCode]; + testCase.assertEqual(sort(errorCodeArray), [0, 0, 1]) + + % Assert correct BPCI failed. + iError = find(errorCodeArray); + testCase.assertTrue(strcmp(ActBpcsArray(iError).Bpci.inputsArray(1).path, INPUT_FILE_1)) + end + + + + % Process + % 2x L1 --> 1x L2 --> 2x L3 + % Empty ref. dir.. + function test2_2_to_1_to_2(testCase, FN_VER_ALGO) + [testDir, P] = bicas.tools.batch.run_BICAS_all_passes___UTEST.get_test_dirs(testCase, {'in', 'out'}); + irf.fs.create_empty_file(fullfile(P.in, 'solo_L1R_rpw-lfr-surv-cwf-e-cdag_20240101_V02.cdf')); + irf.fs.create_empty_file(fullfile(P.in, 'solo_L1_rpw-bia-current_20240101-20240131_V02.cdf')); + + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes___UTEST.test2(... + {P.in, P.out}, '', P.out, FN_VER_ALGO); + + assert(numel(ActBpcsArray) == 2) + irf.assert.file_exists(fullfile(P.out, 'solo_L2_rpw-lfr-surv-cwf-e_20240101_V01.cdf')) + irf.assert.file_exists(fullfile(P.out, 'solo_L3_rpw-bia-density_20240101_V01.cdf')) + irf.assert.file_exists(fullfile(P.out, 'solo_L3_rpw-bia-density-10-seconds_20240101_V01.cdf')) + end + + + + % Process + % 1x L2 --> 2x L3 + % 1x L3 ref. dir. collision. + % ==> Ref. dir. does not block since there is still one output dataset + % which is not in the ref. dir.. + function test2_1_to_2_ref_collision(testCase, FN_VER_ALGO) + [testDir, P] = bicas.tools.batch.run_BICAS_all_passes___UTEST.get_test_dirs(testCase, {'in', 'ref', 'out'}); + irf.fs.create_empty_file(fullfile(P.in, 'solo_L2_rpw-lfr-surv-cwf-e_20240101_V01.cdf')); + irf.fs.create_empty_file(fullfile(P.ref, 'solo_L3_rpw-bia-density_20240101_V01.cdf')); + + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes___UTEST.test2(... + {P.in, P.out}, P.ref, P.out, FN_VER_ALGO); + + assert(numel(ActBpcsArray) == 1) + irf.assert.file_exists(fullfile(P.out, 'solo_L3_rpw-bia-density-10-seconds_20240101_V01.cdf')) + switch(FN_VER_ALGO) + case 'HIGHEST_USED' + irf.assert.file_exists(fullfile(P.out, 'solo_L3_rpw-bia-density_20240101_V01.cdf')) + + case 'ABOVE_HIGHEST_USED' + irf.assert.file_exists(fullfile(P.out, 'solo_L3_rpw-bia-density_20240101_V02.cdf')) + otherwise + error('') + end + + end + + + + end % methods(Test) + + + + %######################## + %######################## + % PRIVATE STATIC METHODS + %######################## + %######################## + methods(Static, Access=private) + + + + % Call BICAS for predefined SWMs, but with datasets specified by caller. + % + % SWMs: + % 1 L1 in --> 1 L2 out + % 1 L2 in --> 1 L3 out + % + % NOTE: DSIs. + % NOTE: Function does not verify result. The caller has to create input + % files and verify output files and BPCSs. + % + % NOTE: Test functions which use this function should be prefixed + % "test1". + % + function ActBpcsArray = test1(inputPathsCa, referenceDir, outputDir, fnVerAlgorithm, varargin) + + switch(numel(varargin)) + case 0 + callNonZeroErrorArray = zeros(0, 1); + case 1 + callNonZeroErrorArray = varargin{1}; + otherwise + error('') + end + + BICAS_SETTINGS_ARGS_CA = {}; + BICAS_CONFIG_FILE = 'NO_CONFIG_FILE.conf'; + SETTINGS = []; + SETTINGS.currentDatasetExtensionDays = 0; + + DSI_1 = 'SOLO_L1R_RPW-LFR-SURV-CWF-E'; + DSI_2 = 'SOLO_L2_RPW-LFR-SURV-CWF-E'; + DSI_3 = 'SOLO_L3_RPW-BIA-DENSITY'; + + SWMP = bicas.tools.batch.TestSwmProcessing(); + + SWM_1 = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_1', 'SWD purpose', ... + bicas.swm.InputDataset( 'cli_in', DSI_1, 'IN_cdf'), ... + bicas.swm.OutputDataset('cli_out', DSI_2, 'OUT_cdf', 'SWD ', 'SWD ', '02') ... + ); + SWM_2 = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_2', 'SWD purpose', ... + bicas.swm.InputDataset( 'cli_in', DSI_2, 'IN_cdf'), ... + bicas.swm.OutputDataset('cli_out', DSI_3, 'OUT_cdf', 'SWD ', 'SWD ', '03') ... + ); + + BPA = bicas.tools.batch.BicasProcessingAccessTest([SWM_1; SWM_2], callNonZeroErrorArray); + + % CALL TESTED CODE + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes(... + BPA, BICAS_SETTINGS_ARGS_CA, ... + BICAS_CONFIG_FILE, outputDir, referenceDir, inputPathsCa, ... + fnVerAlgorithm, false, [SWM_1; SWM_2], SETTINGS); + end + + + + % Call BICAS for predefined SWMs, but with datasets specified by caller. + % + % SWMs: + % 2 L1 in --> 1 L2 out + % 1 L2 in --> 2 L3 out + % + % NOTE: Test functions which use this function should be prefixed + % "test2". + % + function ActBpcsArray = test2(inputPathsCa, referenceDir, outputDir, fnVerAlgorithm) + BICAS_SETTINGS_ARGS_CA = {}; + BICAS_CONFIG_FILE = 'NO_CONFIG_FILE.conf'; + SETTINGS = []; + SETTINGS.currentDatasetExtensionDays = 0; + DSI_1a = 'SOLO_L1R_RPW-LFR-SURV-CWF-E'; + DSI_1b = 'SOLO_L1_RPW-BIA-CURRENT'; + DSI_2 = 'SOLO_L2_RPW-LFR-SURV-CWF-E'; + DSI_3a = 'SOLO_L3_RPW-BIA-DENSITY'; + DSI_3b = 'SOLO_L3_RPW-BIA-DENSITY-10-SECONDS'; + + SWMP = bicas.tools.batch.TestSwmProcessing(); + + SWM_1 = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_1', 'SWD purpose', ... + [... + bicas.swm.InputDataset( 'cli_in1', DSI_1a, 'IN_1_cdf'), ... + bicas.swm.InputDataset( 'cli_in2', DSI_1b, 'IN_2_cdf') ... + ], ... + bicas.swm.OutputDataset('cli_out', DSI_2, 'OUT_cdf', 'SWD ', 'SWD ', '02') ... + ); + SWM_2 = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_2', 'SWD purpose', ... + bicas.swm.InputDataset( 'cli_in', DSI_2, 'IN_cdf'), ... + [ + bicas.swm.OutputDataset('cli_out1', DSI_3a, 'OUT_1_cdf', 'SWD ', 'SWD ', '03'), ... + bicas.swm.OutputDataset('cli_out2', DSI_3b, 'OUT_2_cdf', 'SWD ', 'SWD ', '03') ... + ] ... + ); + + BPA = bicas.tools.batch.BicasProcessingAccessTest([SWM_1; SWM_2], []); + + % CALL TESTED CODE + ActBpcsArray = bicas.tools.batch.run_BICAS_all_passes(... + BPA, BICAS_SETTINGS_ARGS_CA, ... + BICAS_CONFIG_FILE, outputDir, referenceDir, inputPathsCa, ... + fnVerAlgorithm, false, [SWM_1; SWM_2], SETTINGS); + end + + + + % Helper function for creating multiple test directories. + % + % subdirsCa + % Subdirectory name. Can not be recursive. + % P (="Paths) + % Struct with fields named after subdirectories, containing full + % paths. + function [testDir, P] = get_test_dirs(testCase, subdirsCa) + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture) + testDir = pwd; + cd('~') % Move to any OTHER unrelated directory. + + P = struct(); + for i = 1:numel(subdirsCa) + subdirName = subdirsCa{i}; + mkdir(fullfile(testDir, subdirName)) + P.(subdirName) = fullfile(testDir, subdirName); + end + end + + + + % For debugging. + % + % Print files in directory tree. For debugging failed tests. + % + function disp_dir_tree(dirPath) + ROW_LINE = [repmat('=', 1, 120), '\n']; + fprintf(ROW_LINE) + + FsoiArray = dir(fullfile(dirPath, '**')); + for i = 1:numel(FsoiArray) + Fsoi = FsoiArray(i); + if ~ismember(Fsoi.name, {'.', '..'}) + path = fullfile(Fsoi.folder, Fsoi.name); + fprintf('%s\n', path); + end + end + + fprintf(ROW_LINE) + end + + + + end % methods(Static, Access=private) + + + +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs.m new file mode 100644 index 000000000..664d7482e --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs.m @@ -0,0 +1,119 @@ +% +% Try run BICAS for array of specified BPCI. If the input datasets can not be +% found immediately before a BICAS run, then that BPCI is skipped. This is to +% prevent the code from confusing a "genuine" BICAS error (which the code can +% not interpret) with BICAS merely not being able to read missing input +% datasets. +% +% +% RETURN VALUE +% ============ +% BpcsArray +% Column array of BPCSs for the BICAS calls made. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +function BpcsArray = try_run_BICAS_for_BPCIs(... + Bpa, BpciArray, configFile, bicasSettingsArgsCa) + + % PROPOSAL: Better name. Omit "try". + % CON: "try" implies handling failure. + % CON: Does imply catching exception, only that exception may occur. + + assert(isa(Bpa, 'bicas.tools.batch.BicasProcessingAccessAbstract')) + assert(isa(BpciArray, 'bicas.tools.batch.BicasProcessingCallInfo')) + assert(iscolumn(BpciArray)) + assert(iscell(bicasSettingsArgsCa)) + + BpcsArray = bicas.tools.batch.BicasProcessingCallSummary.empty(0, 1); + + for iBpci = 1:numel(BpciArray) + Bpci = BpciArray(iBpci); + + assert(numel(Bpci.outputsArray) >= 1) + + argsCa = bicas.tools.batch.BPCI_to_BICAS_call_args(Bpci); + + % NOTE: Uses first output file (i.e. no other output files) to name the + % log file. + outputPath = Bpci.outputsArray(1).path; + timestampStr = datestr(now, 'YYYY-mm-ddTHH.MM.SS'); + logFile = sprintf('%s.%s.log', outputPath, timestampStr); + + % IMPLEMENTATION NOTE: Overriding BICAS settings wrt. SWMs used for + % identifying can NOT be done here only since the exact set of SWMs + % is needed earlier for grouping input datasets. + argsCa(end+1:end+2) = {'--config', configFile}; + argsCa(end+1:end+2) = {'--log-matlab', logFile}; + argsCa = [argsCa(:); bicasSettingsArgsCa(:)]; + + %======================================== + % Check if input datasets are available, + % otherwise skip and try the next BPCI + %======================================== + [inputPathsValid, firstInvalidPath] = input_dataset_paths_valid(Bpci); + if ~inputPathsValid + warning([... + 'Can not find previously identified input file needed for a call', ... + ' to BICAS. This may e.g. be due to that', ... + ' (1) it has been moved during the execution of this script, or', ... + ' (2) the disk was temporarily inaccessible (nas24 automount problem).', ... + ' It will', ... + ' be searched for again in the next pass.\n firstInvalidPath="%s"'], ... + firstInvalidPath) + continue + end + + %==================================== + % Run BICAS + % Create BPCS for the completed call + %==================================== + fprintf(... + 'CALLING BICAS WITH THE FOLLOWING ARGUMENTS: %s\n', ... + ['"', strjoin(argsCa, sprintf('"\n "')), '"']) + + %############ + % CALL BICAS + %############ + errorCode = Bpa.bicas_main(argsCa{:}); + + Bpcs = bicas.tools.batch.BicasProcessingCallSummary(... + Bpci, errorCode); + + BpcsArray(end+1, 1) = Bpcs; + end +end + + + +% Check whether the input datasets for one particular BPCI actually exist. Meant +% to be used just before launching BICAS with that BPCI to prevent launching +% BICAS for input datasets that have just been moved (in particular moved to +% subdirectory former_versions/ due to syncing ROC data while executing this +% code). +% +% NOTE: Does NOT work on an BPCI array since the check has to be done JUST +% BEFORE launching BICAS. +% +% +% RETURN VALUES +% ============= +% valid +% Logical. Whether Bpci input datasets can be found or not. +% firstInvalidPath +% String. Path. Only set when valid==true. +% +function [valid, firstInvalidPath] = input_dataset_paths_valid(Bpci) + assert(isscalar(Bpci)) + + valid = true; % Value until set to the opposite. + firstInvalidPath = []; + for i = 1:numel(Bpci.inputsArray) + path = Bpci.inputsArray(i).path; + if ~exist(path, 'file') + firstInvalidPath = path; + valid = false; + end + end +end diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs___UTEST.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs___UTEST.m new file mode 100644 index 000000000..b9d168ebd --- /dev/null +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/try_run_BICAS_for_BPCIs___UTEST.m @@ -0,0 +1,285 @@ +% +% matlab.unittest automatic test code for +% bicas.tools.batch.try_run_BICAS_for_BPCIs(). +% +% The code works, but is too complicated to be worth it in hindsight?! +% +% NOTE: Does not test BICAS failing (returning on-zero error code). +% +% IMPLEMENTATION NOTE: Code has to use (almost) real DSIs due to BICAS data +% structure assertions. +% +% +% Author: Erik P G Johansson, IRF, Uppsala, Sweden +% +classdef try_run_BICAS_for_BPCIs___UTEST < matlab.unittest.TestCase + + + + %############## + %############## + % TEST METHODS + %############## + %############## + methods(Test) + + + + function test_zero_BPCIs(testCase) + + BPA = bicas.tools.batch.BicasProcessingAccessTest(... + bicas.swm.SoftwareMode.empty(0, 1), []); + + bicas.tools.batch.try_run_BICAS_for_BPCIs___UTEST.test(... + testCase, BPA, ... + bicas.tools.batch.BicasProcessingCallInfo.empty(0, 1), ... + bicas.tools.batch.BicasProcessingCallSummary.empty(0, 1)) + + end + + + + function test_one_BPCI(testCase) + + % NOTE: Changes current working directory. + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture) + + % NOTE: Dependence on BICAS. + SWMP = bicas.tools.batch.TestSwmProcessing(); + + DSI_1 = 'SOLO_L1R_RPW-LFR-SURV-CWF-E'; + DSI_2 = 'SOLO_L2_RPW-LFR-SURV-CWF-E'; + + SWM = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_1', 'SWD purpose', ... + bicas.swm.InputDataset(... + 'cli_input', DSI_1, 'IN_cdf' ... + ), ... + bicas.swm.OutputDataset(... + 'cli_output', DSI_2, 'OUT_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + ); + BPA = bicas.tools.batch.BicasProcessingAccessTest(SWM, []); + + OUTPUT_DIR = fullfile(pwd, 'dir2'); + mkdir(OUTPUT_DIR) + + INPUT_FILE = fullfile(pwd, 'dir1', 'input_dataset.cdf'); + OUTPUT_FILE = fullfile(OUTPUT_DIR, 'output_dataset.cdf'); + + BPCI = bicas.tools.batch.BicasProcessingCallInfo(... + 'CLI_SWM_1', ... + bicas.tools.batch.BpciInput( 'cli_input', DSI_1, INPUT_FILE), ... + bicas.tools.batch.BpciOutput('cli_output', DSI_2, OUTPUT_FILE)); + + % NOTE: BPCS should only contain filenames. + EXP_BPCS = bicas.tools.batch.BicasProcessingCallSummary(... + BPCI, 0); + + + + bicas.tools.batch.try_run_BICAS_for_BPCIs___UTEST.test(... + testCase, BPA, ... + BPCI, ... + EXP_BPCS) + end + + + + function test_complex(testCase) + + % NOTE: Dependence on BICAS. + SWMP = bicas.tools.batch.TestSwmProcessing(); + + DSI_1 = 'SOLO_L1R_RPW-LFR-SURV-CWF-E'; + DSI_2 = 'SOLO_L1R_RPW-LFR-SURV-SWF-E'; + DSI_3 = 'SOLO_L1R_RPW-LFR-SBM1-CWF-E'; + + SWM_1 = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_1', 'SWD purpose', ... + [ + bicas.swm.InputDataset('cli_input_1', DSI_1, 'IN_1_cdf'), ... + bicas.swm.InputDataset('cli_input_2', DSI_2, 'IN_2_cdf') ... + ], ... + bicas.swm.OutputDataset(... + 'cli_output', DSI_3, 'OUT_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + ); + SWM_2 = bicas.swm.SoftwareMode(... + SWMP, 'CLI_SWM_2', 'Human readable purpose', ... + bicas.swm.InputDataset('cli_input', DSI_1, 'IN_cdf'), ... + [... + bicas.swm.OutputDataset(... + 'cli_output_1', DSI_2, 'OUT_1_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + bicas.swm.OutputDataset(... + 'cli_output_2', DSI_3, 'OUT_2_cdf', ... + 'SWD name', 'SWD description', '02' ... + ) ... + ] ... + ); + + BPA = bicas.tools.batch.BicasProcessingAccessTest([SWM_1; SWM_2], []); + + + + function [Bpci, Bpcs] = get_BPCI_1(inputFile1, inputFile2, outputFile) + Bpci = bicas.tools.batch.BicasProcessingCallInfo(... + 'CLI_SWM_1', ... + [ + bicas.tools.batch.BpciInput('cli_input_1', DSI_1, inputFile1); ... + bicas.tools.batch.BpciInput('cli_input_2', DSI_2, inputFile2) ... + ], ... + bicas.tools.batch.BpciOutput('cli_output', DSI_3, outputFile)); + + Bpcs = bicas.tools.batch.BicasProcessingCallSummary(... + Bpci, 0); + end + + function [Bpci, Bpcs] = get_BPCI_2(inputFile, outputFile1, outputFile2) + Bpci = bicas.tools.batch.BicasProcessingCallInfo(... + 'CLI_SWM_2', ... + bicas.tools.batch.BpciInput('cli_input', DSI_1, inputFile), ... + [... + bicas.tools.batch.BpciOutput('cli_output_1', DSI_2, outputFile1); ... + bicas.tools.batch.BpciOutput('cli_output_2', DSI_3, outputFile2) ... + ]... + ); + Bpcs = bicas.tools.batch.BicasProcessingCallSummary(... + Bpci, 0); + end + + function C = prepare_test() + % Create temporary directory. + % NOTE: Changes current working directory. + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture) + + % NOTE: Only relative paths. + % Using subdirectories just to test paths (not just filenames). + [C.BPCI_1a, C.EXP_BPCS_1a] = get_BPCI_1('11/i1', '11/i2', '11/o1' ); + [C.BPCI_1b, C.EXP_BPCS_1b] = get_BPCI_1('12/i1', '12/i2', '12/o1' ); + [C.BPCI_2a, C.EXP_BPCS_2a] = get_BPCI_2('21/i1', '21/o1', '21/o2'); + [C.BPCI_2b, C.EXP_BPCS_2b] = get_BPCI_2('22/i1', '22/o1', '22/o2'); + end + + % NOTE: BPCS should only contain filenames. + % NOTE: Comparisons rely on that the order in arrays stays the same + % even though the code does not really guarantee it. Could + % sort CAs of strings in relevant data structures to make + % comparisons easier. + % IMPLEMENTATION NOTE: Makes sure to have a new temporary + % directory for every call to + % bicas.tools.batch.try_run_BICAS_for_BPCIs___UTEST.test() + % so that no files from previous call can affect the new call. + + C = prepare_test(); + bicas.tools.batch.try_run_BICAS_for_BPCIs___UTEST.test(... + testCase, BPA, ... + C.BPCI_1a, ... + C.EXP_BPCS_1a) + + C = prepare_test(); + bicas.tools.batch.try_run_BICAS_for_BPCIs___UTEST.test(... + testCase, BPA, ... + C.BPCI_2a, ... + C.EXP_BPCS_2a) + + C = prepare_test(); + bicas.tools.batch.try_run_BICAS_for_BPCIs___UTEST.test(... + testCase, BPA, ... + [ C.BPCI_1a; C.BPCI_2a; C.BPCI_1b; C.BPCI_2b], ... + [C.EXP_BPCS_1a; C.EXP_BPCS_2a; C.EXP_BPCS_1b; C.EXP_BPCS_2b]) + end + + + + end % methods(Test) + + + + %######################## + %######################## + % PRIVATE STATIC METHODS + %######################## + %######################## + methods(Static, Access=private) + + + + % Make test call to function. Create input files before, check existence + % of output files after. + % + % NOTE: Can not create test directory in this function, since the BPCIs + % contain the full paths to datasets. + function test(testCase, Bpa, BpciArray, ExpBpcsArray) + + % =========================================== + % Create empty input files specified in BPCIs + % =========================================== + function create_input_files(BpciArray) + for iBpci = 1:numel(BpciArray) + Bpci = BpciArray(iBpci); + for iFile = 1:numel(Bpci.inputsArray) + + path = Bpci.inputsArray(iFile).path; + + % Create parent directory if needed. + parentDir = fileparts(path); + if ~isempty(parentDir) && ~exist(parentDir, 'dir') + mkdir(parentDir) + end + + irf.fs.create_empty_file(path); + end + end + end + + function main() + assert(iscolumn(ExpBpcsArray)) + + CONFIG_FILE = 'NO_CONFIG_FILE'; + BICAS_SETTINGS_ARGS_CA = {}; + + create_input_files(BpciArray) + + + + %====================== + % CALL TESTED FUNCTION + %====================== + ActBpcsArray = bicas.tools.batch.try_run_BICAS_for_BPCIs(... + Bpa, BpciArray, ... + CONFIG_FILE, BICAS_SETTINGS_ARGS_CA); + + + + testCase.assertEqual(... + ActBpcsArray, ... + ExpBpcsArray) + + %===================================== + % Assert that BPCI output files exist + %===================================== + for i = 1:numel(BpciArray) + Bpci = BpciArray(i); + outputPathCa = {Bpci.outputsArray(:).path}; + + for iFile = 1:numel(outputPathCa) + irf.assert.file_exists(outputPathCa{iFile}) + end + end + end + + main() + end + + + + end % methods(Static, Access=private) + + + +end From 69395aced2ce6d6497c52d4f23119e49c961c273 Mon Sep 17 00:00:00 2001 From: Erik P G Johansson Date: Mon, 26 Feb 2024 11:10:56 +0100 Subject: [PATCH 7/7] BICAS: Bugfix: get_directory_DSMDs(): Assert directories --- .../bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m index 241134697..00225dd7d 100644 --- a/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m +++ b/mission/solar_orbiter/bicas/src/+bicas/+tools/+batch/get_directory_DSMDs.m @@ -34,10 +34,16 @@ DsmdArray = solo.adm.DSMD.empty(0,1); for i = 1:numel(dirPathsCa) + % Assert that directory exists. + % IMPLEMENTATION NOTE: dir(fullfile(dirPathsCa{i}, '**')) will NOT raise + % error for non-existing directory. + irf.assert.dir_exists(dirPathsCa{i}) + DirOiArray = dir(fullfile(dirPathsCa{i}, '**')); DirOiArray = DirOiArray(~[DirOiArray.isdir]); - dirFilesPathsCa = arrayfun(@(Oi) (fullfile(Oi.folder, Oi.name)), DirOiArray, 'UniformOutput', false); + DirOiArray = DirOiArray(:); % CASE: DirOiArray is a column array. + dirFilesPathsCa = arrayfun(@(Oi) (fullfile(Oi.folder, Oi.name)), DirOiArray, 'UniformOutput', false); [DirDsmdArray, bIsDataSetArray] = solo.adm.paths_to_DSMD_array(dirFilesPathsCa); DirOiArray = DirOiArray(bIsDataSetArray);