Merge branch 'devel' into SOdevel

Merge to gain access to bicas.tools.batch on SOdevel.

irf/+irf/+fs/create_empty_file.m

@@ -0,0 +1,26 @@
+%
+% Create empty file. This is useful for debugging purposes sometimes, e.g.
+% if batch code selects the correct files to be created.
+%
+%
+% ARGUMENTS
+% =========
+% path
+%       Path to file.
+%       NOTE: Parent directory has to pre-exist.
+%
+%
+% Author: Erik P G Johansson, IRF, Uppsala, Sweden
+%
+function create_empty_file(path)
+    % PROPOSAL: Error if fails to create file.
+    % PROPOSAL: Policy arguments
+    %   (1) Whether to permit path not available:
+    %       Permit overwrite
+    %       Assert no overwrite
+    %   (2) What happens when createing, writing file:
+    %       Assert write success (does not assert no overwrite)
+    %       Permit write failure (does not assert no overwrite), return ~error code/boolean
+
+    fclose(fopen(path, 'w'));
+end

mission/solar_orbiter/+solo/+adm/assert_no_time_overlap___UTEST.m

@@ -19,14 +19,14 @@
     function test0(testCase)
 
       function test(filePathCa)
-        DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa);
+        DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa(:));
 
         % Test that does not raise exception.
         solo.adm.assert_no_time_overlap(DsmdArray);
       end
 
       function test_exc(filePathCa)
-        DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa);
+        DsmdArray = solo.adm.paths_to_DSMD_array(filePathCa(:));
         testCase.verifyError(...
           @() solo.adm.assert_no_time_overlap(DsmdArray), ...
           ?MException)

mission/solar_orbiter/+solo/+adm/filter_DSMD_CURRENT_largest_time_coverage___UTEST.m

@@ -18,12 +18,12 @@
 
     function test0(testCase)
 
-      function test(inputFileList, expFileList)
-        inputDsmdArray = solo.adm.paths_to_DSMD_array( inputFileList);
-        expDsmdArray   = solo.adm.paths_to_DSMD_array(expFileList);
+      function test(inputFileCa, expFileCa)
+        InputDsmdArray = solo.adm.paths_to_DSMD_array(inputFileCa(:));
+        ExpDsmdArray   = solo.adm.paths_to_DSMD_array(expFileCa(:));
 
-        actDsmdArray = solo.adm.filter_DSMD_CURRENT_largest_time_coverage(inputDsmdArray);
-        testCase.assertEqual(actDsmdArray, expDsmdArray)
+        ActDsmdArray = solo.adm.filter_DSMD_CURRENT_largest_time_coverage(InputDsmdArray);
+        testCase.assertEqual(ActDsmdArray, ExpDsmdArray)
       end
 
       X1  = 'solo_L2_rpw-lfr-surv-swf-e-cdag_20200314_V01.cdf';

mission/solar_orbiter/+solo/+adm/find_overlapping_DSMD_groups.m

@@ -6,7 +6,8 @@
 %   NOTE: Overlap must have non-zero length.
 %
 % NOTE: Algorithm only works under the assumption that datasets with the same
-% DATASET_ID do not overlap (in time) (assertion).
+%       DATASET_ID do not overlap (in time) (assertion). Can thus not handle
+%       SOLO_L2_RPW-LFR-SBM1/2-CWF-E.
 %
 %
 % ARGUMENTS

mission/solar_orbiter/+solo/+adm/parse_dataset_filename_many.m

@@ -10,6 +10,12 @@
 % wrong that function name mentions filenames, not paths.
 %
 %
+% ARGUMENT
+% ========
+% filePathCa
+%       Cell column array of paths to files. Can be both datasets and not.
+%
+%
 % RETURN VALUE
 % ============
 % fiCa
@@ -18,12 +24,15 @@
 %           solo.adm.parse_dataset_filename() plus extra field
 %           below:
 %               .path : Path in filePathList{iFile}.
+% bIsDatasetArray
+%       Logical column array. Same size as argument. True iff the corresponding
+%       input path was interpreted as a dataset (was translated into a DSMD).
 %
 %
 % Author: Erik P G Johansson, IRF, Uppsala, Sweden
 % First created 2020-04-25.
 %
-function fiCa = parse_dataset_filename_many(filePathCa)
+function [fiCa, bIsDatasetArray] = parse_dataset_filename_many(filePathCa)
 % PROPOSAL: Change name
 %   PROPOSAL: parse_dataset_filenames_many  ("FILENAMES" in plural)
 %   PROPOSAL: parse_dataset_filename_many_paths
@@ -38,10 +47,15 @@
 %   TODO-DEC: Different policy for suffix .cdf and not?
 % PROPOSAL: Assertion for parsable *.cdf filenames. Ignore filenames without
 %           suffix ".cdf".
+%
+% NOTE: Has no separate test code. Is indirectly tested by
+%       solo.adm.paths_to_DSMD_array___UTEST.
 
-assert(iscell(filePathCa), 'filePathList is not a cell array.')
+assert(iscell(filePathCa),   'filePathCa is not a cell array.')
+assert(iscolumn(filePathCa), 'filePathCa is not a column array.')
 
-fiCa = cell(0,1);
+fiCa            = cell(0, 1);
+bIsDatasetArray = false(numel(filePathCa), 1);
 for iFile = 1:numel(filePathCa)
 
   filename = irf.fs.get_name(filePathCa{iFile});
@@ -55,12 +69,14 @@
     Fi      = R;
     Fi.path = filePathCa{iFile};
 
-    fiCa{end+1, 1} = Fi;
+    fiCa{           end+1, 1} = Fi;
+    bIsDatasetArray(iFile, 1) = true;
   else
     % CASE: Can not identify as dataset filename.
 
     % Do nothing. (Silently ignore files that can not be identified as
     % datasets.)
+    bIsDatasetArray(iFile, 1) = false;
   end
 end
 end

mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array.m

@@ -17,13 +17,16 @@
 % RETURN VALUE
 % ============
 % DsmdArray
-%       Array of DSMD objects.
+%       Column array of DSMD objects.
+% bIsDatasetArray
+%       Logical column array. Same size as argument. True iff the corresponding
+%       input path was interpreted as a dataset (was translated into a DSMD).
 %
 %
 % Author: Erik P G Johansson, IRF, Uppsala, Sweden
 % First created 2020-05-08.
 %
-function DsmdArray = paths_to_DSMD_array(filePathCa)
+function [DsmdArray, bIsDatasetArray] = paths_to_DSMD_array(filePathCa)
 % PROPOSAL: Rename
 %   PROPOSAL: DSMDs_from_paths()
 %   PROPOSAL: paths_to_DSMDs().
@@ -47,9 +50,10 @@
 %           from parsable filename: ignore, warning, error.
 
 % FI = File Info
-fiCa = solo.adm.parse_dataset_filename_many(filePathCa);
+[fiCa, bIsDatasetArray] = solo.adm.parse_dataset_filename_many(filePathCa);
+
+DsmdArray = solo.adm.DSMD.empty(0, 1);
 
-DsmdArray = solo.adm.DSMD.empty(0,1);
 for i = 1:numel(fiCa)
   Fi = fiCa{i};
 

mission/solar_orbiter/+solo/+adm/paths_to_DSMD_array___UTEST.m

@@ -26,9 +26,10 @@ function test0(testCase)
       %   solo_L1_rpw-bia-sweep-cdag_20200307T053018-20200307T053330_V01.cdf
       %   solo_L1_rpw-bia-current-cdag_20200401T000000-20200421T000000_V01.cdf
 
-      function test(filePathList, expDsmdArray)
-        actDsmdArray = solo.adm.paths_to_DSMD_array(filePathList);
-        testCase.assertEqual(expDsmdArray, actDsmdArray)
+      function test(filePathList, expDsmdArray, expBIsDatasetArray)
+        [actDsmdArray, actBIsDatasetArray] = solo.adm.paths_to_DSMD_array(filePathList(:));
+        testCase.assertEqual(actDsmdArray,       expDsmdArray)
+        testCase.assertEqual(actBIsDatasetArray, expBIsDatasetArray)
       end
 
       function dt = dtu(varargin)
@@ -60,26 +61,37 @@ function test(filePathList, expDsmdArray)
 
 
       % Empty argument (no files)
-      test({}, solo.adm.DSMD.empty(0,1));
+      test(...
+          {}, ...
+          solo.adm.DSMD.empty(0,1), ...
+          false(0,1));
 
-      test({FILE_CDF_IGNORE},    solo.adm.DSMD.empty(0,1));
-      test({FILE_NONCDF_IGNORE}, solo.adm.DSMD.empty(0,1));
+      test(...
+          {FILE_CDF_IGNORE}, ...
+          solo.adm.DSMD.empty(0,1), ...
+          false(1,1));
+      test(...
+          {FILE_NONCDF_IGNORE}, ...
+          solo.adm.DSMD.empty(0,1), ...
+          false(1,1));
 
       test(...
         {DSMD_1.path}, ...
-        [DSMD_1]);
-
+        [DSMD_1], ...
+        true(1,1));
       test(...
         {DSMD_2.path}, ...
-        [DSMD_2]);
-
+        [DSMD_2], ...
+        true(1,1));
       test(...
         {DSMD_3.path}, ...
-        [DSMD_3]);
+        [DSMD_3], ...
+        true(1,1));
 
       test(...
         {DSMD_1.path; FILE_NONCDF_IGNORE; DSMD_2.path; FILE_CDF_IGNORE; DSMD_3.path}, ...
-        [DSMD_1; DSMD_2; DSMD_3]);
+        [DSMD_1; DSMD_2; DSMD_3], ...
+        logical([1; 0; 1; 0; 1]));
     end
 
 

mission/solar_orbiter/+solo/+qli/README.TXT

@@ -3,5 +3,14 @@ QLI = QuickLooks IRFU ("IRFU quicklooks")
 
 SolO quicklooks generated at IRFU.
 
-NOTE: Uses NOT only SolO/RPW data.
+NOTE: Uses SolO/RPW data AND data from instruments.
 NOTE: Not to be confused with LESIA's summary plots or BIA_QL3.
+
+
+
+If you want to edit the plots, check files
+    quicklooks_24_6_2_h.m
+    quicklooks_7days.m
+
+If you want to produce plots, call
+    quicklooks_main.m

mission/solar_orbiter/+solo/+qli/quicklooks_24_6_2_h.m

@@ -1,9 +1,16 @@
 function quicklooks_24_6_2_h(data,paths,Tint_24h,logoPath)
+%
 % Given data in the struct 'data' (see solo.qli.quicklooks_main), generates
 % plots and saves them in the paths specified in the struct 'paths' (see
 % solo.qli.quicklooks_main). Computes spectrum of B, so takes a while to run.
 % Tint_24h should be a 24hour time interval, e.g.
 % irf.tint('2020-06-01T00:00:00.00Z','2020-06-02T00:00:00.00Z');
+%
+% NOTE: Function uses solo.read_TNR() indirectly which in turns relies on a
+%       hardcoded path to "/data/solo/remote/data/L2/thr/" and selected
+%       subdirectories.
+
+
 
 % BUG?: Panel 2/density/abs(B): Sometimes has no left-hand ticks (for density?).
 %   /EJ 2023-05-10
@@ -33,6 +40,21 @@ function quicklooks_24_6_2_h(data,paths,Tint_24h,logoPath)
 % TODO-NI Panel 10 (log) is hardcoded to YLim~[10, 100] (because that is what
 %         it used to be). This does not cover the entire interval of data
 %         (there is more data at lower y). Should it be that way?
+%
+% PROPOSAL: Make function not directly call solo.read_TNR()
+%   PRO: Makes function testable.
+%   CON: Must understand the solo.read_TNR() return value.
+%     CON: Seems feasible.
+%       case 0:
+%         out = 0;
+%       case 1:
+%         out = struct('t', time_.epochUnix, 'f', freq_tnr, 'p',vp.^10);
+%         out.p_label={'dB'};
+%
+%   PROPOSAL: Only call solo.read_TNR() via dependency injection.
+%     CON: Overkill.
+%   PROPOSAL: Submit the return value of solo.read_TNR() as argument instead of
+%             calling it.
 
 
 

mission/solar_orbiter/+solo/+qli/quicklooks_main.m

@@ -5,17 +5,20 @@
 % Intended for batch processing, e.g. being called from bash script, e.g. cron
 % job via a MATLAB wrapper script.
 %
-% NOTE: Requires solo.db_init() to have been properly used to initialize dataset
-%       lookup.
-% NOTE: Uses SPICE implicitly, and therefore relies on some path convention. Not
-%       sure which, but presumably it does at least find /data/solo/SPICE/.
-% NOTE: Uses solo.read_TNR() indirectly which in turns relies on a hardcoded
-%       path to "/data/solo/remote/data/L2/thr/" and selected subdirectories.
-% NOTE: Creates subdirectories to the output directory if not pre-existing.
-% NOTE: 7-day plots will only cover that largest sub-time interval that is
-%       composed of full 7-day periods. Note: 7-day periods begin with a
-%       hardcoded weekday (Wednesday as of 2023-05-09).
-% NOTE: Overwrites pre-existing plot files without warning.
+%
+% NOTES
+% =====
+% * Requires solo.db_init() to have been properly used to initialize dataset
+%   lookup.
+% * Uses SPICE implicitly, and therefore relies on some path convention. Not
+%   sure which, but presumably it does at least find /data/solo/SPICE/.
+% * Uses solo.read_TNR() indirectly which in turns relies on a hardcoded
+%   path to "/data/solo/remote/data/L2/thr/" and selected subdirectories.
+% * Creates subdirectories to the output directory if not pre-existing.
+% * 7-day plots will only cover that largest sub-time interval that is
+%   composed of full 7-day periods. Note: 7-day periods begin with a
+%   hardcoded weekday (Wednesday as of 2023-07-24).
+% * Overwrites pre-existing plot files without warning.
 %
 %
 % ARGUMENTS
@@ -34,8 +37,11 @@
 %       Whether to run the resp. groups of plots.
 %       NOTE: Permits chars "0" and "1" for when calling from bash.
 %       Useful for testing and not re-running unnecessary time-consuming plots.
-% utcBegin, utcEnd : Strings.
-%       Defines time interval for which quicklooks should be generated.
+% utcBegin, utcEnd
+%       Strings. Defines time interval for which quicklooks should be generated.
+%       NOTE: Weekly plots will only be produced for those weeks which are
+%       contained entirely inside the specified time interval (?). Weekly plots
+%       always (as of 2023-07-24) begin on a Wednesday(!) at 00:00:00.
 %
 %
 % Initially created ~<2021-03-11, based on code by Konrad Steinvall, IRF,
@@ -105,21 +111,25 @@ function quicklooks_main(...
 % 2022-03-12T07:22:03.090373000Z -- 2022-03-13T00
 %   Missing data.Tpas
 
+% ##############################################################################
+
 
 %============
 % ~Constants
 %============
 % IMPLEMENTATION NOTE: Disabling B (use empty; pretend there is no B data)
 % speeds up solo.qli.quicklooks_24_6_2_h() greatly. Useful for some debugging.
 % Should be enabled by default.
-ENABLE_B                      = 1;
-% Whether to catch plotting exceptions (and continue) in order to produce as
-% many plots as possible. Should be enabled by default.
-CATCH_PLOT_EXCEPTIONS_ENABLED = 1;
+ENABLE_B                      = 1;    % 0 or 1.
+% Whether to catch plotting exceptions, continue plotting other days/weeks, and
+% then re-raise the last caught exception at the very end. This produces as many
+% plots as possible when one or some plots fail.
+% Should be enabled by default.
+CATCH_PLOT_EXCEPTIONS_ENABLED = 1;    % 0 or 1.
 
 
 
-% NOTE: Usually found on solo/data_yuri.
+% NOTE: Usually found at /data/solo/data_yuri/.
 VHT_1H_DATA_FILENAME = 'V_RPW_1h.mat';
 VHT_6H_DATA_FILENAME = 'V_RPW.mat';
 
@@ -277,7 +287,7 @@ function log_plot_function_time_interval(Tint)
 function handle_plot_exception(catchExceptionEnabled, Exc)
 if catchExceptionEnabled
   % Print stack trace without rethrowing exception.
-  % One wants that in log.
+  % One wants that in the log.
   % NOTE: fprintf(FID=2) => stderr
   fprintf(2, 'Caught plotting error without rethrowing it.\n')
   fprintf(2, 'Plot error/exception: "%s"\n', Exc.message)
@@ -482,13 +492,12 @@ function quicklooks_7days_local(Tint, vht6h, Paths, logoPath)
 
 
 
-% Wrapper around solo.db_get_ts() that normalizes the output to a TSeries.
+% Wrapper around solo.db_get_ts() which normalizes the output to always return
+% one TSeries object.
 %
-% NOTE: solo.db_get_ts() has been observed to return cell array of TSeries
-% (instead of TSeries) for Npas, Tpas and Vpas for
-% solo.qli.quicklooks_main('2020-10-21T00:00:00', '2020-10-28T00:00:00', '/data/solo/data_yuri', ...)
-% There might be other cases but those are as of yet unknown.
-% /Erik P G Johansson 2021-03-22
+% NOTE: solo.db_get_ts() returns a cell array of TSeries instead of a single
+% TSeries when the underlying code thinks that the underlying CDFs do not have
+% consistent metadata. See solo.db_get_ts().
 %
 function Ts = db_get_ts(varargin)
 
@@ -504,7 +513,7 @@ function quicklooks_7days_local(Tint, vht6h, Paths, logoPath)
 
 
 
-% Takes a cell-array of TSeries and merges them to one TSeries.
+% Take a cell array of TSeries and merges them into one TSeries.
 function OutputTs = cell_array_TS_to_TS(InputTs)
 assert(iscell(InputTs))