SolO QLI: Cleanup

mission/solar_orbiter/+solo/+qli/+batch/README.TXT

@@ -5,6 +5,9 @@ The code is designed to generic in the sense that it can be reused for
 any system. In practice, it is primarily intended for official generation of
 quicklooks.
 
+See irfu-matlab/mission/solar_orbiter/+solo/+qli/README.TXT for naming
+conventions.
+
 
 ===================================================
 How to do batch generation of quicklooks in general

mission/solar_orbiter/+solo/+qli/+batch/extract_dataset_dates_from_logs.m

@@ -16,8 +16,8 @@
 %       NOTE: Must match at least one file (~failsafe).
 % dsiCa
 %       Cell array of dataset IDs for datasets which should be searched for.
-%       NOTE: This excludes any "-cdag" suffix. The code will match both CDAG
-%       and non-CDAG files.
+%       NOTE: This excludes any "-cdag" suffix. The function will match both
+%       CDAG and non-CDAG files.
 %
 %
 % RETURN VALUES
@@ -39,7 +39,7 @@
 % PROPOSAL: Specify filename patterns (not dataset IDs).
 % PROPOSAL: Require at least one dataset ID.
 
-% FSOI File System Object Info
+% FSOI = File System Object Info
 FsoiArray = dir(logFileDirPattern);
 FsoiArray = FsoiArray(~[FsoiArray.isdir]);
 
@@ -57,9 +57,9 @@
 % -----------------------------------------------------------------
 % PROBLEM: Not obvious how to select log file when there are multiple
 % simultaneous log files being built on simultaneously (most relevant when
-% running multiple batch BICAS processing runs). Can use last file name (last in
-% alphabetic order) or last file modification timestamp, but neither truly
-% solves the problem alone.
+% running multiple batch BICAS processing runs). Can use the last file name
+% (last in alphabetic order) or last file modification timestamp, but neither
+% truly solves the problem alone.
 
 % IMPLEMENTATION NOTE: Could use file modification date, but this is bad when
 % running multiple processes simultaneously since that means multiple files are
@@ -70,8 +70,8 @@
 % latter's log file is still continuously updated and may have a later file
 % modification date.
 
-%[~, iSort] = sort([FsioArray.datenum], 'ascend');   % Sort by file modification date.
-[~, iSort]  = sort({FsoiArray.name});   % Sort by filename (not entire path).
+%[~, iSort] = sort([FsioArray.datenum], 'ascend');   % Sort by log file modification date.
+[~, iSort]  = sort({FsoiArray.name});   % Sort by log filename (not entire path).
 Fsoi        = FsoiArray(iSort(end));
 logFilePath = fullfile(Fsoi.folder, Fsoi.name);
 
@@ -87,9 +87,10 @@
   % covering multiple datasets/filenames.
   % Ex: Over multiple rows. ==> Exclude line feed in filename.
   % Ex: On the same row.    ==> Exclude period in filename (except before file suffix).
-  % NOTE: solo.adm.parse_dataset_filename()'s "unofficial" basename extension
-  % can cause problems if there are multiple dataset filenames on the same row
-  % and one does not exclude e.g. period.
+  %
+  % NOTE: solo.adm.parse_dataset_filename()'s support for the unofficial
+  % basename extension can cause problems if there are multiple dataset
+  % filenames on the same row and one does not exclude e.g. period.
   % NOTE: Must permit filenames with and without "-cdag".
   pattern = sprintf('%s(|-cdag)_[^\\n.]*\\.cdf', dsi);
 

mission/solar_orbiter/+solo/+qli/+batch/generate_quicklooks.m

@@ -237,17 +237,20 @@ function generate_quicklooks(...
 
 
 % Log arguments
-irf.log('n', sprintf('irfLogoPath                 = "%s"', irfLogoPath))
-irf.log('n', sprintf('vhtDataDir                  = "%s"', vhtDataDir))
-irf.log('n', sprintf('outputDir                   = "%s"', outputDir))
-irf.log('n', sprintf('generateNonweeklyQuicklooks = %d',   generateNonweeklyQuicklooks))
-irf.log('n', sprintf('generateWeeklyQuicklooks    = %d',   generateWeeklyQuicklooks))
-irf.log('n', sprintf('numel(DaysDtArray)          = %d',   numel(DaysDtArray)))
+irf.log('n', sprintf('irfLogoPath                   = "%s"', irfLogoPath))
+irf.log('n', sprintf('vhtDataDir                    = "%s"', vhtDataDir))
+irf.log('n', sprintf('outputDir                     = "%s"', outputDir))
+irf.log('n', sprintf('generateNonweeklyQuicklooks   = %d',   generateNonweeklyQuicklooks))
+irf.log('n', sprintf('generateWeeklyQuicklooks      = %d',   generateWeeklyQuicklooks))
+irf.log('n', sprintf('numel(DaysDtArray)            = %d',   numel(DaysDtArray)))
 % Log misc. variables
-irf.log('n', sprintf('isOfficialGeneration        = %d',   isOfficialGeneration))
+irf.log('n', sprintf('isOfficialGeneration          = %d',   isOfficialGeneration))
 % Log selected constants.
-irf.log('n', sprintf('B_SPECTRA_ENABLED           = %d',   solo.qli.const.B_SPECTRA_ENABLED))
-irf.log('n', sprintf('NONWEEKLY_ALL_PLOTS_ENABLED = %d',   solo.qli.const.NONWEEKLY_ALL_PLOTS_ENABLED))
+irf.log('n', sprintf('B_SPECTRA_ENABLED             = %d',   solo.qli.const.B_SPECTRA_ENABLED))
+irf.log('n', sprintf('NONWEEKLY_ALL_PLOTS_ENABLED   = %d',   solo.qli.const.NONWEEKLY_ALL_PLOTS_ENABLED))
+irf.log('n', sprintf('NONWEEKLY_6H_2H_PLOTS_ENABLED = %d',   solo.qli.const.NONWEEKLY_6H_2H_PLOTS_ENABLED))
+% Log current working directory so that relative paths can be interpreted.
+irf.log('n', sprintf('Current working directory     = %s',   pwd))
 
 
 

mission/solar_orbiter/+solo/+qli/README.TXT

@@ -47,3 +47,5 @@ GQL
     solo.qli.batch.GenerateQuicklooksAbstract and subclasses thereof.
 OFFGEN, "official generation"
     Official Generation of quicklooks for the homepage.
+DSI
+    Dataset ID

mission/solar_orbiter/+solo/+qli/const.m

@@ -30,6 +30,11 @@
   %#####################
   %#####################
   properties(Constant)
+
+    %========================================
+    % Enable/disable functionality for speed
+    %========================================
+
     % Whether to enable two time-consuming spectra based on B (24h6h2h)
     % -----------------------------------------------------------------
     % IMPLEMENTATION NOTE: Disabling these speeds up
@@ -55,6 +60,10 @@
 
 
 
+    %===============
+    % Miscellaneous
+    %===============
+
     % Whether to catch plotting exceptions, continue plotting other days/weeks,
     % and then re-raise the last caught exception at the very end. This produces
     % as many quicklooks as possible when one or some quicklooks fail. Should be
@@ -85,7 +94,7 @@
     OFFICIAL_GENERATION_AUTOMOUNT_DIR = '/data/solo/';
 
     % Constant which is useful to have for defining tests.
-    EMPTY_DT_ARRAY = datetime(cell(0,1), 'TimeZone', 'UTCLeapSeconds')
+    EMPTY_DT_ARRAY = NaT([0, 1], 'TimeZone', 'UTCLeapSeconds');
   end
 
 

mission/solar_orbiter/+solo/+qli/generate_quicklooks_24h_6h_2h.m

@@ -64,17 +64,8 @@ function generate_quicklooks_24h_6h_2h(Data, OutputPaths, Tint24h, logoPath)
 % Should be solved.
 % /Erik P G Johansson 2024-03-25
 %
-% BUG?: 24h, panel 10, TNR spectrum (irf_spectrogram()):
-% Spectral data (irfu-matlab version 2024-03-22) freuently looks different
-% compared to earlier spectras (cron job, irfu-matlab version circa 2023-04).
-% More sections with constant spectrum, some new rectangular "holes" (white,
-% unfilled) in spectra. Some sections are unaltered. Unclear if this si due to
-% changes in underlying data or in code (e.g. recent change in solo.read_TNR()).
-% Ex: 24h quicklook for 2023-01-27:
-%   Large part of spectrum has changed from time changing to constant.
-%   New "hole" in spectrum.
-%   Last section is the same as before.
-% /Erik P G Johansson 2024-03-25
+% https://github.com/irfu/irfu-matlab/issues/138
+% /Erik P G Johansson 2024-03-28
 %
 % BUG: There is no date label for the data under the lowest panel, if there is
 % no data. Do not think this was a problem previously(?).

mission/solar_orbiter/+solo/+qli/utils.m

@@ -25,6 +25,7 @@
 
     % Assert that datetime object only contains timestamps which refer to
     % midnight.
+    %
     % NOTE: Does not require scalar object.
     function assert_UTC_midnight_datetime(Dt)
       assert(isa(Dt, 'datetime'))
@@ -169,6 +170,7 @@ function assert_UTC_midnight_datetime(Dt)
     %       [], if no matching dataset was found.
     %
     function zvData = read_constant_metadata(filePrefix, zvName, Tint)
+      % PROPOSAL: Refactor to use dataobj().
 
       FileArray = solo.db_list_files(filePrefix, Tint);
       if ~isempty(FileArray)
@@ -177,7 +179,7 @@ function assert_UTC_midnight_datetime(Dt)
 
         % NOTE: Reads CDFs using cdfread() which is a MATLAB function (i.e. not
         %       dataobj(), not NASA SPDF). *Might* be faster (or might not)
-        %       since only reading a specified zVariable.
+        %       since only reading one specific zVariable.
         zvCa   = cdfread(filePath, 'variables', zvName);
         zvData = zvCa{1};
       else
@@ -187,6 +189,71 @@ function assert_UTC_midnight_datetime(Dt)
 
 
 
+    % EXPERIMENTAL. SOMEWHAT UGLY CIRCUMVENTION. UNCLEAR IF PERMANENT SOLUTION.
+    %
+    % Read selected zVariables from datasets selected using solo.db_list_files()
+    % but not by using solo.db_get_ts().
+    %
+    % Function exists as a workaround to avoid the removal of samples for
+    % identical timestamps in solo.db_get_ts() which has an unduly large effect
+    % on QLI (panel 10, 24h6h2h), unknown why.
+    %
+    % 2024-03-28: It seems likely that this code can be deleted, but it not
+    % clear yet. Function was created as a possible workaround/bugfix for
+    % https://github.com/irfu/irfu-matlab/issues/138 .
+    %
+    %
+    % RETURN VALUE
+    % ============
+    % zvCa
+    %     Column cell array of TSeries.
+    %     [], if there are no datasets for the selected time interval.
+    %
+    % NOTE: Overlaps with solo.qli.utils.read_constant_metadata().
+%     function zvCa = read_CDFs(filePrefix, Tint, zvNamesCa)
+%       % PROPOSAL: Merge with solo.qli.utils.read_constant_metadata().
+%       %   PRO: Overlaps with it. If calling both for the same dataset(s), then
+%       %        reading the same file(s) twice.
+%       %     CON: Does not whether cdfread() reads entire files or just part of them.
+%
+%       % IMPLEMENTATION NOTE: Not storing return TSeries values in TSeries array
+%       % since indexing () has special meaning for TSeries.
+%
+%
+%       FileArray = solo.db_list_files(filePrefix, Tint);
+%
+%       zvCa = cell(numel(zvNamesCa), 1);
+%       if ~isempty(FileArray)
+%         for iFile = 1:numel(FileArray)
+%           File     = FileArray(iFile);
+%           filePath = fullfile(File.path, File.name);
+%           Do       = dataobj(filePath);
+%
+%           for iZv = 1:numel(zvNamesCa)
+%             Ts = get_ts(Do, zvNamesCa{iZv});
+%
+%             TsOld = zvCa{iZv, 1};
+%             if ~isequal(TsOld, [])
+%               Ts = TsOld.combine(Ts);
+%             end
+%             zvCa{iZv} = Ts;
+%           end
+%         end
+%         zvCa{1} = zvCa{1};
+%       else
+%         % CASE: There are no datasets to read
+%         % IMPLEMENTATION NOTE: Can not return e.g. empty TSeries with consistent
+%         % sizes since the sizes can not be known without zVariables.
+%         zvCa = [];
+%       end
+%
+%       for iZv = 1:numel(zvNamesCa)
+%         zvCa{iZv} = zvCa{iZv}.tlim(Tint);
+%       end
+%     end
+
+
+
     % Wrapper around solo.db_get_ts() which normalizes the output to always
     % return one TSeries object, or [].
     %
@@ -361,7 +428,8 @@ function assert_UTC_midnight_datetime(Dt)
     %
     % Author: Erik P G Johansson, IRF, Uppsala, Sweden
     %
-    function [soloStr, earthStr] = get_context_info_strings(soloPosTSeries, earthPosTSeries, Tint)
+    function [soloStr, earthStr] = get_context_info_strings(...
+        soloPosTSeries, earthPosTSeries, Tint)
       % PROPOSAL: No Tint argument. Caller submits already truncated TSeries.
       %   PRO: One fewer arguments.
       %   CON: Caller has to truncate twice.
@@ -507,7 +575,8 @@ function set_YLim_automatically(hAxesArray)
 
 
 
-    function filename = get_plot_filename(Tint)
+    % Ex: 20240313T20_20240313T22.png
+    function filename = create_quicklook_filename(Tint)
       assert(isa(Tint, 'EpochTT') && (length(Tint) == 2))
 
       ett1           = Tint(1);
@@ -528,7 +597,7 @@ function set_YLim_automatically(hAxesArray)
     function save_figure_to_file(parentDirPath, Tint)
       % PROPOSAL: Include fig.PaperPositionMode='auto';
 
-      filename = solo.qli.utils.get_plot_filename(Tint);
+      filename = solo.qli.utils.create_quicklook_filename(Tint);
       filePath = fullfile(parentDirPath, filename);
       print('-dpng', filePath);
     end

mission/solar_orbiter/+solo/+qli/utils___UTEST.m

@@ -186,7 +186,7 @@ function test_get_data_source_info_string(testCase)
 
     function test_get_context_info_strings(testCase)
 
-      % Arbitrary number output variables.
+      % Arbitrary number of output variables.
       function test(...
           soloPosTSeries, earthPosTSeries, Tint, ...
           expSoloStr, expEarthStr)
@@ -234,11 +234,10 @@ function test(...
 
 
 
-    function test_get_plot_filename(testCase)
-
-      function test(inputsCa, expOutput)
-        actOutput = solo.qli.utils.get_plot_filename(inputsCa{:});
-        testCase.verifyEqual(actOutput, expOutput)
+    function test_create_quicklook_filename(testCase)
+      function test(Tint, expFilename)
+        actFilename = solo.qli.utils.create_quicklook_filename(Tint);
+        testCase.assertEqual(actFilename, expFilename)
       end
 
       %===================================================================
@@ -249,7 +248,7 @@ function test(inputsCa, expOutput)
         '2024-01-11T04:04:09.900000004Z'; ...
         ] ...
         );
-      test({Tint}, '20240110T02_20240111T04.png')
+      test(Tint, '20240110T02_20240111T04.png')
     end