Skip to content

Commit

Permalink
solo.qli.batch: Use abbrev. DFMDD & IDMRQ
Browse files Browse the repository at this point in the history
  • Loading branch information
@ErikPGJ
ErikPGJ committed Apr 11, 2024
1 parent 688a155 commit 629d909
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 55 deletions.
84 changes: 43 additions & 41 deletions mission/solar_orbiter/+solo/+qli/+batch/fmd.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,16 @@
% DFMDD
% Day-to-FMD Dictionary.
% Dictionary with
% * keys = datetime (UTC; midnight) representing days of measured data
% * values = datetime (no timezone) representing relevant FMD (e.g. most
% recent for datasets on the day specified in the key).
% * keys = datetime (UTC; midnight) representing specific days of measured
% data
% * values = datetime (no timezone) representing relevant FMDs (e.g. most
% recent FMDs for all input datasets on the day specified in the
% key).
% IDMRQ = Input Datasets More Recent than QLI
% Algorithm for generating array of dates for which to (optionally) generate
% QLIs. A day is included if (1) the most recent QLI FMD (for that day) is
% more recent than the most recent input dataset (for that day), or (2) there
% is no QLI for that day.
%
%
% Author: Erik P G Johansson, IRF, Uppsala, Sweden
Expand Down Expand Up @@ -98,7 +105,7 @@



function DaysDtArray = get_days_from_FMDs(datasetDirsCa, qliDir, dsiCa)
function DaysDtArray = get_days_from_IDMRQ(datasetDirsCa, qliDir, dsiCa)
assert(iscell(datasetDirsCa) && iscolumn(datasetDirsCa))
assert(ischar(qliDir))

Expand Down Expand Up @@ -126,7 +133,7 @@
% Derive dates
%==============
irf.log('n', 'Determining days for which quicklooks could/should be updated.')
DaysDtArray = solo.qli.batch.fmd.get_days_from_FMDs_from_file_info(...
DaysDtArray = solo.qli.batch.fmd.get_days_from_IDMRQ_from_file_info(...
DsmdArray, DatasetFmdDtArray, dsiCa, QliPathsCa, QliFmdDtArray);
end

Expand All @@ -139,30 +146,30 @@
% file system reading so as to have a function that is nice for automated
% testing.
%
function DaysDtArray = get_days_from_FMDs_from_file_info(...
function DaysDtArray = get_days_from_IDMRQ_from_file_info(...
DsmdArray, DatasetFmdDtArray, dsiCa, QliPathsCa, QliFmdDtArray)

DatasetsFmdDict = solo.qli.batch.fmd.get_dataset_DFMDD_all(...
DatasetsDfmdd = solo.qli.batch.fmd.get_dataset_DFMDD_for_all_DSIs(...
DsmdArray, DatasetFmdDtArray, dsiCa);
QliFmdDict = solo.qli.batch.fmd.get_QLI_DFMDD(...
QliDfmdd = solo.qli.batch.fmd.get_QLI_DFMDD(...
QliPathsCa, QliFmdDtArray);

DaysDtArray = solo.qli.batch.fmd.get_days_from_FMDs_algorithm(...
DatasetsFmdDict, QliFmdDict);
DaysDtArray = solo.qli.batch.fmd.get_days_from_IDMRQ_algorithm(...
DatasetsDfmdd, QliDfmdd);

solo.qli.utils.assert_UTC_midnight_datetime(DaysDtArray)
end



function ChangedDatasetsDtArray = get_days_from_FMDs_algorithm(...
DatasetsFmdDict, QliFmdDict)
function ChangedDatasetsDtArray = get_days_from_IDMRQ_algorithm(...
DatasetsDfmdd, QliDfmdd)

% IMPLEMENTATION NOTE: An empty dictionary can not specify timezone in
% keys/values. Must therefore always normalize to UTC first.
AllDatasetsDtArray = intersect(...
datetime(DatasetsFmdDict.keys, 'TimeZone', 'UTCLeapSeconds'), ...
datetime(QliFmdDict.keys, 'TimeZone', 'UTCLeapSeconds'));
datetime(DatasetsDfmdd.keys, 'TimeZone', 'UTCLeapSeconds'), ...
datetime(QliDfmdd.keys, 'TimeZone', 'UTCLeapSeconds'));

% Preallocate.
ChangedDatasetsDtArray = NaT(...
Expand All @@ -172,7 +179,7 @@
for iDatasetDt = 1:numel(AllDatasetsDtArray)
DatasetDt = AllDatasetsDtArray(iDatasetDt);

if DatasetsFmdDict(DatasetDt) >= QliFmdDict(DatasetDt)
if DatasetsDfmdd(DatasetDt) >= QliDfmdd(DatasetDt)
nChangedDatasets = nChangedDatasets + 1;
ChangedDatasetsDtArray(nChangedDatasets, 1) = DatasetDt;
end
Expand All @@ -183,36 +190,31 @@



% Same concept as solo.qli.batch.fmd.get_dataset_DFMDD_DSI(),
% except that it covers multiple DSIs and only keeps the latest FMD for data
% timestamp (dictionary key) collisions.
function DayFmdDict = get_dataset_DFMDD_all(DsmdArray, FmdDtArray, dsiCa)
DayFmdDictCa = cell(0, 1);
% Given DSMDs and corresponding FMDs, get DFMDD for the most recent dataset
% FMDs for multiple DSIs.
function Dfmdd = get_dataset_DFMDD_for_all_DSIs(DsmdArray, FmdDtArray, dsiCa)
DfmddCa = cell(0, 1);

for iDsi = 1:numel(dsiCa)
dsi = dsiCa{iDsi};
DayFmdDictCa{iDsi, 1} = solo.qli.batch.fmd.get_dataset_DFMDD_DSI(DsmdArray, FmdDtArray, dsi);
DfmddCa{iDsi, 1} = solo.qli.batch.fmd.get_dataset_DFMDD_for_one_DSI(...
DsmdArray, FmdDtArray, dsiCa{iDsi});
end

DayFmdDict = solo.qli.batch.utils.merge_dictionaries_max(DayFmdDictCa, datetime.empty, datetime.empty);
Dfmdd = solo.qli.batch.utils.merge_dictionaries_max(...
DfmddCa, datetime.empty, datetime.empty);
end



% Given DSMDs and FMDs, get dictionary of the latest FMDs for every day for
% a specified DSI.
% Given DSMDs and corresponding FMDs, get DFMDD for the most recent dataset
% FMDs for one specific DSI.
%
% ARGUMENTS
% =========
% FmdDtArray
% Column array of FMDs for every DSMD.
%
% RETURN VALUE
% ============
% DayFmdDict
% Dictionary day-->FMD
%
function DayFmdDict = get_dataset_DFMDD_DSI(DsmdArray, FmdDtArray, dsi)
function Dfmdd = get_dataset_DFMDD_for_one_DSI(DsmdArray, FmdDtArray, dsi)
assert(isa(DsmdArray, 'solo.adm.DSMD'))
assert(isa(FmdDtArray, 'datetime') )
assert(ischar(dsi))
Expand All @@ -224,7 +226,7 @@
DsmdArray = DsmdArray(bKeep);
FmdDtArray = FmdDtArray(bKeep);

DayFmdDict = dictionary(datetime.empty, datetime.empty);
Dfmdd = dictionary(datetime.empty, datetime.empty);

for iDsmd = 1:numel(DsmdArray)
% IMPLEMENTATION NOTE: Handle datasets which cover an arbitrary length
Expand All @@ -237,25 +239,25 @@
DatasetDtArray = DatasetDt1:caldays(1):DatasetDt2;

for iDatasetDt = 1:numel(DatasetDtArray)
DayFmdDict = solo.qli.batch.utils.dictionary_set_value_max(...
DayFmdDict, DatasetDtArray(iDatasetDt), FmdDtArray(iDsmd));
Dfmdd = solo.qli.batch.utils.dictionary_set_value_max(...
Dfmdd, DatasetDtArray(iDatasetDt), FmdDtArray(iDsmd));
end
end
end



% Given FMDs for paths to potential QLI files, get
% dictionary of the latest FMDs for every day.
% Given FMDs for paths to potential QLI files, get DFMDD for the most recent
% QLI FMDs.
%
function DayFmdDict = get_QLI_DFMDD(QliPathsCa, QliFmdDtArray)
assert(iscall(QliPathsCa))
function Dfmdd = get_QLI_DFMDD(QliPathsCa, QliFmdDtArray)
assert(iscell(QliPathsCa))
assert(isa(QliFmdDtArray, 'datetime'))
irf.assert.sizes(...
QliPathsCa, [-1], ...
QliFmdDtArray, [-1])

DayFmdDict = dictionary(datetime.empty, datetime.empty);
Dfmdd = dictionary(datetime.empty, datetime.empty);

for iFile = 1:numel(QliPathsCa)
[FilenameDt1, FilenameDt2] = solo.qli.utils.parse_quicklook_filename(...
Expand All @@ -276,8 +278,8 @@
% (2) the same filename in multiple locations (should not happen), and
% (3) the same path multiple times (should not happen).
for iDay = 1:numel(FilenameDtArray)
DayFmdDict = solo.qli.batch.utils.dictionary_set_value_max(...
DayFmdDict, FilenameDtArray(iDay), QliFmdDtArray(iFile));
Dfmdd = solo.qli.batch.utils.dictionary_set_value_max(...
Dfmdd, FilenameDtArray(iDay), QliFmdDtArray(iFile));
end
end
end
Expand Down
12 changes: 6 additions & 6 deletions mission/solar_orbiter/+solo/+qli/+batch/fmd___UTEST.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@


% Test highest-level function.
function test_get_days_from_FMDs_from_file_info(testCase)
function test_get_days_from_IDMRQ_from_file_info(testCase)

function test(...
datasetPathsCa, datasetsFmdStrCa, dsiCa, ...
Expand All @@ -35,7 +35,7 @@ function test(...
QliFmdDtArray = datetime(qliFmdStrCa);
ExpDaysDtArray = solo.qli.utils.umdt(expDaysStrCa);

ActDaysDtArray = solo.qli.batch.fmd.get_days_from_FMDs_from_file_info(...
ActDaysDtArray = solo.qli.batch.fmd.get_days_from_IDMRQ_from_file_info(...
DsmdArray, DatasetFmdDtArray, dsiCa, ...
qliPathsCa, QliFmdDtArray);

Expand Down Expand Up @@ -129,10 +129,10 @@ function test(...



% NOTE: Not solo.qli.batch.fmd.testing get_dataset_DFMDD_DSI()
% NOTE: Not solo.qli.batch.fmd.testing get_dataset_DFMDD_for_one_DSI()
% which is wrapped by
% solo.qli.batch.fmd.get_dataset_DFMDD_all().
function test_get_dataset_DFMDD_all(testCase)
% solo.qli.batch.fmd.get_dataset_DFMDD_for_all_DSIs().
function test_get_dataset_DFMDD_for_all_DSIs(testCase)

function test(datasetPathsCa, fmdStrCa, dsiCa, expDaysStrCa, expFmdStrCa)
[DsmdArray, ~] = solo.adm.paths_to_DSMD_array(datasetPathsCa);
Expand All @@ -141,7 +141,7 @@ function test(datasetPathsCa, fmdStrCa, dsiCa, expDaysStrCa, expFmdStrCa)
solo.qli.utils.umdt(expDaysStrCa), ...
datetime( expFmdStrCa));

ActDayFmdDict = solo.qli.batch.fmd.get_dataset_DFMDD_all(...
ActDayFmdDict = solo.qli.batch.fmd.get_dataset_DFMDD_for_all_DSIs(...
DsmdArray, FmdDtArray, dsiCa);

testCase.assertEqual(ActDayFmdDict, ExpDayFmdDict)
Expand Down
2 changes: 1 addition & 1 deletion mission/solar_orbiter/+solo/+qli/+batch/generate_quicklooks_interface.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
% See solo.qli.batch.interface.get_days_from_logs().
% .datasetDirsCa
% Cell array of paths to directories with datasets. Used for FMDS.
% See solo.qli.batch.interface.get_days_from_FMDs().
% See solo.qli.batch.interface.get_days_from_IDMRQ().
% .Gql
% See solo.qli.batch.generate_quicklooks().
%
Expand Down
6 changes: 3 additions & 3 deletions mission/solar_orbiter/+solo/+qli/+batch/interface.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
algorithmArgumentsCa{:});

case 'FMDS'
DaysDtArray = solo.qli.batch.interface.get_days_from_FMDs(...
DaysDtArray = solo.qli.batch.interface.get_days_from_IDMRQ(...
Settings.datasetDirsCa, ...
outputDir, ...
algorithmArgumentsCa{:});
Expand Down Expand Up @@ -246,15 +246,15 @@ function check_interface_date_str(dateStr)
%
% Author: Erik P G Johansson, IRF, Uppsala, Sweden
%
function DaysDtArray = get_days_from_FMDs(...
function DaysDtArray = get_days_from_IDMRQ(...
datasetDirsCa, qliDir, maxNDaysStr, beginDayUtcInclStr, endDayUtcExclStr)

solo.qli.batch.interface.check_interface_date_str(beginDayUtcInclStr)
solo.qli.batch.interface.check_interface_date_str(endDayUtcExclStr)

dsiCa = [solo.qli.batch.const.SOURCE_DSI_DICT.values{:}]';

DaysDtArray = solo.qli.batch.fmd.get_days_from_FMDs(...
DaysDtArray = solo.qli.batch.fmd.get_days_from_IDMRQ(...
datasetDirsCa, qliDir, dsiCa);

% Filter list of days.
Expand Down
8 changes: 4 additions & 4 deletions ...ch/interface_get_days_from_FMDs___UTEST.m → ...h/interface_get_days_from_IDMRQ___UTEST.m
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
%
% matlab.unittest automatic test code for
% solo.qli.batch.interface.get_days_from_FMDs().
% solo.qli.batch.interface.get_days_from_IDMRQ().
%
%
% Author: Erik P G Johansson, IRF, Uppsala, Sweden
%
classdef interface_get_days_from_FMDs___UTEST < matlab.unittest.TestCase
classdef interface_get_days_from_IDMRQ___UTEST < matlab.unittest.TestCase



Expand Down Expand Up @@ -55,7 +55,7 @@ function setup(testCase)
function test_empty(testCase)
datasetDirsCa = cell(0, 1);

ActDaysDtArray = solo.qli.batch.interface.get_days_from_FMDs(...
ActDaysDtArray = solo.qli.batch.interface.get_days_from_IDMRQ(...
datasetDirsCa, testCase.fmdQliDir, '999', '2000-01-01', '2099-01-01');

testCase.assertEqual(ActDaysDtArray, solo.qli.const.EMPTY_DT_ARRAY)
Expand Down Expand Up @@ -100,7 +100,7 @@ function test_nonempty(testCase)
%==============
datasetDirsCa = {dir1; dir2};

ActDaysDtArray = solo.qli.batch.interface.get_days_from_FMDs(...
ActDaysDtArray = solo.qli.batch.interface.get_days_from_IDMRQ(...
datasetDirsCa, testCase.fmdQliDir, '9999', '2000-01-01', '2099-01-01');

testCase.assertEqual(ActDaysDtArray, ...
Expand Down

0 comments on commit 629d909

Please sign in to comment.