Merge pull request #1239 from wadpac/issue967_selfreport

Add new improved visual report
wadpac · Dec 11, 2024 · ca82700 · ca82700
2 parents a06d0d2 + f2cd5d6
commit ca82700
Show file tree

Hide file tree

Showing 35 changed files with 1,965 additions and 497 deletions.
diff --git a/NAMESPACE b/NAMESPACE
@@ -42,14 +42,13 @@ export(g.analyse, g.calibrate,
        g.part5_analyseSegment, g.part5_initialise_ts,
        g.part5.analyseRest, part6AlignIndividuals,
        part6PairwiseAggregation, g.part6, g.report.part6, 
-       check_log, g.report.part5_dictionary, DFA, ABI, SSP)
-
+       check_log, g.report.part5_dictionary, DFA, ABI, SSP, visualReport)
 
 importFrom("grDevices", "colors", "dev.off", "pdf",
-           "rgb", "rainbow", "palette")
+           "rgb", "rainbow", "palette", "adjustcolor", "gray.colors")
 importFrom("graphics", "abline", "axis", "par", "plot", "plot.new",
            "rect","axis.POSIXct", "barplot", "box", "legend",
-           "lines", "mtext", "text", "arrows", "segments")
+           "lines", "mtext", "text", "arrows", "segments", "layout")
 importFrom("utils", "read.csv", "sessionInfo", "write.csv",
            "installed.packages", "setTxtProgressBar",
            "txtProgressBar", "write.table", "head",

diff --git a/NEWS.md b/NEWS.md
@@ -1,7 +1,21 @@
 # CHANGES IN GGIR VERSION 3.1-?
 
+- Part 4: Parameter sib_must_fully_overlap_with_TimeInBed added to control whether sib should overlap fully with the start and/or end of time in bed to be considered sleep (default TRUE),
+this is consistent with functionality in the past. #1223
+
+- Part 5: Expand functionality for exploring possibility of nap detection, this includes the addition of new parameter possible_nap_gap.
+
+- Visual report: Added new visualreport that is automatically generated when visualreport = TRUE and intended to eventually replace the problematic legacy report. Add parameter old_visualreport to turn off the old visualreport generation. #1173
+
+- Part 4 + 5 + visualreport: If sleeplog has both reported sleeplog and time in bed process both. In part 4 we will still use one of the two but in the visualreport we display both. #967
+
 - Part 2: Add parameters nonwearFiltermaxHours and nonwearFilterWindow to give user the option to filter short lasting nighttime nonwear #1218.
 
+- Part 5 and 6: part6_threshold_combi when not specified now defaults to first threshold as
+specified for light, moderate and vigorous intensity respectively.
+
+- Part 4: Handle unexpected combinations of sleep diary column names. For example, if only inbed and wakeup are available then treat these as the time in bed period, while if only sleeponset and outbed are available then these are treated as SPT window.
+
 # CHANGES IN GGIR VERSION 3.1-7
 
 - Part 3: Improved handling of DST, #1225

diff --git a/R/GGIR.R b/R/GGIR.R
@@ -380,10 +380,8 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
       if (N.files.ms4.out < f1) f1 = N.files.ms4.out
       if (f1 == 0) f1 = N.files.ms4.out
       g.report.part4(datadir = datadir, metadatadir = metadatadir, f0 = f0, f1 = f1,
-                     loglocation = params_sleep[["loglocation"]],
                      data_cleaning_file = params_cleaning[["data_cleaning_file"]],
-                     sleepwindowType = params_sleep[["sleepwindowType"]],
-                     params_output = params_output,
+                     params_sleep = params_sleep, params_output = params_output,
                      verbose = verbose)
     }
   }
@@ -424,20 +422,32 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
 
     if (length(files.available) > 0) {
       if (verbose == TRUE) print_console_header("Generate visual reports")
-      g.plot5(metadatadir = metadatadir,
-              dofirstpage = params_output[["dofirstpage"]],
-              viewingwindow = params_output[["viewingwindow"]],
-              f0 = f0, f1 = f1,
-              overwrite = params_general[["overwrite"]],
-              metric = params_general[["acc.metric"]],
-              desiredtz = params_general[["desiredtz"]],
-              threshold.lig = params_phyact[["threshold.lig"]],
-              threshold.mod = params_phyact[["threshold.mod"]],
-              threshold.vig = params_phyact[["threshold.vig"]],
-              visualreport_without_invalid = params_output[["visualreport_without_invalid"]],
-              includedaycrit = params_cleaning[["includedaycrit"]][1],
-              includenightcrit = params_cleaning[["includenightcrit"]],
-              verbose = TRUE)
+      # The new visual report
+      visualReport(metadatadir = metadatadir,
+                   f0 = f0, f1 = f1,
+                   overwrite = params_general[["overwrite"]],
+                   desiredtz = params_general[["desiredtz"]],
+                   verbose = TRUE,
+                   part6_threshold_combi = params_phyact[["part6_threshold_combi"]],
+                   GGIRversion = GGIRversion,
+                   params_sleep = params_sleep,
+                   params_output = params_output)
+      if (params_output[["old_visualreport"]] == TRUE) {
+        g.plot5(metadatadir = metadatadir,
+                dofirstpage = params_output[["dofirstpage"]],
+                viewingwindow = params_output[["viewingwindow"]],
+                f0 = f0, f1 = f1,
+                overwrite = params_general[["overwrite"]],
+                metric = params_general[["acc.metric"]],
+                desiredtz = params_general[["desiredtz"]],
+                threshold.lig = params_phyact[["threshold.lig"]],
+                threshold.mod = params_phyact[["threshold.mod"]],
+                threshold.vig = params_phyact[["threshold.vig"]],
+                visualreport_without_invalid = params_output[["visualreport_without_invalid"]],
+                includedaycrit = params_cleaning[["includedaycrit"]][1],
+                includenightcrit = params_cleaning[["includenightcrit"]],
+                verbose = TRUE)
+      }
     }
   }
 }
diff --git a/R/check_params.R b/R/check_params.R
@@ -32,9 +32,10 @@ check_params = function(params_sleep = c(), params_metrics = c(),
     numeric_params = c("anglethreshold", "timethreshold", "longitudinal_axis", 
                        "possible_nap_window", "possible_nap_dur",
                        "colid", "coln1", "def.noc.sleep", "nnights", 
-                       "sleepefficiency.metric", "possible_nap_edge_acc", "HDCZA_threshold")
+                       "sleepefficiency.metric", "possible_nap_edge_acc", "HDCZA_threshold",
+                       "possible_nap_gap")
     boolean_params = c("ignorenonwear", "HASPT.ignore.invalid",
-                       "relyonguider", "sleeplogidnum")
+                       "relyonguider", "sleeplogidnum", "sib_must_fully_overlap_with_TimeInBed")
     character_params = c("HASPT.algo", "HASIB.algo", "Sadeh_axis", "nap_model",
                          "sleeplogsep", "sleepwindowType", "loglocation")
     check_class("Sleep", params = params_sleep, parnames = numeric_params, parclass = "numeric")
@@ -83,7 +84,7 @@ check_params = function(params_sleep = c(), params_metrics = c(),
     numeric_params = c("qlevels", "ilevels", "IVIS_windowsize_minutes", "IVIS_epochsize_seconds",
                        "IVIS.activity.metric", "IVIS_acc_threshold",
                        "qM5L5", "MX.ig.min.dur", "M5L5res", "winhr", "LUXthresholds", "LUX_cal_constant",
-                       "LUX_cal_exponent", "LUX_day_segments", "L5M5window")
+                       "LUX_cal_exponent", "LUX_day_segments", "L5M5window", "clevels")
     boolean_params = c("cosinor", "part6CR", "part6HCA", "part6DFA")
     character_params = c("qwindow_dateformat", "part6Window")
     check_class("247", params = params_247, parnames = numeric_params, parclass = "numeric")
@@ -106,6 +107,7 @@ check_params = function(params_sleep = c(), params_metrics = c(),
                        "hrs.del.end", "includedaycrit.part5", "minimum_MM_length.part5",
                        "includenightcrit", "max_calendar_days", "includecrit.part6", "includenightcrit.part5",
                        "nonwearFiltermaxHours", "nonwearFilterWindow")
+
     boolean_params = c("excludefirstlast.part5", "do.imp", "excludefirstlast",
                        "excludefirst.part4", "excludelast.part4", "nonWearEdgeCorrection")
     character_params = c("data_cleaning_file", "TimeSegments2ZeroFile")
@@ -114,13 +116,15 @@ check_params = function(params_sleep = c(), params_metrics = c(),
     check_class("cleaning", params = params_cleaning, parnames = character_params, parclass = "character")
   }
   if (length(params_output) > 0) {
-    numeric_params = c("viewingwindow", "criterror")
+    numeric_params = c("viewingwindow", "criterror", "visualreport_hrsPerRow",
+                       "visualreport_validcrit")
     boolean_params = c("epochvalues2csv", "save_ms5rawlevels", "save_ms5raw_without_invalid",
                        "storefolderstructure", "dofirstpage", "visualreport", "week_weekend_aggregate.part5",
                        "do.part3.pdf", "outliers.only", "do.visual", "do.sibreport", "visualreport_without_invalid",
-                       "do.part2.pdf", "require_complete_lastnight_part5")
+                       "do.part2.pdf", "old_visualreport", "require_complete_lastnight_part5")
+
     character_params = c("save_ms5raw_format", "timewindow", "sep_reports", "sep_config",
-                         "dec_reports", "dec_config")
+                         "dec_reports", "dec_config", "visualreport_focus", "method_research_vars")
     check_class("output", params = params_output, parnames = numeric_params, parclass = "numeric")
     check_class("output", params = params_output, parnames = boolean_params, parclass = "boolean")
     check_class("output", params = params_output, parnames = character_params, parclass = "character")
@@ -187,6 +191,27 @@ check_params = function(params_sleep = c(), params_metrics = c(),
     } else if (length(params_sleep[["def.noc.sleep"]]) == 2) {
       params_sleep[["HASPT.algo"]] = "notused"
     }
+    if (length(params_sleep[["possible_nap_gap"]]) != 1) {
+      stop(paste0("Parameter possible_nap_gap has length ", length(params_sleep[["possible_nap_gap"]]), 
+                  " while length 1 is expected"), call. = FALSE)
+    }
+    if (!is.null(params_sleep[["possible_nap_window"]]) &&
+        length(params_sleep[["possible_nap_window"]]) != 2) {
+      stop(paste0("Parameter possible_nap_window has length ", length(params_sleep[["possible_nap_window"]]), 
+                  " while length 2 is expected"), call. = FALSE)
+    }
+    if (!is.null(params_sleep[["possible_nap_dur"]]) &&
+        length(params_sleep[["possible_nap_dur"]]) != 2) {
+      stop(paste0("Parameter possible_nap_dur has length ", length(params_sleep[["possible_nap_dur"]]), 
+                  " while length 2 is expected"), call. = FALSE)
+    }
+    if (!is.null(params_sleep[["possible_nap_window"]]) &&
+        !is.null(params_sleep[["possible_nap_dur"]])) {
+      params_output[["do.sibreport"]] = TRUE
+      params_output[["save_ms5raw_format"]] = unique(c(params_output[["save_ms5raw_format"]], "RData"))
+      params_output[["save_ms5rawlevels"]] = TRUE
+      params_output[["save_ms5raw_without_invalid"]] = FALSE
+    }
   }
 
   if (length(params_metrics) > 0 & length(params_sleep) > 0) {
@@ -240,8 +265,11 @@ check_params = function(params_sleep = c(), params_metrics = c(),
       params_sleep[["sleepwindowType"]] = "TimeInBed"
     }
 
-    if (length(params_sleep[["loglocation"]]) == 0 & params_sleep[["HASPT.algo"]][1] != "HorAngle" & params_sleep[["sleepwindowType"]] != "SPT") {
-      warning("\nAuto-updating sleepwindowType to SPT because no sleeplog used and neither HASPT.algo HorAngle used.", call. = FALSE)
+    if (length(params_sleep[["loglocation"]]) == 0 &
+        params_sleep[["HASPT.algo"]][1] != "HorAngle" &
+        params_sleep[["HASPT.algo"]][1] != "NotWorn" &
+        params_sleep[["sleepwindowType"]] != "SPT") {
+      warning("\nAuto-updating sleepwindowType to SPT because no sleeplog used and neither HASPT.algo HorAngle or NotWorn used.", call. = FALSE)
       params_sleep[["sleepwindowType"]] = "SPT"
     }
   }
@@ -327,9 +355,9 @@ check_params = function(params_sleep = c(), params_metrics = c(),
       params_phyact[["mvpadur"]] = c(1,5,10)
       warning("\nmvpadur needs to be a vector with length three, value now reset to default c(1, 5, 10)", call. = FALSE)
     }
-    if (length(params_phyact[["threshold.lig"]]) == 1 &&
+    if ((length(params_phyact[["threshold.lig"]]) == 1 &&
         length(params_phyact[["threshold.mod"]]) == 1 &&
-        length(params_phyact[["threshold.vig"]]) == 1) {
+        length(params_phyact[["threshold.vig"]]) == 1) | is.null(params_phyact[["part6_threshold_combi"]])) {
       params_phyact[["part6_threshold_combi"]] = paste(params_phyact[["threshold.lig"]],
                                                        params_phyact[["threshold.mod"]],
                                                        params_phyact[["threshold.vig"]], sep = "_")
@@ -353,6 +381,10 @@ check_params = function(params_sleep = c(), params_metrics = c(),
       stop(paste0("\nYou have set sep_config and dec_config both to ",
                   params_output[["sep_config"]], " this is ambiguous. Please fix."))
     }
+    if (params_output[["visualreport_hrsPerRow"]] < 24 ||
+        params_output[["visualreport_hrsPerRow"]] > 48) {
+      stop("Parameter visualreport_hrsPerRow is expected to be set in the range 24-48")
+    }
   }
   # params 247
   if (length(params_247) > 0) {
@@ -383,6 +415,9 @@ check_params = function(params_sleep = c(), params_metrics = c(),
       params_output[["save_ms5rawlevels"]] = TRUE
       params_output[["save_ms5raw_without_invalid"]] = FALSE
     }
+    if (length(params_247[["clevels"]]) == 1) {
+      warning("\nParameter clevels expects a number vector of at least 2 values, current length is 1", call. = FALSE)
+    }
   }
   if (!is.null(params_general[["expand_tail_max_hours"]])) {
     if (is.null(params_general[["recordingEndSleepHour"]]) & params_general[["expand_tail_max_hours"]] != 0) {

diff --git a/R/create_test_sleeplog_csv.R b/R/create_test_sleeplog_csv.R
@@ -2,7 +2,8 @@ create_test_sleeplog_csv = function(Nnights = 7,
                                     storagelocation = c(),
                                     advanced = FALSE,
                                     sep = ",",
-                                    begin_date = "2016/06/25") {
+                                    begin_date = "2016/06/25",
+                                    type = "sleeplog") {
 
   # function to create a test sleeplog file needed for testing GGIR
   # Nnights is the number of nights the sleeplog has
@@ -28,14 +29,22 @@ create_test_sleeplog_csv = function(Nnights = 7,
     dates = seq(as.Date(begin_date), as.Date(begin_date) + Nnights, by = 1)
     Cnames = c(
       "_wakeup",
-      "_inbed",
+      "_onset",
       "_nap1_start",
       "_nap1_end",
       "_nap2_start",
       "_nap2_end",
       "_nonwear1_off",
       "_nonwear1_on"
     )
+    if (type == "bedlog") {
+      Cnames[1:2] = c("_bedstart", "_bedend")
+    }
+    if (type == "both") {
+      Cnames = c(Cnames[1:2], c("_bedend", "_bedstart"), Cnames[3:8])
+      bedtimes = c("08:00:00", "22:00:00")
+      times = c(times[1:2], bedtimes, times[3:8])
+    }
     names = "ID"
     log = c()
     for (di in 1:length(dates)) {

diff --git a/R/filterNonwearNight.R b/R/filterNonwearNight.R
@@ -60,7 +60,7 @@ filterNonwearNight = function(r1, metalong, qwindowImp, desiredtz,
           # convert to continuous scale to ease finding start and end
           below18 = which(qwindow_temp < 18)
           if (length(below18) > 0) {
-            qwindow_temp = qwindow_temp[below18] + 24
+            qwindow_temp[below18] = qwindow_temp[below18] + 24
           }
           start = min(qwindow_temp)
           end = max(qwindow_temp)

diff --git a/R/g.conv.actlog.R b/R/g.conv.actlog.R
@@ -4,7 +4,7 @@ g.conv.actlog = function(qwindow, qwindow_dateformat="%d-%m-%Y", epochSize = 5)
   # local functions:
   time2numeric = function(x) {
     x = unlist(x)
-    x = x[grep(pattern = "impute", x = names(x), invert = TRUE)]
+    x = x[grep(pattern = "impute|uncertain", x = names(x), invert = TRUE)]
     c2t = function(x2) {
       tmp = as.numeric(unlist(strsplit(as.character(x2),":")))
       if (length(tmp) == 2) hourinday = tmp[1] + (tmp[2]/60)