added second button for downloading final dataset

R/app_server.R

@@ -8,7 +8,7 @@
 # Below increases the max data upload size from the Shiny default of 5MB per file
 # to 30MB for file
 options(shiny.maxRequestSize = 30 * 1024^2)
-
+options(warn = 2)
 app_server <- function(input, output, session) {
   # Your application server logic
   tadat <- shiny::reactiveValues() # create a list object that holds reactive values passed between modules
@@ -53,32 +53,34 @@ app_server <- function(input, output, session) {
 
   # update the master 'Remove' column anytime data is added to the 'remove' table
   shiny::observeEvent(tadat$removals, {
+    print("Removals changed")
+    print(dim(tadat$removals))
+    print(sum(tadat$raw$TADA.Remove))
     if (dim(tadat$removals)[2] > 0) {
+      print("Updating remove col")
       tadat$raw$TADA.Remove <- apply(tadat$removals, 1, any)
     }
+    print(sum(tadat$raw$TADA.Remove))
   })
 
-  # Update the default switches if a progress file is uploaded.
-  shiny::observeEvent(tadat$selected_flags, {
-    switch_defaults = tadat$selected_flags
-    })
-
-
-  # this observes when the user switches tabs and adds the current tab they're on as a column to their dataset.
   shiny::observe({
-    shiny::req(tadat$raw)
+    # JCH - is this necessary?
+    #shiny::req(tadat$raw)
     tadat$raw$TADAShiny.tab <- input$tabbar
     tadat$tab <- input$tabbar
   })
 
+  # JCH - disabling this for now. I think progress files provide this functionality
+  # this observes when the user switches tabs and adds the current tab they're on as a column to their dataset.
+
   # switch to tab user left off on when tadat$reup changes, which only happens when someone uploads a workbook with the column "Removed" in it
-  shiny::observeEvent(tadat$reup, {
-    shiny::showModal(shiny::modalDialog(
-      title = "Data Loaded",
-      "Your working dataset has been uploaded and the app switched to the tab where you left off."
-    ))
-    # the switch tab command
-    shiny::updateTabsetPanel(session = session, inputId = "tabbar", selected = unique(tadat$raw$tab))
-    tadat$reup <- NULL
-  })
+  #shiny::observeEvent(tadat$reup, {
+  #  shiny::showModal(shiny::modalDialog(
+  #    title = "Data Loaded",
+  #    "Your working dataset has been uploaded and the app switched to the tab where you left off."
+  #  ))
+  #  # the switch tab command
+  #  shiny::updateTabsetPanel(session = session, inputId = "tabbar", selected = unique(tadat$raw$tab))
+  #  tadat$reup <- NULL
+  #})
 }

R/mod_TADA_summary.R

@@ -81,6 +81,8 @@ mod_TADA_summary_server <- function(id, tadat) {
     # calculate the stats needed to fill the summary box
     shiny::observe({
       shiny::req(tadat$raw)
+      print("tadat$raw has been changed (i think?)")
+      print(nrow(tadat$raw))
       summary_things$rem_rec <-
         length(tadat$raw$ResultIdentifier[tadat$raw$TADA.Remove ==
                                             TRUE])
@@ -95,6 +97,7 @@ mod_TADA_summary_server <- function(id, tadat) {
         length(unique(tadat$raw$MonitoringLocationIdentifier[!tadat$raw$MonitoringLocationIdentifier %in%
                                                                clean_sites]))
       summary_things$removals <- sort_removals(tadat$removals)
+      print(nrow(tadat$raw))
     })
     summary_things$removals <- data.frame(matrix(
       ncol = 2,
@@ -118,6 +121,8 @@ mod_TADA_summary_server <- function(id, tadat) {
       if (is.null(tadat$raw)) {
         "Total Results in Dataset: 0"
       } else {
+        print("Updating result count")
+        print(nrow(tadat$raw))
         paste0("Total Results in Dataset: ", scales::comma(length(tadat$raw$ResultIdentifier)))
       }
     })
@@ -171,14 +176,23 @@ mod_TADA_summary_server <- function(id, tadat) {
     # download dataset button - only appears if there data exists in the app already
     output$dwn_all <- shiny::renderUI({
       shiny::req(tadat$raw)
-      shiny::downloadButton(ns("download_all"),
+      shiny::downloadButton(ns("download_working"),
                             "Download Working Dataset (.zip)",
-                            style = "color: #fff; background-color: #337ab7; border-color: #2e6da4")
+                            style = "color: #fff; background-color: #337ab7; border-color: #2e6da4",
+                            contentType = "application/zip")
     })
 
-    output$download_all <- shiny::downloadHandler(
+    output$dwn_ts <- shiny::renderUI({
+      shiny::req(tadat$raw)
+      shiny::downloadButton(ns("download_final"),
+                            "Download Final Dataset (.zip)",
+                            style = "color: #fff; background-color: #337ab7; border-color: #2e6da4",
+                            contentType = "application/zip")
+    })
+
+    output$download_working <- shiny::downloadHandler(
       filename = function() {
-        paste0(tadat$default_outfile, ".zip")
+        paste0(tadat$default_outfile, "_working.zip")
       },
       content = function(fname) {
         fs <- c()
@@ -187,6 +201,7 @@ mod_TADA_summary_server <- function(id, tadat) {
         datafile_name = paste0(tadat$default_outfile, ".xlsx")
         progress_file_name = paste0(tadat$default_outfile, "_prog.RData")
         desc <- writeNarrativeDataFrame(tadat)
+        out_data = TADA::TADA_OrderCols(tadat$raw[tadat$raw])
         dfs <-
           list(Data = TADA::TADA_OrderCols(tadat$raw), Parameterization = desc)
         writeFile(tadat, progress_file_name)
@@ -197,22 +212,26 @@ mod_TADA_summary_server <- function(id, tadat) {
       contentType = "application/zip"
     )
 
-    # # Download TADA progress file
-    # output$dwn_ts = shiny::renderUI({
-    #   shiny::req(tadat$raw)
-    #   shiny::downloadButton(ns("download_ts_file"),
-    #                         "Download Progress File (.Rdata)",
-    #                         style = "color: #fff; background-color: #337ab7; border-color: #2e6da4")
-    # })
-
-    # output$download_ts_file = shiny::downloadHandler(
-    #   filename = function() {
-    #     paste0(tadat$job_id, '.Rdata')
-    #   },
-    #   content = function(file) {
-    #     writeFile(tadat, file)
-    #   }
-    # )
+    output$download_final <- shiny::downloadHandler(
+      filename = function() {
+        paste0(tadat$default_outfile, "_final.zip")
+      },
+      content = function(fname) {
+        fs <- c()
+        tmpdir <- tempdir()
+        setwd(tempdir())
+        datafile_name = paste0(tadat$default_outfile, ".xlsx")
+        progress_file_name = paste0(tadat$default_outfile, "_prog.RData")
+        desc <- writeNarrativeDataFrame(tadat)
+        dfs <-
+          list(Data = TADA::TADA_OrderCols(tadat$raw), Parameterization = desc)
+        writeFile(tadat, progress_file_name)
+        writexl::write_xlsx(dfs, path = datafile_name)
+        utils::zip(zipfile = fname,
+                   files = c(datafile_name, progress_file_name))
+      },
+      contentType = "application/zip"
+    )
 
     shiny::observeEvent(input$disclaimer, {
       shiny::showModal(

R/mod_censored_data.R

@@ -299,7 +299,10 @@ mod_censored_data_server <- function(id, tadat) {
         ) # reset to detection quantitation limit value
       tadat$raw$TADA.ResultMeasureValueDataTypes.Flag[tadat$raw$TADA.ResultMeasureValueDataTypes.Flag == "Result Value/Unit Estimated from Detection Limit"] <-
         "Result Value/Unit Copied from Detection Limit" # reset data types flag to what it was before simpleCensoredMethods function run
+      print(1234)
+      print(nrow(tadat$raw))
       tadat$raw <- tadat$raw %>% dplyr::select(-TADA.CensoredMethod)
+      print(nrow(tadat$raw))
       tadat$censor_applied = FALSE
     })
 

R/mod_data_flagging.R

@@ -89,8 +89,6 @@ mod_data_flagging_server <- function(id, tadat) {
 
     # Runs whenever selected flags are changed
     shiny::observeEvent(tadat$selected_flags, {
-      print(paste0("Selected flags changed. ", length(tadat$selected_flags), " flags selected"))
-
       if (!is.null(tadat$removals)) {
         tadat$removals = dplyr::select(tadat$removals, -(dplyr::starts_with(flag_prefix)))
       }
@@ -126,6 +124,7 @@ mod_data_flagging_server <- function(id, tadat) {
 
     shiny::observeEvent(tadat$flags_present, {
       if (tadat$flags_present) {
+
         # A table (raw rows, flags) indicating whether each record passes each test
         values$testResults <- flagCensus(tadat$raw)
 
@@ -193,9 +192,12 @@ mod_data_flagging_server <- function(id, tadat) {
 
       # Add flagging columns to raw table, make sure line below is
       # not commented out once done with testing
-      #tadat$raw <- applyFlags(tadat$raw, tadat$orgs)
+      print(2345)
+      print(nrow(tadat$raw))
+      tadat$raw <- applyFlags(tadat$raw, tadat$orgs)
+      print(nrow(tadat$raw))
       #write.csv(tadat$raw, "flagged.csv")
-      tadat$raw = utils::read.csv("flagged.csv") # THIS IS TRIPS WORKING FILE FOR TESTING, COMMENT OUT WHEN COMMITTING TO DEVELOP
+      #tadat$raw = utils::read.csv("flagged.csv") # THIS IS TRIPS WORKING FILE FOR TESTING, COMMENT OUT WHEN COMMITTING TO DEVELOP
 
       # Remove progress bar and display instructions
       shinybusy::remove_modal_spinner(session = shiny::getDefaultReactiveDomain())
@@ -219,16 +221,20 @@ mod_data_flagging_server <- function(id, tadat) {
         tadat$raw <-
           TADA::TADA_ConvertDepthUnits(tadat$raw, unit = "ft")
       }
+      print(3456)
+      print(nrow(tadat$raw))
       if (input$m2f == "inches") {
         shinybusy::show_modal_spinner(
           spin = "double-bounce",
           color = "#0071bc",
           text = "Converting depth units to inches...",
           session = shiny::getDefaultReactiveDomain()
         )
+
         tadat$raw <-
           TADA::TADA_ConvertDepthUnits(tadat$raw, unit = "in")
-      }
+
+        }
       if (input$m2f == "meters") {
         shinybusy::show_modal_spinner(
           spin = "double-bounce",
@@ -239,6 +245,7 @@ mod_data_flagging_server <- function(id, tadat) {
         tadat$raw <-
           TADA::TADA_ConvertDepthUnits(tadat$raw, unit = "m")
       }
+      print(nrow(tadat$raw))
       shinybusy::remove_modal_spinner(session = shiny::getDefaultReactiveDomain())
     })
   })

R/mod_filtering.R

@@ -61,14 +61,17 @@ mod_filtering_server <- function(id, tadat) {
     shinyjs::hide("addExcludes")
 
     # make sure dataset being used to create filters is only REMOVE = FALSE
-    shiny::observe({
-      shiny::req(tadat$tab)
+    shiny::observeEvent(tadat$tab, {
       if (tadat$tab == "Filter") {
+        print("Filter tab selected")
         # only show unique values from data that have not been flagged for removal
+        print(tadat$raw$TADA.Remove)
         tables$dat <-
           subset(tadat$raw, tadat$raw$TADA.Remove == FALSE)
         tables$filter_fields <-
           TADA::TADA_FieldCounts(tables$dat, display = "key")
+      } else {
+        print(paste0(tadat$tab, " tab selected"))
       }
     })
 
@@ -253,6 +256,7 @@ mod_filtering_server <- function(id, tadat) {
       values$locked = field_filters$Filter
       names(values$locked) <- field_filters$Field
       prefix = "Filter: "
+
       # Remove all the filter columns from the removals table (start fresh)
       if (!is.null(tadat$removals)) {
         tadat$removals <-

R/mod_harmonize_np.R

@@ -173,9 +173,11 @@ mod_harmonize_np_server <- function(id, tadat) {
       dat <- subset(tadat$raw, tadat$raw$TADA.Remove == FALSE)
       rem <- subset(tadat$raw, tadat$raw$TADA.Remove == TRUE)
       dat <- TADA::TADA_HarmonizeSynonyms(dat, ref = harm$ref)
+      print(4567)
+      print(nrow(tadat$raw))
       tadat$raw <- plyr::rbind.fill(dat, rem)
       tadat$raw <- TADA::TADA_OrderCols(tadat$raw)
-
+      print(nrow(tadat$raw))
       # remove the modal once the dataset has been harmonized
       shinybusy::remove_modal_spinner(session = shiny::getDefaultReactiveDomain())
 
@@ -222,10 +224,11 @@ mod_harmonize_np_server <- function(id, tadat) {
       new_df <- as.data.frame(matrix(FALSE, ncol = ncols, nrow = nrows))
       names(new_df) <- names(tadat$removals)
       tadat$removals <- plyr::rbind.fill(tadat$removals, new_df)
-
+      print(5678)
+      print(nrow(tadat$raw))
       tadat$raw <- plyr::rbind.fill(dat, rem)
       tadat$raw <- TADA::TADA_OrderCols(tadat$raw)
-
+      print(nrow(tadat$raw))
       nitrolen <- length(dat$TADA.NutrientSummation.Flag[dat$TADA.NutrientSummation.Flag %in% c("Nutrient summation from one or more subspecies.")])
       phoslen <- length(dat$TADA.NutrientSummation.Flag[dat$TADA.NutrientSummation.Flag %in% c("Nutrient summation from one subspecies.")])
       # remove the modal once the dataset has been harmonized

R/mod_overview.R

@@ -87,6 +87,7 @@ mod_overview_server <- function(id, tadat) {
         dplyr::group_by(OrganizationFormalName, OrganizationIdentifier) %>%
         dplyr::summarise("Result_Count" = length(unique(ResultIdentifier))) %>%
         dplyr::ungroup()
+
       tadat$org_table <- orgs %>%
         dplyr::arrange(-Result_Count) %>%
         dplyr::mutate("Rank" = 1:length(Result_Count))

R/mod_query_data.R

@@ -230,11 +230,12 @@ mod_query_data_server <- function(id, tadat) {
       # user uploaded data
       raw <-
         suppressWarnings(readxl::read_excel(input$file$datapath, sheet = 1))
+      raw$TADA.Remove <- NULL
       initializeTable(tadat, raw)
       if (!is.null(tadat$original_source)){
         tadat$original_source <- "Upload"
       }
-
+      
       shinybusy::remove_modal_spinner(session = shiny::getDefaultReactiveDomain())
 
     })
@@ -496,8 +497,10 @@ initializeTable <- function(tadat, raw) {
   removals <- data.frame(matrix(nrow = nrow(raw), ncol = 0))
   # removals["Media Type"] = ifelse(!raw$TADA.ActivityMediaName%in%c("WATER"),TRUE,raw$Removed)
   # removals["Special Characters"] = ifelse(raw$TADA.ResultMeasureValueDataTypes.Flag%in%c("ND or NA","Text","Coerced to NA"),TRUE,raw$Removed)
-
+  print(6789)
+  print(nrow(tadat$raw))
   tadat$raw <- raw
+  print(nrow(tadat$raw))
   tadat$removals <- removals
 }
 

R/sandbox.R

@@ -1,4 +0,0 @@
-a = c(1, 2, 3, 4, 5)
-print(a)
-a = 7
-print(a)

R/utils_track_progress.R

@@ -56,12 +56,13 @@ writeFile <- function(tadat, filename) {
 readFile <- function(tadat, filename) {
   load(filename, verbose = FALSE)
   tadat$load_progress_file = filename
-
+  print("loading progress file")
+  print(nrow(tadat$raw))
   # Confirm compatibility
   job_id = job_id
-  if (!is.null(m2f)) {
-    tadat$m2f = m2f
-  }
+  #if (!is.null(m2f)) {
+  #  tadat$m2f = m2f
+  #}
 
 
   if (!is.null(selected_flags)) {
@@ -90,12 +91,13 @@ readFile <- function(tadat, filename) {
   tadat$startDate = startDate
   tadat$endDate = endDate
   tadat$org_table = org_table
-  tadat$m2f = m2f
   tadat$selected_filters =  selected_filters
   tadat$nd_method = nd_method
   tadat$od_method = od_method
   tadat$nd_mult = nd_mult
   tadat$od_mult
+  print("Progress file loaded")
+  print(nrow(tadat$raw))
 }