diff --git a/DESCRIPTION b/DESCRIPTION index 216061b..83870e1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -74,6 +74,7 @@ Collate: 'containerit.R' 'defaults.R' 'dockerfile.R' + 'package-installation-bespoke.R' 'package-installation-methods.R' 'sessionInfo-localbuild-methods.R' 'utility-functions.R' diff --git a/NAMESPACE b/NAMESPACE index cb171ce..fbb94d5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -32,6 +32,7 @@ export(dockerfile) export(getGitHubRef) export(getImageForVersion) export(getRVersionTag) +export(getSessionInfoFromRdata) export(parseFrom) exportClasses(Add) exportClasses(Arg) diff --git a/R/Class-Run.R b/R/Class-Run.R index 47a5913..eccdc6b 100644 --- a/R/Class-Run.R +++ b/R/Class-Run.R @@ -21,7 +21,7 @@ setClass("Run_shell", # or ["param1","param2"] (for CMD as default parameters to ENTRYPOINT) commands <- methods::slot(obj, "commands") - string <- paste(commands, collapse = " \\\n && ") + string <- paste(commands, collapse = " \\\n && ") return(string) } diff --git a/R/dockerfile.R b/R/dockerfile.R index 1a10a84..6b0a6b7 100644 --- a/R/dockerfile.R +++ b/R/dockerfile.R @@ -28,14 +28,12 @@ #' This executes the whole file to obtain a complete \code{sessionInfo} object, see section "Based on \code{sessionInfo}", and copies required files and documents into the container. #' #' @param from The source of the information to construct the Dockerfile. Can be a \code{sessionInfo} object, a path to a file, or the path to a workspace). If \code{NULL} then no automatic derivation of dependencies happens. +#' @param image (\linkS4class{From}-object or character) Specifes the image that shall be used for the Docker container (\code{FROM} instruction). +#' By default, the image is determinded from the given session. Alternatively, use \code{getImageForVersion(..)} to get an existing image for a manually defined version of R, matching the version with tags from the base image rocker/r-ver (see details about the rocker/r-ver at \url{'https://hub.docker.com/r/rocker/r-ver/'}). Or provide a correct image name yourself. +#' @param maintainer Specify the maintainer of the dockerfile. See documentation at \url{'https://docs.docker.com/engine/reference/builder/#maintainer'}. Defaults to \code{Sys.info()[["user"]]}. Can be removed with \code{NULL}. #' @param save_image When TRUE, it calls \link[base]{save.image} and include the resulting .RData in the container's working directory. #' Alternatively, you can pass a list of objects to be saved, which may also include arguments to be passed down to \code{save}. E.g. save_image = list("object1", "object2", file = "path/in/wd/filename.RData"). #' \code{save} will be called with default arguments file = ".RData" and envir = .GlobalEnv -#' @param maintainer Specify the maintainer of the dockerfile. See documentation at \url{'https://docs.docker.com/engine/reference/builder/#maintainer'}. Defaults to \code{Sys.info()[["user"]]}. Can be removed with \code{NULL}. -#' @param r_version (character) optionally specify the R version that should run inside the container. By default, the R version from the given sessioninfo is used (if applicable) or the version of the currently running R instance -#' @param image (From-object or character) optionally specify the image that shall be used for the Docker container (FROM-statement) -#' By default, the image is determinded from the given r_version, while the version is matched with tags from the base image rocker/r-ver -#' see details about the rocker/r-ver at \url{'https://hub.docker.com/r/rocker/r-ver/'} #' @param env optionally specify environment variables to be included in the image. See documentation: \url{'https://docs.docker.com/engine/reference/builder/#env} #' @param soft (boolean) Whether to include soft dependencies when system dependencies are installed, default is no. #' @param offline (boolean) Whether to use an online database to detect system dependencies or use local package information (slower!), default is no. @@ -47,6 +45,7 @@ #' @param vanilla Whether to use an empty vanilla session when packaging scripts and markdown files (equivalent to \code{R --vanilla}) #' @param silent Whether or not to print information during execution #' @param versioned_libs [EXPERIMENTAL] Whether it shall be attempted to match versions of linked external libraries +#' @param versioned_packages [EXPERIMENTAL] Whether it shall be attempted to match versions of R packages #' #' @return An object of class Dockerfile #' @@ -59,14 +58,13 @@ #' print(dockerfile) #' dockerfile <- function(from = utils::sessionInfo(), - save_image = FALSE, + image = getImageForVersion(getRVersionTag(from)), maintainer = Sys.info()[["user"]], - r_version = getRVersionTag(from), - image = getImageForVersion(r_version), + save_image = FALSE, env = list(generator = paste("containerit", utils::packageVersion("containerit"))), soft = FALSE, offline = FALSE, - copy = "script", + copy = NA, # "script", # nolint start container_workdir = "/payload/", # nolint end @@ -75,12 +73,12 @@ dockerfile <- function(from = utils::sessionInfo(), add_self = FALSE, vanilla = TRUE, silent = FALSE, - versioned_libs = FALSE) { + versioned_libs = FALSE, + versioned_packages = FALSE) { if (silent) { invisible(futile.logger::flog.threshold(futile.logger::WARN)) } - futile.logger::flog.debug("Creating a new Dockerfile from '%s'", from) .dockerfile <- NA .originalFrom <- class(from) @@ -88,6 +86,7 @@ dockerfile <- function(from = utils::sessionInfo(), if (is.character(image)) { image <- parseFrom(image) } + futile.logger::flog.debug("Creating a new Dockerfile from '%s' with base image %s", from, toString(image)) if (is.character(maintainer)) { .label <- Label_Maintainer(maintainer) @@ -150,12 +149,13 @@ dockerfile <- function(from = utils::sessionInfo(), futile.logger::flog.debug("Creating from sessionInfo object") .dockerfile <- dockerfileFromSession( - session = from, .dockerfile = .dockerfile, + session = from, soft = soft, offline = offline, add_self = add_self, versioned_libs = versioned_libs, + versioned_packages = versioned_packages, workdir = workdir ) } else if (inherits(x = from, "character")) { @@ -175,10 +175,11 @@ dockerfile <- function(from = utils::sessionInfo(), vanilla = vanilla, silent = silent, versioned_libs = versioned_libs, + versioned_packages = versioned_packages, workdir = workdir ) } else if (file.exists(from)) { - futile.logger::flog.debug("'%s' is a file") + futile.logger::flog.debug("'%s' is a file", from) .originalFrom <- from .dockerfile <- dockerfileFromFile( @@ -191,6 +192,7 @@ dockerfile <- function(from = utils::sessionInfo(), vanilla = vanilla, silent = silent, versioned_libs = versioned_libs, + versioned_packages = versioned_packages, workdir = workdir ) } else { @@ -198,22 +200,19 @@ dockerfile <- function(from = utils::sessionInfo(), } } else if (is.expression(from) || (is.list(from) && all(sapply(from, is.expression)))) { - futile.logger::flog.debug("Creating from expession: %s", toString(from)) - - .sessionInfo <- - clean_session(expr = from, - slave = silent, - vanilla = vanilla) - .dockerfile <- - dockerfileFromSession( - session = .sessionInfo, - .dockerfile = .dockerfile, - soft = soft, - offline = offline, - add_self = add_self, - versioned_libs = versioned_libs, - workdir = workdir - ) + futile.logger::flog.debug("Creating from expession: '%s' with a clean session", toString(from)) + + .sessionInfo <- clean_session(expr = from, + slave = silent, + vanilla = vanilla) + .dockerfile <- dockerfileFromSession(.dockerfile = .dockerfile, + session = .sessionInfo, + soft = soft, + offline = offline, + add_self = add_self, + versioned_libs = versioned_libs, + versioned_packages = versioned_packages, + workdir = workdir) } else { stop("Unsupported 'from': ", class(from), " ", from) } @@ -244,6 +243,7 @@ dockerfileFromSession <- function(session, offline, add_self, versioned_libs, + versioned_packages, workdir) { futile.logger::flog.debug("Creating from sessionInfo") @@ -259,20 +259,21 @@ dockerfileFromSession <- function(session, image_name = .dockerfile@image@image if (image_name %in% .debian_images) { platform = .debian_platform - futile.logger::flog.debug("Found image %s in list of debian images.") + futile.logger::flog.debug("Found image %s in list of Debian images", image_name) } - futile.logger::flog.debug("Detected platform %s", platform) + futile.logger::flog.debug("Detected platform: %s", platform) - .dockerfile <- - .create_run_install( - .dockerfile = .dockerfile, - pkgs = pkgs, - platform = platform, - soft = soft, - offline = offline, - versioned_libs = versioned_libs, - workdir = workdir # will be added as last instruction - ) + .dockerfile <- .create_run_install( + dockerfile = .dockerfile, + pkgs = pkgs, + platform = platform, + soft = soft, + offline = offline, + versioned_libs = versioned_libs, + versioned_packages = versioned_packages) + + # after all installation is done, set the workdir + addInstruction(.dockerfile) <- workdir return(.dockerfile) } @@ -287,11 +288,11 @@ dockerfileFromFile <- vanilla, silent, versioned_libs, + versioned_packages, workdir) { futile.logger::flog.debug("Creating from file") - ################################################# + # prepare context ( = working directory) and normalize paths: - ################################################# context = normalizePath(getwd()) file = normalizePath(file) @@ -304,9 +305,7 @@ dockerfileFromFile <- # make sure that the path is relative to context rel_path <- .makeRelative(file, context) - #################################################### - # execute script / markdowns and obtain sessioninfo - ##################################################### + # execute script / markdowns or read Rdata file to obtain sessioninfo if (stringr::str_detect(file, ".R$")) { futile.logger::flog.info("Executing R script file in %s locally.", rel_path) sessionInfo <- @@ -316,7 +315,7 @@ dockerfileFromFile <- slave = silent, echo = !silent ) - } else if (stringr::str_detect(file, ".Rmd$")) { + } else if (stringr::str_detect(file, ".Rmd$")) { futile.logger::flog.info("Processing the given file %s locally using rmarkdown::render(...)", rel_path) sessionInfo <- obtain_localSessionInfo( @@ -325,13 +324,13 @@ dockerfileFromFile <- slave = silent, echo = !silent ) + } else if (stringr::str_detect(file, ".Rdata$")) { + sessionInfo <- getSessionInfoFromRdata(file) } else{ futile.logger::flog.info("The supplied file %s has no known extension. containerit will handle it as an R script for packaging.", rel_path) } - #################################################### # append system dependencies and package installation instructions - #################################################### .dockerfile <- dockerfileFromSession( session = sessionInfo, @@ -340,11 +339,11 @@ dockerfileFromFile <- offline = offline, add_self = add_self, versioned_libs = versioned_libs, + versioned_packages = versioned_packages, workdir = workdir ) ## working directory must be set before. Now add copy instructions - #################################################### if (!is.null(copy) && !is.na(copy)) { copy = unlist(copy) if (!is.character(copy)) { @@ -400,6 +399,7 @@ dockerfileFromWorkspace <- vanilla, silent, versioned_libs, + versioned_packages, workdir) { futile.logger::flog.debug("Creating from workspace directory") target_file <- NULL #file to be packaged @@ -453,13 +453,13 @@ dockerfileFromWorkspace <- vanilla = vanilla, silent = silent, versioned_libs = versioned_libs, + versioned_packages = versioned_packages, workdir = workdir ) return(.df) } - #' getImageForVersion-method #' #' Get a suitable Rocker image based on the R version. @@ -483,7 +483,7 @@ getImageForVersion <- function(r_version, nearest = TRUE) { image <- From(.rocker_images[["versioned"]], tag = r_version) closestMatch <- function(version, versions) { - if(version %in% versions) return(version); + if (version %in% versions) return(version); factors <- list(major = 1000000, minor = 1000, patch = 1) @@ -506,23 +506,21 @@ getImageForVersion <- function(r_version, nearest = TRUE) { } if (!r_version %in% tags) { - if(nearest) { + if (nearest) { # get numeric versions with all parts (maj.min.minor), i.e. two dots numeric_tags <- tags[which(grepl("\\d.\\d.\\d", tags))] closest <- as.character(closestMatch(r_version, numeric_tags)) image <- From(.rocker_images[["versioned"]], tag = closest) - warning( - "No Docker image found for the given R version, returning closest match: ", - closest, - " Existing tags (list only available when online): ", - paste(tags, collapse = " ") + warning("No Docker image found for the given R version, returning closest match: ", + closest, + " Existing tags (list only available when online): ", + paste(tags, collapse = " ") ) } else { - warning( - "No Docker image found for the given R version, returning input. ", - "Existing tags (list only available when online): ", - paste(tags, collapse = " ") + warning("No Docker image found for the given R version, returning input. ", + "Existing tags (list only available when online): ", + paste(tags, collapse = " ") ) } } @@ -531,10 +529,9 @@ getImageForVersion <- function(r_version, nearest = TRUE) { } .tagsfromRemoteImage <- function(image) { - urlstr <- - paste0("https://registry.hub.docker.com/v2/repositories/", - image, - "/tags/?page_size=9999") + urlstr <- paste0("https://registry.hub.docker.com/v2/repositories/", + image, + "/tags/?page_size=9999") tryCatch({ con <- url(urlstr) diff --git a/R/package-installation-bespoke.R b/R/package-installation-bespoke.R new file mode 100644 index 0000000..e65b21e --- /dev/null +++ b/R/package-installation-bespoke.R @@ -0,0 +1,72 @@ + +.skipable_deps <- function(image_name) { + # we may ad more no-apt or analogue no-package exceptions here and handle them with the json config - + # as far as we know that certain images have sertain dependencies pre-installed, + # but at the moment it won't be necessary + if (image_name == "rocker/geospatial") { + return(c("libproj-dev", "libgeos-dev", "gdal-bin")) + } + return(c()) +} + +.install_sf_with_outdated_system_deps <- function() { + if ("sf" %in% pkg_names) { + # sf-dependencies proj and gdal cannot be installed directly from apt get, because the available packages are outdated. + sf_installed <- requireNamespace("sf") + + if (sf_installed && versioned_libs) { + # Exceptions are handled by json config here: + ext_soft <- sf::sf_extSoftVersion() + mapply(function(lib, version) { + if (!.isVersionSupported(lib, version, .package_config)) { + futile.logger::flog.warn("No explicit for support for the version %s of the linked external software %s", version, lib) + return() + } + + add_apt <<- append(add_apt, + .get_lib_apt_requirements( + lib, + version, + .package_config)) + no_apt <<- append(add_apt, + .get_lib_pkgs_names( + lib = lib, + platform = .debian_platform, + config = .package_config)) + add_inst <<- append(add_inst, + .get_lib_install_instructions( + lib = lib, + version = version, + config = .package_config)) + + return(invisible()) + }, + lib = names(ext_soft), + version = as.character(ext_soft)) + + + #NOTE: The following is the "old" way to do it. Getting sf to work only requires a more current version gdal, while all other dependencies can be installed from APT + } else if (!image_name == "rocker/geospatial") { + futile.logger::flog.info( + "The dependent package simple features for R requires current versions from gdal, geos and proj that may not be available by standard apt-get.\nWe recommend using the base image rocker/geospatial." + ) + futile.logger::flog.info("Docker will try to install GDAL 2.1.3 from source") + + add_apt <- append(add_apt, c("wget", "make")) + add_inst <- append(add_inst, Workdir("/tmp/gdal")) + add_inst <- + append(add_inst, Run_shell( + c( + "wget http://download.osgeo.org/gdal/2.1.3/gdal-2.1.3.tar.gz", + "tar zxf gdal-2.1.3.tar.gz", + "cd gdal-2.1.3", + "./configure", + "make", + "make install", + "ldconfig", + "rm -r /tmp/gdal" + ) + )) + } + } +} diff --git a/R/package-installation-methods.R b/R/package-installation-methods.R index 28754ed..07cc1aa 100644 --- a/R/package-installation-methods.R +++ b/R/package-installation-methods.R @@ -1,181 +1,83 @@ # Copyright 2017 Opening Reproducible Research (http://o2r.info) -#pkgs list of packages as returned by sessionInfo +# pkgs is a list of packages as returned by sessionInfo() +# function returns a the dockerfile with the required instructions .create_run_install <- - function(.dockerfile, + function(dockerfile, pkgs, platform, soft, offline, versioned_libs, - workdir) { - #create RUN expressions + versioned_packages, + filter_deps_by_image = FALSE) { package_reqs <- character(0) cran_packages <- character(0) github_packages <- character(0) - local_packages <- character(0) - other_packages <- character(0) pkg_names <- character(0) package_versions <- character(0) + # 1. identify where to install the package from sapply(pkgs, function(pkg) { #determine package name if ("Package" %in% names(pkg)) name <- pkg$Package else - stop("Package name cannot be dertermined for ", pkg) #should hopefully never occure + stop("Package name cannot be dertermined for ", pkg) if ("Priority" %in% names(pkg) && stringr::str_detect(pkg$Priority, "(?i)base")) { - #packages with these priorities are normally included and don't need to be installed; do nothing - return() + futile.logger::flog.debug("Skipping Priority package %s, is included with R", name) + } else { + #if necessary, determine package dependencies (outside the loop) + pkg_names <<- append(pkg_names, name) + package_versions <<- append(package_versions, pkg$Version) + + #check if package come from CRAN or GitHub + if ("Repository" %in% names(pkg) && + stringr::str_detect(pkg$Repository, "(?i)CRAN")) { + cran_packages <<- append(cran_packages, pkg$Package) + } else if ("RemoteType" %in% names(pkg) && + stringr::str_detect(pkg$RemoteType, "(?i)github")) { + github_packages <<- append(github_packages, getGitHubRef(pkg$Package, pkgs)) + } else + warning("Failed to identify a source for package ", + pkg$Package, + ". Therefore the package cannot be installed in the Docker image.\n") } - #if necessary, determine package dependencies (outside the loop) - pkg_names <<- append(pkg_names, name) - package_versions <<- - append(package_versions, pkg$Version) - - #check if package come from CRAN or GitHub - if ("Repository" %in% names(pkg) && - stringr::str_detect(pkg$Repository, "(?i)CRAN")) - { - cran_packages <<- append(cran_packages, pkg$Package) - return() - } else if ("RemoteType" %in% names(pkg) && - stringr::str_detect(pkg$RemoteType, "(?i)github")) - { - github_packages <<- - append(github_packages, getGitHubRef(pkg$Package)) - return() - } - - else - warning( - "Failed to identify source for package ", - pkg$Package, - ". Therefore the package cannot be installed in the Docker image.\n" - ) }) - image_name <- .dockerfile@image@image + image_name <- dockerfile@image@image - # installing github packages requires the package 'remotes' + # installing github packages, requires the package 'remotes' if (length(github_packages) > 0 && !"remotes" %in% cran_packages && !image_name %in% c("rocker/tidyverse", "rocker/verse", "rocker/geospatial")) { cran_packages <- append(cran_packages, "remotes") pkg_names <- append(pkg_names, "remotes") - if (requireNamespace("remotes")) - package_versions <- - append(package_versions, utils::packageVersion("remotes")) - else - package_versions <- append(package_versions, "latest") + #if (requireNamespace("remotes")) + # package_versions <- + # append(package_versions, utils::packageVersion("remotes")) + #else + package_versions <- append(package_versions, "latest") } - if(is.null(platform)) { + if (is.null(platform)) { warning("Platform could not be detected, proceed at own risk.") } else if (!isTRUE(platform %in% .supported_platforms)) { - warning( - "The determined platform '", - platform, - "' is currently not supported for handling system dependencies. Therefore, the created manifests might not work." - ) + warning("The determined platform '", + platform, + "' is currently not supported for handling system dependencies. Therefore, the created manifests might not work.") } + # 2. get system dependencies if packages must be installed if (length(pkg_names) > 0) { - # dependencies that can be left out - no_apt <- character(0) - - # additional dependencies - add_apt <- character(0) - - # additional instructions that shall be appended -after- installing system requirements - add_inst <- list() - - if ("sf" %in% pkg_names) { - # sf-dependencies proj and gdal cannot be installed directly from apt get, because the available packages are outdated. - sf_installed <- requireNamespace("sf") - - if (sf_installed && - versioned_libs) { - # Exceptions are handled by json config here: - ext_soft <- sf::sf_extSoftVersion() - mapply(function(lib, version) { - if (!.isVersionSupported(lib, version, .package_config)) { - msg <- - paste( - "No explicit for support for the version", - version, - "of the linked external software", - lib - ) - futile.logger::flog.warn(msg) - return() - } - add_apt <<- - append(add_apt, - .get_lib_apt_requirements(lib, version, .package_config)) - no_apt <<- - append( - add_apt, - .get_lib_pkgs_names( - lib = lib, - platform = .debian_platform, - config = .package_config - ) - ) - add_inst <<- - append( - add_inst, - .get_lib_install_instructions( - lib = lib, - version = version, - config = .package_config - ) - ) - return(invisible()) - }, - lib = names(ext_soft), - version = as.character(ext_soft)) - - - #NOTE: The following is the "old" way to do it. Getting sf to work only requires a more current version gdal, while all other dependencies can be installed from APT - } else if (!image_name == "rocker/geospatial") { - futile.logger::flog.info( - "The dependent package simple features for R requires current versions from gdal, geos and proj that may not be available by standard apt-get.\nWe recommend using the base image rocker/geospatial." - ) - futile.logger::flog.info("Docker will try to install GDAL 2.1.3 from source") - - add_apt <- append(add_apt, c("wget", "make")) - add_inst <- append(add_inst, Workdir("/tmp/gdal")) - add_inst <- - append(add_inst, Run_shell( - c( - "wget http://download.osgeo.org/gdal/2.1.3/gdal-2.1.3.tar.gz", - "tar zxf gdal-2.1.3.tar.gz", - "cd gdal-2.1.3", - "./configure", - "make", - "make install", - "ldconfig", - "rm -r /tmp/gdal" - ) - )) - } - } - - # we may ad more no-apt or analogue no-package exceptions here and handle them with the json config - - # as far as we know that certain images have sertain dependencies pre-installed, - # but at the moment it won't be necessary - if (image_name == "rocker/geospatial") { - #these packages are pre-installed - no_apt <- - append(no_apt, c("libproj-dev", "libgeos-dev", "gdal-bin")) - } + # see package-installation-bespoke.R + # add_inst <- list() - # determine package dependencies (if applicable by given platform) - pkg_dep <- .find_system_dependencies( + # determine package dependencies (if applicable by given platform) + package_reqs <- .find_system_dependencies( pkg_names, platform = platform, soft = soft, @@ -183,108 +85,107 @@ package_version = package_versions ) - # fix duplicates and parsing https://github.com/o2r-project/containerit/issues/79 - pkg_dep_deduped <- unique(unlist(pkg_dep, use.names = FALSE)) - - # fix if depends come back with a space https://github.com/r-hub/sysreqsdb/issues/22 - pkg_dep <- - unlist(lapply(pkg_dep_deduped, function(x) { - unlist(strsplit(x, split = " ")) - })) - - package_reqs <- append(package_reqs, pkg_dep) - - #some packages may not need to be installed, e.g. because they are pre-installed for a certain image - package_reqs <- package_reqs[!package_reqs %in% no_apt] - package_reqs <- append(package_reqs, add_apt) + # dependencies that can be left out because they are pre-installed for a certain image + if (filter_deps_by_image) { + skipable <- .skipable_deps(image_name) + package_reqs <- package_reqs[!package_reqs %in% skipable] + futile.logger::flog.info("Skipping deps for image %s: %s", image_name, toString(skipable)) + } - #remove dublicate system requirements and sort (to increase own reproducibility) + # remove dublicate system requirements and sort (to increase own reproducibility) package_reqs <- levels(as.factor(package_reqs)) package_reqs <- sort(package_reqs) # if platform is debian and system dependencies need to be installed - if (platform == .debian_platform && length(package_reqs) > 0) { - commands <- - "export DEBIAN_FRONTEND=noninteractive; apt-get -y update" - install_command <- - paste("apt-get install -y", - paste(package_reqs, collapse = " \\\n\t")) - commands <- append(commands, install_command) - addInstruction(.dockerfile) <- Run_shell(commands) - - if (length(add_inst) > 0) - addInstruction(.dockerfile) <- add_inst - # For using the exec form (??): - # Run("/bin/sh", params = c("-c", "export", "DEBIAN_FRONTEND=noninteractive"))) - # Run("apt-get", params = c("update", "-qq", "&&", "install", "-y" , package_reqs))) + if (length(package_reqs) > 0) { + if (platform == .debian_platform) { + commands <- "export DEBIAN_FRONTEND=noninteractive; apt-get -y update" + install_command <- paste("apt-get install -y", + paste(package_reqs, collapse = " \\\n\t")) + commands <- append(commands, install_command) + addInstruction(dockerfile) <- Run_shell(commands) + + #if (length(add_inst) > 0) + # addInstruction(dockerfile) <- add_inst + # For using the exec form (??): + # Run("/bin/sh", params = c("-c", "export", "DEBIAN_FRONTEND=noninteractive"))) + # Run("apt-get", params = c("update", "-qq", "&&", "install", "-y" , package_reqs))) + } else { + warning("Platform ", platform, " not supported, cannot add installation commands for system requirements.") + } } - } # length(package_names) if (length(cran_packages) > 0) { cran_packages <- sort(cran_packages) # sort, to increase own reproducibility futile.logger::flog.info("Adding CRAN packages: %s", toString(cran_packages)) - addInstruction(.dockerfile) <- Run("install2.r", cran_packages) + addInstruction(dockerfile) <- Run("install2.r", cran_packages) } - if (length(github_packages) > 0) { # sort, to increase own reproducibility - github_packages <- sort(github_packages) + if (length(github_packages) > 0) { + github_packages <- sort(github_packages) # sort, to increase own reproducibility futile.logger::flog.info("Adding GitHub packages: %s", toString(github_packages)) - addInstruction(.dockerfile) <- Run("installGithub.r", github_packages) + addInstruction(dockerfile) <- Run("installGithub.r", github_packages) } - # after all installation is done, set the workdir - addInstruction(.dockerfile) <- workdir - - return(.dockerfile) + return(dockerfile) } -.find_system_dependencies <- - function(package, - platform, - soft = TRUE, - offline = FALSE, - package_version = utils::packageVersion(package)) { - method = if (offline == TRUE) - method = "sysreq-package" - else - method = "sysreq-api" - .dependencies <- NA - - futile.logger::flog.info("Going online? %s ... to retrieve system dependencies (%s)", !offline, method) - futile.logger::flog.debug("Retrieving sysreqs for %s packages and platform %s:\n%s", length(package), platform, toString(package)) +.find_system_dependencies <- function(package, + platform, + soft = TRUE, + offline = FALSE, + package_version = utils::packageVersion(package)) { + method = if (offline == TRUE) + method = "sysreq-package" + else + method = "sysreq-api" + + .dependencies <- NA + + futile.logger::flog.info("Going online? %s ... to retrieve system dependencies (%s)", !offline, method) + futile.logger::flog.debug("Retrieving sysreqs for %s packages and platform %s: %s", length(package), platform, toString(package)) # slower, because it analyzes all package DESCRIPTION files of attached / loaded packages. # That causes an overhead of database-requests, because dependent packages appear in the sessionInfo as well as in the DESCRIPTION files - if (method == "sysreq-package") - .dependencies <- .find_by_sysreqs_pkg( - package = package, - package_version = package_version, - platform = platform, - soft = soft - ) + if (method == "sysreq-package") { + .dependencies <- .find_by_sysreqs_pkg( + package = package, + package_version = package_version, + platform = platform, + soft = soft + ) + } - # faster, but only finds direct package dependencies from all attached / loaded packages - if (method == "sysreq-api") - .dependencies <- .find_by_sysreqs_api(package = package, platform = platform) + # faster, but only finds direct package dependencies from all attached / loaded packages + if (method == "sysreq-api") { + .dependencies <- .find_by_sysreqs_api(package = package, platform = platform) - futile.logger::flog.debug("Found system %s dependencies:\n%s", length(.dependencies), toString(.dependencies)) - return(.dependencies) + if (length(.dependencies) > 0) { + # remove duplicates and unlist dependency string from sysreqs + .dependencies <- unique(unlist(.dependencies, use.names = FALSE)) + .dependencies <- unlist(lapply(.dependencies, function(x) { + unlist(strsplit(x, split = " ")) + })) + } } -.find_by_sysreqs_pkg <- - function(package, - platform, - soft = TRUE, - package_version, - localFirst = TRUE) { - #for more than one package: + futile.logger::flog.debug("Found system %s dependencies: %s", length(.dependencies), toString(.dependencies)) + return(.dependencies) +} + +.find_by_sysreqs_pkg <- function(package, + platform, + soft = TRUE, + package_version, + localFirst = TRUE) { + # for more than one package: if (length(package) > 1) { out <- mapply(function(pkg, version) { .find_by_sysreqs_pkg(pkg, platform, soft, version, localFirst) }, pkg = package, version = package_version) - return(out) #there might be dublicate dependencies here but they are removed by the invoking method + return(out) # there might be dublicate dependencies, they must be removed by the invoking method } sysreqs <- character(0) @@ -306,7 +207,7 @@ ) } else{ sysreqs <- NA - if(is.null(platform)) { + if (is.null(platform)) { futile.logger::flog.warn("Platform could not be determined, possibly because of unknown base image.", " Using '%s'", sysreqs::current_platform()) sysreqs <- @@ -325,12 +226,9 @@ futile.logger::flog.info("Trying to determine system requirements for '%s' from the DESCRIPTION file on CRAN", package) - con <- - url(paste0( - "https://CRAN.R-project.org/package=", - package, - "/DESCRIPTION" - )) + con <- url(paste0("https://CRAN.R-project.org/package=", + package, + "/DESCRIPTION")) temp <- tempfile() success <- TRUE tryCatch({ @@ -359,18 +257,13 @@ } -.find_by_sysreqs_api <- - function(package, platform) { - #calls like e.g. https://sysreqs.r-hub.io/pkg/rgdal,curl,rmarkdown/linux-x86_64-ubuntu-gcc are much faster than doing separate calls for each package +.find_by_sysreqs_api <- function(package, platform) { + # calls like e.g. https://sysreqs.r-hub.io/pkg/rgdal,curl,rmarkdown/linux-x86_64-ubuntu-gcc are much faster than doing separate calls for each package if (length(package) > 0) { package = paste(package, collapse = ",") } - package_msg <- stringr::str_replace_all(package, ",", ", ") - futile.logger::flog.info( - "Trying to determine system requirements for the package(s) '%s' from sysreq online DB", - package_msg - ) + futile.logger::flog.info("Trying to determine system requirements for the package(s) '%s' from sysreqs online DB", package) .url <- paste0("https://sysreqs.r-hub.io/pkg/", package, "/", platform) con <- url(.url) @@ -400,11 +293,17 @@ #' Get GitHub reference from package #' -#' If a package is not installed from CRAN, this functions tries to determine if it was installed from GitHub using \code{\link[devtools]{session_info}}. +#' If a package is installed from GitHub this function tries to retrieve the reference (i.e. repository accoutn, name, and commit) from #' -#' @param pkg The name of the package to retrieve the +#' \enumerate{ +#' \item the provided sessionInfo +#' \item locally, and only if the package is installed (!), using \code{\link[devtools]{session_info}} +#' } #' -#' @return A character string with a short refernce, e.g. \code{r-hub/sysreqs@481d263} +#' @param pkg The name of the package to retrieve the reference for +#' @param pkgs Lists of packages from a sessionInfo object +#' +#' @return A character string with a short refernce, e.g. \code{r-hub/sysreqs@481d263}, \code{NA} is nothign could be found #' @export #' #' @examples @@ -412,38 +311,45 @@ #' getGitHubRef(rsysreqs) #' } #' -getGitHubRef = function(pkg) { - if (!requireNamespace(pkg)) - stop("Package ", pkg, " cannot be loaded.") - - si_devtools <- devtools::session_info() - selected <- si_devtools$packages$package == pkg - ref_devtools <- si_devtools$packages$source[selected] - futile.logger::flog.debug("Looking for references for package %s", ref_devtools) - - #try to determine github reference from devools - if (stringr::str_detect(ref_devtools, "(?i)^GitHub \\(.*/.*@|#.*\\)$")) { - ref_devtools <- - stringr::str_replace(ref_devtools, "(?i)^GitHub \\(", replacement = "") - ref_devtools <- - stringr::str_replace(ref_devtools, "\\)$", replacement = "") - - futile.logger::flog.debug("GitHub reference for %s found with devtools: %s", - pkg, - ref_devtools) - return(ref_devtools) - } else { - #alternatively, try with 'normal' sessioninfo (normally does not reference a commit) - si_regular <- sessionInfo() - pkgs = c(si_regular$otherPkgs, si_regular$loadedOnly) +getGitHubRef = function(pkg, pkgs = c(sessionInfo()$otherPkgs, sessionInfo()$loadedOnly)) { + ref <- NA_character_ + + if (!is.null(pkgs[[pkg]]$GithubRepo)) repo <- pkgs[[pkg]]$GithubRepo + else repo <- pkgs[[pkg]]$RemoteRepo + + if (!is.null(pkgs[[pkg]]$GithubUsername)) uname <- pkgs[[pkg]]$GithubUsername - ghr <- pkgs[[pkg]]$GithubRef + else uname <- pkgs[[pkg]]$RemoteUsername + + if (!is.null(pkgs[[pkg]]$GithubSHA1)) + ghr <- pkgs[[pkg]]$GithubSHA1 + else ghr <- pkgs[[pkg]]$RemoteSha + + if (any(sapply(X = c(repo, uname, ghr), FUN = is.null))) { + futile.logger::flog.warn("Exact reference of GitHub package %s could not be determined from session info: %s %s %s", + pkg, repo, uname, ghr) + } else { ref = paste0(uname, "/", repo, "@", ghr) + } - futile.logger::flog.warn("Exact reference of GitHub package %s could not be determined: %s", - pkg, - ref) - return(ref) + if (is.na(ref)) { + if (requireNamespace(pkg)) + #try to determine github reference from devools + si_devtools <- devtools::session_info() + ref_devtools <- si_devtools$packages$source[si_devtools$packages$package == pkg] + futile.logger::flog.debug("Looking for references with devtools for package %s", ref_devtools) + + if (stringr::str_detect(ref_devtools, "(?i)^GitHub \\(.*/.*@|#.*\\)$")) { + ref_devtools <- stringr::str_replace(ref_devtools, "(?i)^GitHub \\(", replacement = "") + ref_devtools <- stringr::str_replace(ref_devtools, "\\)$", replacement = "") + futile.logger::flog.debug("GitHub reference for %s found with devtools: %s", + pkg, + ref_devtools) + ref <- ref_devtools + } else + futile.logger::flog.warn("GitHub ref is unknown, but package %s is not available locally, no fallback.", pkg) } + return(ref) } + diff --git a/R/utility-functions.R b/R/utility-functions.R index 918f2dd..f7ca43f 100644 --- a/R/utility-functions.R +++ b/R/utility-functions.R @@ -121,11 +121,11 @@ addInstruction <- function(dockerfileObject, value) { #' addInstruction(df) <- Label(myKey = "myContent") "addInstruction<-" <- addInstruction -#' Get R version from a variety of sources in a string format used for image tags +#' Get R version in string format used for image tags #' #' Returns either a version extracted from a given object or the default version. #' -#' @param from the source to extract an R version: a `sessionInfo()` object +#' @param from the source to extract an R version: a `sessionInfo()` object, or an \code{Rdata} file with a \code{sessionInfo} object #' @param default if 'from' does not contain version information (e.g. its an Rscript), use this default version information. #' #' @export @@ -135,14 +135,51 @@ addInstruction <- function(dockerfileObject, value) { #' getRVersionTag() getRVersionTag <- function(from = NULL, default = R.Version()) { r_version <- NULL - if (inherits(from, "sessionInfo")) { + if (is.null(from)) { + r_version <- default + } else if (inherits(from, "sessionInfo")) { r_version <- from$R.version - } else + futile.logger::flog.debug("Got R version from sessionInfo: %s", r_version) + } else if (file.exists(from) && stringr::str_detect(from, ".Rdata$")) { + sessionInfo <- getSessionInfoFromRdata(from) + r_version <- sessionInfo$R.version + futile.logger::flog.debug("Got R version from file %s: %s", from, r_version) + } else { r_version <- default + futile.logger::flog.debug("Falling back to default R version: %s", r_version) + } return(paste(r_version$major, r_version$minor, sep = ".")) } +#' Reads a \code{sessionInfo} object from an \code{Rdata} file +#' +#' @param file file path +#' +#' @return An object of class \code{sessionInfo} +#' @export +#' +#' @examples +#' sessionInfo <- sessionInfo() +#' file <- tempfile(tmpdir = tempdir(), fileext = ".Rdata") +#' save(sessionInfo, file = file) +#' getSessionInfoFromRdata(file) +getSessionInfoFromRdata <- function(file) { + futile.logger::flog.info("Reading object 'sessionInfo' from the given file %s", file) + e1 <- new.env() + load(file = file, envir = e1) + + futile.logger::flog.debug("Loaded Rdata file with objects %s", toString(ls(envir = e1))) + if (!all(grepl(pattern = "sessionInfo", ls(envir = e1)))) + stop("Provided Rdata file must contain only one object of name 'sessionInfo'") + + sessionInfo <- get("sessionInfo", envir = e1) + if (!inherits(sessionInfo, "sessionInfo")) + stop("Provided sessionInfo objects must have class 'sessionInfo' but is ", class(sessionInfo)) + + return(sessionInfo) +} + #' Creates an empty R session via system commands and captures the session information #' #' @param expr optional list of expressions to be executed in the session diff --git a/demo/fullstack.R b/demo/fullstack.R index 62fec89..5063f8c 100644 --- a/demo/fullstack.R +++ b/demo/fullstack.R @@ -26,7 +26,7 @@ close(fileConn) #' Create Dockerfile workspace with containerit setwd(workspace) cmd <- CMD_Rscript("script.R") -df <- containerit::dockerfile(from = workspace, cmd = cmd, r_version = "3.3.3", copy = "script_dir") +df <- containerit::dockerfile(from = workspace, cmd = cmd, image = getImageForVersion("3.3.3"), copy = "script_dir") save(df) #' Create image with harbor (see also utility function `containerit::docker_build(..)`). diff --git a/man/dockerfile.Rd b/man/dockerfile.Rd index 20b7117..a0fab19 100644 --- a/man/dockerfile.Rd +++ b/man/dockerfile.Rd @@ -4,28 +4,26 @@ \alias{dockerfile} \title{dockerfile-method} \usage{ -dockerfile(from = utils::sessionInfo(), save_image = FALSE, - maintainer = Sys.info()[["user"]], r_version = getRVersionTag(from), - image = getImageForVersion(r_version), env = list(generator = - paste("containerit", utils::packageVersion("containerit"))), soft = FALSE, - offline = FALSE, copy = "script", container_workdir = "/payload/", - cmd = Cmd("R"), entrypoint = NULL, add_self = FALSE, vanilla = TRUE, - silent = FALSE, versioned_libs = FALSE) +dockerfile(from = utils::sessionInfo(), + image = getImageForVersion(getRVersionTag(from)), + maintainer = Sys.info()[["user"]], save_image = FALSE, + env = list(generator = paste("containerit", + utils::packageVersion("containerit"))), soft = FALSE, offline = FALSE, + copy = NA, container_workdir = "/payload/", cmd = Cmd("R"), + entrypoint = NULL, add_self = FALSE, vanilla = TRUE, silent = FALSE, + versioned_libs = FALSE, versioned_packages = FALSE) } \arguments{ \item{from}{The source of the information to construct the Dockerfile. Can be a \code{sessionInfo} object, a path to a file, or the path to a workspace). If \code{NULL} then no automatic derivation of dependencies happens.} -\item{save_image}{When TRUE, it calls \link[base]{save.image} and include the resulting .RData in the container's working directory. - Alternatively, you can pass a list of objects to be saved, which may also include arguments to be passed down to \code{save}. E.g. save_image = list("object1", "object2", file = "path/in/wd/filename.RData"). -\code{save} will be called with default arguments file = ".RData" and envir = .GlobalEnv} +\item{image}{(\linkS4class{From}-object or character) Specifes the image that shall be used for the Docker container (\code{FROM} instruction). +By default, the image is determinded from the given session. Alternatively, use \code{getImageForVersion(..)} to get an existing image for a manually defined version of R, matching the version with tags from the base image rocker/r-ver (see details about the rocker/r-ver at \url{'https://hub.docker.com/r/rocker/r-ver/'}). Or provide a correct image name yourself.} \item{maintainer}{Specify the maintainer of the dockerfile. See documentation at \url{'https://docs.docker.com/engine/reference/builder/#maintainer'}. Defaults to \code{Sys.info()[["user"]]}. Can be removed with \code{NULL}.} -\item{r_version}{(character) optionally specify the R version that should run inside the container. By default, the R version from the given sessioninfo is used (if applicable) or the version of the currently running R instance} - -\item{image}{(From-object or character) optionally specify the image that shall be used for the Docker container (FROM-statement) -By default, the image is determinded from the given r_version, while the version is matched with tags from the base image rocker/r-ver -see details about the rocker/r-ver at \url{'https://hub.docker.com/r/rocker/r-ver/'}} +\item{save_image}{When TRUE, it calls \link[base]{save.image} and include the resulting .RData in the container's working directory. + Alternatively, you can pass a list of objects to be saved, which may also include arguments to be passed down to \code{save}. E.g. save_image = list("object1", "object2", file = "path/in/wd/filename.RData"). +\code{save} will be called with default arguments file = ".RData" and envir = .GlobalEnv} \item{env}{optionally specify environment variables to be included in the image. See documentation: \url{'https://docs.docker.com/engine/reference/builder/#env}} @@ -48,6 +46,8 @@ see details about the rocker/r-ver at \url{'https://hub.docker.com/r/rocker/r-ve \item{silent}{Whether or not to print information during execution} \item{versioned_libs}{[EXPERIMENTAL] Whether it shall be attempted to match versions of linked external libraries} + +\item{versioned_packages}{[EXPERIMENTAL] Whether it shall be attempted to match versions of R packages} } \value{ An object of class Dockerfile diff --git a/man/getGitHubRef.Rd b/man/getGitHubRef.Rd index fc3e71c..3cf3963 100644 --- a/man/getGitHubRef.Rd +++ b/man/getGitHubRef.Rd @@ -4,16 +4,24 @@ \alias{getGitHubRef} \title{Get GitHub reference from package} \usage{ -getGitHubRef(pkg) +getGitHubRef(pkg, pkgs = c(sessionInfo()$otherPkgs, sessionInfo()$loadedOnly)) } \arguments{ -\item{pkg}{The name of the package to retrieve the} +\item{pkg}{The name of the package to retrieve the reference for} + +\item{pkgs}{Lists of packages from a sessionInfo object} } \value{ -A character string with a short refernce, e.g. \code{r-hub/sysreqs@481d263} +A character string with a short refernce, e.g. \code{r-hub/sysreqs@481d263}, \code{NA} is nothign could be found } \description{ -If a package is not installed from CRAN, this functions tries to determine if it was installed from GitHub using \code{\link[devtools]{session_info}}. +If a package is installed from GitHub this function tries to retrieve the reference (i.e. repository accoutn, name, and commit) from +} +\details{ +\enumerate{ + \item the provided sessionInfo + \item locally, and only if the package is installed (!), using \code{\link[devtools]{session_info}} +} } \examples{ \dontrun{ diff --git a/man/getRVersionTag.Rd b/man/getRVersionTag.Rd index 5a63806..7e7fdcb 100644 --- a/man/getRVersionTag.Rd +++ b/man/getRVersionTag.Rd @@ -2,12 +2,12 @@ % Please edit documentation in R/utility-functions.R \name{getRVersionTag} \alias{getRVersionTag} -\title{Get R version from a variety of sources in a string format used for image tags} +\title{Get R version in string format used for image tags} \usage{ getRVersionTag(from = NULL, default = R.Version()) } \arguments{ -\item{from}{the source to extract an R version: a `sessionInfo()` object} +\item{from}{the source to extract an R version: a `sessionInfo()` object, or an \code{Rdata} file with a \code{sessionInfo} object} \item{default}{if 'from' does not contain version information (e.g. its an Rscript), use this default version information.} } diff --git a/man/getSessionInfoFromRdata.Rd b/man/getSessionInfoFromRdata.Rd new file mode 100644 index 0000000..3d5c5ff --- /dev/null +++ b/man/getSessionInfoFromRdata.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utility-functions.R +\name{getSessionInfoFromRdata} +\alias{getSessionInfoFromRdata} +\title{Reads a \code{sessionInfo} object from an \code{Rdata} file} +\usage{ +getSessionInfoFromRdata(file) +} +\arguments{ +\item{file}{file path} +} +\value{ +An object of class \code{sessionInfo} +} +\description{ +Reads a \code{sessionInfo} object from an \code{Rdata} file +} +\examples{ +sessionInfo <- sessionInfo() +file <- tempfile(tmpdir = tempdir(), fileext = ".Rdata") +save(sessionInfo, file = file) +getSessionInfoFromRdata(file) +} diff --git a/tests/testthat/github/Dockerfile b/tests/testthat/github/Dockerfile index 7b4e45e..09d76fb 100644 --- a/tests/testthat/github/Dockerfile +++ b/tests/testthat/github/Dockerfile @@ -1,9 +1,10 @@ FROM rocker/r-ver:3.3.2 LABEL maintainer="matthiashinz" RUN export DEBIAN_FRONTEND=noninteractive; apt-get -y update \ - && apt-get install -y git-core \ - libapparmor-dev + && apt-get install -y git-core \ + libapparmor-dev \ + NULL RUN ["install2.r", "assertthat", "backports", "clisymbols", "colorspace", "crayon", "debugme", "desc", "futile.logger", "futile.options", "jsonlite", "lambda.r", "magrittr", "munsell", "plyr", "R6", "Rcpp", "remotes", "rprojroot", "rstudioapi", "scales", "semver", "sessioninfo", "stringi", "stringr", "sys", "withr", "yaml"] -RUN ["installGithub.r", "r-hub/sysreqs@e4050e6", "wch/harbor@4e6ce36"] +RUN ["installGithub.r", "r-hub/sysreqs@e4050e6068655ce519bb39f0508c7f10e19b6f0b", "wch/harbor@4e6ce36dee3571f95a6c8ee7010e298e94bcd976"] WORKDIR /payload/ CMD ["R"] diff --git a/tests/testthat/package_markdown/sf_vignette_Dockerfile b/tests/testthat/package_markdown/sf_vignette_Dockerfile index 2cc08b7..6831901 100644 --- a/tests/testthat/package_markdown/sf_vignette_Dockerfile +++ b/tests/testthat/package_markdown/sf_vignette_Dockerfile @@ -1,7 +1,7 @@ FROM rocker/geospatial LABEL maintainer="matthiashinz" RUN export DEBIAN_FRONTEND=noninteractive; apt-get -y update \ - && apt-get install -y libudunits2-dev \ + && apt-get install -y libudunits2-dev \ pandoc \ pandoc-citeproc RUN ["install2.r", "dplyr", "sf", "Rcpp", "assertthat", "digest", "rprojroot", "R6", "DBI", "backports", "magrittr", "evaluate", "units", "rlang", "stringi", "rmarkdown", "udunits2", "stringr", "glue", "yaml", "htmltools", "knitr", "tibble"] diff --git a/tests/testthat/package_markdown/units_Dockerfile b/tests/testthat/package_markdown/units_Dockerfile index b8fef3d..7a8d5eb 100644 --- a/tests/testthat/package_markdown/units_Dockerfile +++ b/tests/testthat/package_markdown/units_Dockerfile @@ -1,7 +1,7 @@ FROM rocker/r-ver:3.3.2 LABEL maintainer="Ted Tester" RUN export DEBIAN_FRONTEND=noninteractive; apt-get -y update \ - && apt-get install -y git-core \ + && apt-get install -y git-core \ libudunits2-dev \ pandoc \ pandoc-citeproc diff --git a/tests/testthat/script_gstat/Dockerfile b/tests/testthat/script_gstat/Dockerfile index aa9d7f7..dc947a9 100644 --- a/tests/testthat/script_gstat/Dockerfile +++ b/tests/testthat/script_gstat/Dockerfile @@ -1,7 +1,7 @@ FROM rocker/r-ver:3.3.2 LABEL maintainer="matthiashinz" RUN export DEBIAN_FRONTEND=noninteractive; apt-get -y update \ - && apt-get install -y gdal-bin \ + && apt-get install -y gdal-bin \ libgdal-dev \ libproj-dev RUN ["install2.r", "FNN", "gstat", "intervals", "lattice", "rgdal", "sp", "spacetime", "xts", "zoo"] diff --git a/tests/testthat/test_dockerfile-method.R b/tests/testthat/test_dockerfile-method.R index 8b5b36b..5d7d38e 100644 --- a/tests/testthat/test_dockerfile-method.R +++ b/tests/testthat/test_dockerfile-method.R @@ -8,7 +8,7 @@ test_that("dockerfile object can be saved to file", { dir.create(t_dir) gen_file <- paste(t_dir, "Dockerfile", sep = "/") - dfile <- dockerfile(from = NULL, maintainer = NULL, r_version = "3.4.1") + dfile <- dockerfile(from = NULL, maintainer = NULL, image = getImageForVersion("3.4.1")) write(dfile, file = gen_file) control_file <- "./Dockerfile" @@ -51,26 +51,18 @@ test_that("users can specify the base image", { expect_equal(as.character(slot(dfile1, "image")), fromstr) #check if from - instruction is the first (may be necessary to ignore comments in later tests) expect_length(which(toString(dfile1) == fromstr), 1) - #expect that custom image is preferred over R version argument - dfile2 <- - dockerfile(from = NULL, - image = imagestr, - r_version = "3.1.0") - expect_equal(as.character(slot(dfile2, "image")), fromstr) - #expect_equal(toString(slot(dfile2, "image")), fromstr) - expect_length(which(toString(dfile2) == fromstr), 1) }) test_that("users can specify the R version", { versionstr <- "3.1.0" - dfile <- dockerfile(from = NULL, r_version = versionstr) + dfile <- dockerfile(from = NULL, image = getImageForVersion(versionstr)) #check content of image and instructions slots expect_equal(toString(slot(slot(dfile, "image"), "postfix")), versionstr) expect_match(toString(dfile), versionstr, all = FALSE) }) test_that("users are warned if an unsupported R version is set", { - expect_warning(dockerfile(from = NULL, r_version = "2.0.0"), "closest") + expect_warning(dockerfile(from = NULL, image = getImageForVersion("2.0.0")), "returning closest match") }) test_that("R version is the current version if not specified otherwise", { diff --git a/tests/testthat/test_find_systemrequirements.R b/tests/testthat/test_find_systemrequirements.R index 4860ee9..15aef3d 100644 --- a/tests/testthat/test_find_systemrequirements.R +++ b/tests/testthat/test_find_systemrequirements.R @@ -2,22 +2,33 @@ context("find system requrirements") -test_that("system requirements for CRAN packages can be determinded OFFLINE", { - deps <- .find_system_dependencies("sp", platform = .debian_platform, soft = TRUE, offline = TRUE) +test_that("system requirements for selected CRAN packages can be determinded OFFLINE", { + deps <- .find_system_dependencies("sp", + platform = .debian_platform, + soft = TRUE, + offline = TRUE) deps_expected <- c("libproj-dev", "libgdal-dev", "gdal-bin", "libgeos-dev") expect_true(all(deps %in% deps_expected)) expect_true(all(deps_expected %in% deps)) - deps <- .find_system_dependencies("sp", platform = .debian_platform, soft = FALSE, offline = TRUE) + deps <- containerit:::.find_system_dependencies("sp", + platform = containerit:::.debian_platform, + soft = FALSE, + offline = TRUE) expect_equal(deps, character(0)) #no direct dependencies - deps <- .find_system_dependencies("rgdal", platform = .debian_platform, soft = TRUE, offline = TRUE) + deps <- .find_system_dependencies("rgdal", + platform = .debian_platform, + soft = TRUE, + offline = TRUE) deps_expected <- c("libgdal-dev", "gdal-bin", "libproj-dev") expect_true(all(deps %in% deps_expected)) expect_true(all(deps_expected %in% deps)) - deps <- .find_system_dependencies("rgdal", platform = .debian_platform, offline = TRUE) + deps <- .find_system_dependencies("rgdal", + platform = .debian_platform, + offline = TRUE) deps_expected <- c("libgdal-dev", "gdal-bin", "libproj-dev") expect_true(all(deps %in% deps_expected)) expect_true(all(deps_expected %in% deps)) diff --git a/tests/testthat/test_install_github.R b/tests/testthat/test_install_github.R index 649efd6..8c53c18 100644 --- a/tests/testthat/test_install_github.R +++ b/tests/testthat/test_install_github.R @@ -11,7 +11,7 @@ test_that("github_packages can be installed", { #library(c("harbor", "sysreqs")); github_test_sessionInfo <- sessionInfo(); save(github_test_sessionInfo, file = "sessionInfo.Rdata") load("./github/sessionInfo.Rdata") - df = dockerfile(github_test_sessionInfo, maintainer = "matthiashinz", r_version = "3.3.2") + df = dockerfile(github_test_sessionInfo, maintainer = "matthiashinz", image = getImageForVersion("3.3.2")) write(df, "./github/Dockerfile") expected_file <- readLines("./github/Dockerfile") @@ -19,8 +19,8 @@ test_that("github_packages can be installed", { expect_equal(generated_file, expected_file) }) -test_that("GitHub references can be retrieved (using package sysreqs); test fails if package was installed locally from source", { +test_that("GitHub references can be retrieved for package sysreqs (test fails if package was installed from source)", { skip_if_not_installed("sysreqs") - ref <- getGitHubRef("sysreqs") + ref <- getGitHubRef("sysreqs", c(sessionInfo()$otherPkgs, sessionInfo()$loadedOnly)) expect_match(ref, "r-hub/sysreqs@([a-f0-9]{7})") }) diff --git a/tests/testthat/test_package_markdown.R b/tests/testthat/test_package_markdown.R index 35b7134..0ba5c80 100644 --- a/tests/testthat/test_package_markdown.R +++ b/tests/testthat/test_package_markdown.R @@ -7,7 +7,7 @@ test_that("A markdown file can be packaged (using markdowntainer-units-expample) df <- dockerfile(from = "package_markdown/markdowntainer-units/", maintainer = "Ted Tester", - r_version = "3.3.2", + image = getImageForVersion("3.3.2"), copy = "script_dir", cmd = CMD_Render("package_markdown/markdowntainer-units/2016-09-29-plot_units.Rmd")) #write(df, "package_markdown/units_Dockerfile") @@ -58,19 +58,21 @@ test_that("The render command supports output directory and output file at the s expect_equal(stringr::str_count(toString(cmd), 'output_file = \\\\\\"myfile.html\\\\\\"'), 1) }) -test_that("The file is automatically copied", { - df_copy <- dockerfile(from = "package_markdown/markdowntainer-units/") +test_that("The file is copied", { + df_copy <- dockerfile(from = "package_markdown/markdowntainer-units/", copy = "script") expect_true(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "at least one Copy instruction") }) -test_that("File copying can be disabled with NA", { - df_copy <- dockerfile(from = "package_markdown/markdowntainer-units/", copy = NA) +test_that("File copying is disabled by default", { + df_copy <- dockerfile(from = "package_markdown/markdowntainer-units/") expect_false(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "no Copy instruction") }) -test_that("File copying can be disabled with NA_character", { +test_that("File copying can be disabled with NA/NA_character", { df_copy <- dockerfile(from = "package_markdown/markdowntainer-units/", copy = NA_character_) expect_false(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "no Copy instruction") + df_copy2 <- dockerfile(from = "package_markdown/markdowntainer-units/", copy = NA_character_) + expect_false(object = any(sapply(df_copy2@instructions, function(x) { inherits(x, "Copy") })), info = "no Copy instruction") }) test_that("File copying can be disabled with NULL", { diff --git a/tests/testthat/test_package_script.R b/tests/testthat/test_package_script.R index eee4291..8e5a7c3 100644 --- a/tests/testthat/test_package_script.R +++ b/tests/testthat/test_package_script.R @@ -11,7 +11,7 @@ test_that("an R script can be created with resources of the same folder ",{ copy = "script_dir", cmd = CMD_Rscript("script_resources/simple_test.R"), maintainer = "matthiashinz", - r_version = "3.3.2") + image = getImageForVersion("3.3.2")) #for overwriting #write(df, "script_resources/Dockerfile") @@ -31,7 +31,7 @@ test_that("a workspace with one R script can be packaged ",{ copy = "script_dir", cmd = CMD_Rscript("script_resources/simple_test.R"), maintainer = "matthiashinz", - r_version = "3.3.2") + image = getImageForVersion("3.3.2")) expected_file <- readLines("script_resources/Dockerfile") expect_equal(toString(df), expected_file) @@ -43,7 +43,7 @@ test_that("a list of resources can be packaged ",{ "script_resources/test_table.csv", "script_resources/test_subfolder/testresource"), maintainer = "matthiashinz", - r_version = "3.3.2") + image = getImageForVersion("3.3.2")) #for overwriting #write(df, "script_resources/Dockerfile2") expected_file <- readLines("script_resources/Dockerfile2") @@ -57,13 +57,14 @@ test_that("there is an error if non-existing resources are to be packages",{ }) test_that("The gstat demo 'zonal' can be packaged ",{ - expect_true(requireNamespace("sp")) - expect_true(requireNamespace("gstat")) + skip_if_not_installed("sp") + skip_if_not_installed("gstat") df <- dockerfile("script_gstat/zonal.R", cmd = CMD_Rscript("script_gstat/zonal.R"), maintainer = "matthiashinz", - r_version = "3.3.2") + image = getImageForVersion("3.3.2"), + copy = "script") #for overwriting #write(df, "script_gstat/Dockerfile") @@ -73,29 +74,36 @@ test_that("The gstat demo 'zonal' can be packaged ",{ expect_equal(generated_file, expected_file) }) -test_that("The file is automatically copied", { - df_copy <- dockerfile(from = "script_resources/simple_test.R") +test_that("The file can be copied", { + df_copy <- dockerfile(from = "script_resources/simple_test.R", copy = "script") expect_true(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "at least one Copy instruction") }) -test_that("File copying can be disabled with NA", { +test_that("File copying is disabled by default", { + df_copy <- dockerfile(from = "script_resources/simple_test.R") + expect_false(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "no Copy instruction") +}) + +test_that("File copying is disabled with NA", { df_copy <- dockerfile(from = "script_resources/simple_test.R", copy = NA) expect_false(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "no Copy instruction") }) -test_that("File copying can be disabled with NA_character", { +test_that("File copying is disabled with NA_character", { df_copy <- dockerfile(from = "script_resources/simple_test.R", copy = NA_character_) expect_false(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "no Copy instruction") }) -test_that("File copying can be disabled with NULL", { +test_that("File copying is disabled with NULL", { df_copy <- dockerfile(from = "script_resources/simple_test.R", copy = NULL) expect_false(object = any(sapply(df_copy@instructions, function(x) { inherits(x, "Copy") })), info = "no Copy instruction") }) test_that("the installation order of packages is alphabetical (= reproducible)", { - df <- dockerfile("script_packages/", maintainer = "o2r", image = getImageForVersion("3.4.3", nearest = FALSE)) + df <- dockerfile("script_packages/", maintainer = "o2r", + image = getImageForVersion("3.4.3", nearest = FALSE), + copy = "script") expected_file = readLines("script_packages/Dockerfile") expect_equal(toString(df), expected_file) }) diff --git a/tests/testthat/test_package_sessionInfo-file.R b/tests/testthat/test_package_sessionInfo-file.R new file mode 100644 index 0000000..9e7ccab --- /dev/null +++ b/tests/testthat/test_package_sessionInfo-file.R @@ -0,0 +1,99 @@ +# Copyright 2017 Opening Reproducible Research (http://o2r.info) + +context("Packaging sessionInfo saved in .Rdata file") + +# create the test data, also a good way to understand what information from session info is actually used! +rdata_file <- tempfile(pattern = "containerit_", tmpdir = tempdir(), fileext = ".Rdata") + +sessionInfo <- list() +sessionInfo$R.version <- list() +sessionInfo$R.version$major <- "1" +sessionInfo$R.version$minor <- "2.3" + +sessionInfo$basePkgs <- c("stats", "graphics", "grDevices", "utils", "datasets", "methods", "base") + +sessionInfo$otherPkgs <- list() +sessionInfo$otherPkgs$testpkg1 <- list() +sessionInfo$otherPkgs$testpkg1$Version <- "1.0" +sessionInfo$otherPkgs$testpkg1$Package <- "testpkg1" +sessionInfo$otherPkgs$testpkg1$RemoteType <- "github" +sessionInfo$otherPkgs$testpkg1$RemoteRepo <- "pkg1" +sessionInfo$otherPkgs$testpkg1$RemoteUsername <- "test" +sessionInfo$otherPkgs$testpkg1$RemoteSha <- "123456abcdef" +sessionInfo$otherPkgs$testpkg2 <- list() +sessionInfo$otherPkgs$testpkg2$Version <- "1.2.3" +sessionInfo$otherPkgs$testpkg2$Package <- "testpkg2" +sessionInfo$otherPkgs$testpkg2$Repository <- "CRAN" +sessionInfo$otherPkgs$testpkg3$Version <- "1.0" +sessionInfo$otherPkgs$testpkg3$Package <- "testpkg3" +sessionInfo$otherPkgs$testpkg3$RemoteType <- "github" +sessionInfo$otherPkgs$testpkg3$GithubRepo <- "pkg3" +sessionInfo$otherPkgs$testpkg3$GithubUsername <- "test" +sessionInfo$otherPkgs$testpkg3$GithubSHA1 <- "a1b2c3d4e5f6" + +sessionInfo$loadedOnly <- list() +sessionInfo$otherPkgs$loadedA <- list() +sessionInfo$otherPkgs$loadedA$Version <- "0.1.2" +sessionInfo$otherPkgs$loadedA$Package <- "loadedA" +sessionInfo$otherPkgs$loadedA$Repository <- "CRAN" +sessionInfo$otherPkgs$loadedB <- list() +sessionInfo$otherPkgs$loadedB$Version <- "1.0-42" +sessionInfo$otherPkgs$loadedB$Package <- "loadedB" +sessionInfo$otherPkgs$loadedB$Repository <- "CRAN" + +class(sessionInfo) <- "sessionInfo" +save(sessionInfo, file = rdata_file) + +test_that("can create dockerfile object from the file with sessionInfo", { + df <- dockerfile(from = rdata_file) + expect_s4_class(df, "Dockerfile") +}) + +df_test <- dockerfile(from = rdata_file) + +test_that("dockerfile object contains expected R version", { + expect_equal(toString(df_test)[1], "FROM rocker/r-ver:3.1.0") +}) + +test_that("dockerfile contains CRAN packages", { + expect_true(any(stringr::str_detect(toString(df_test), + "^RUN \\[\"install2.r\", \"loadedA\", \"loadedB\", \"remotes\", \"testpkg2\"\\]$"))) +}) + +test_that("dockerfile contains GitHub packages", { + expect_true(any(stringr::str_detect(toString(df_test), + "^RUN \\[\"installGithub.r\", \"test/pkg1@123456abcdef\", \"test/pkg3@a1b2c3d4e5f6\"\\]$"))) +}) + +test_that("dockerfile contains package versions", { + # FIXME not implmented yet +}) + +test_that("GitHub reference can be retrieved from sessionInfo", { + ref <- getGitHubRef("testpkg1", c(sessionInfo$otherPkgs, sessionInfo$loadedOnly)) + expect_equal(ref, "test/pkg1@123456abcdef") +}) + +test_that("R version can be retrieved from sessionInfo", { + ver <- getRVersionTag(sessionInfo) + expect_equal(ver, "1.2.3") +}) + +test_that("error if Rdata file contains more than one object", { + file <- tempfile(tmpdir = tempdir(), fileext = ".Rdata") + a <- "1" + b <- "2" + save(a, sessionInfo, b, file = file) + expect_error(getSessionInfoFromRdata(file), "only one") +}) + +test_that("Matching image can be retrieved from sessionInfo, with a warning", { + expect_warning(getImageForVersion(getRVersionTag(sessionInfo)), "No Docker image found") + image <- getImageForVersion(getRVersionTag(sessionInfo)) + expect_equal(toString(image), "FROM rocker/r-ver:3.1.0") +}) + +test_that("Version tag can be retrieved from sessionInfo.Rdata file, with a warning", { + ver <- getRVersionTag(rdata_file) + expect_equal(ver, "1.2.3") +}) diff --git a/tests/testthat/test_sessioninfo_reproduce.R b/tests/testthat/test_sessioninfo_reproduce.R index 1dad0fd..287a5c3 100644 --- a/tests/testthat/test_sessioninfo_reproduce.R +++ b/tests/testthat/test_sessioninfo_reproduce.R @@ -205,7 +205,7 @@ test_that("the locales are the same ", { }) # manual comparison -if(FALSE) { +if (FALSE) { cat("\nlocal sessionInfo: \n\n") print(local_sessionInfo) cat("\n------------------------------------") diff --git a/vignettes/containerit.Rmd b/vignettes/containerit.Rmd index 47f7244..d48b123 100644 --- a/vignettes/containerit.Rmd +++ b/vignettes/containerit.Rmd @@ -345,7 +345,7 @@ Note that while choosing an R version for the Dockerfile explicitly is possible, The following examples show usage of these options and the respective `FROM` statements in the Dockerfile. ```{r base_image, comment=NA} -df_custom <- dockerfile(from = NULL, r_version = "3.1.0", silent = TRUE) +df_custom <- dockerfile(from = NULL, image = getImageForVersion("3.3.3"), silent = TRUE) print(df_custom@image) df_custom <- dockerfile(from = NULL, image = "rocker/geospatial", silent = TRUE) print(df_custom@image) @@ -379,7 +379,7 @@ CLI Examples: # Creates an empty R session with the given R commands # Set R version of the container to 3.3.0 - containerit session -p -e "library(sp)" -e "demo(meuse, ask=FALSE)" --r_version 3.3.0 + containerit session -p -e "library(sp)" -e "demo(meuse, ask=FALSE)" ``` ## 7. Challenges