Updated slide preprocessing

.github/workflows/main.yml

@@ -3,78 +3,47 @@ on:
   push:
     branches-ignore:
       - gh-pages
+env:
+  REGISTRY: ghcr.io
+  ORG: nbisweden
+  IMAGE: workshop-rnaseq:latest
+
 jobs:
   rmd-render-job:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - {os: ubuntu-18.04, r: '4.1'}
-    runs-on: ${{ matrix.config.os }}
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v1
-      - uses: r-lib/actions/setup-r@old
-        with:
-          r-version: ${{ matrix.config.r }}
-      - uses: r-lib/actions/setup-pandoc@old
+      - name: Set time zone  
+        uses: szenius/[email protected]
         with:
-          r-version: ${{ matrix.config.r }}
-      - name: Install linux dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libxml2-dev libssl-dev libcurl4-openssl-dev libudunits2-dev libopenblas-base
-          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-          sudo apt install ./google-chrome-stable_current_amd64.deb
-          rm -rf google-chrome-stable_current_amd64.deb
+          timezoneLinux: "Europe/Stockholm"
+          timezoneMacos: "Europe/Stockholm"
+          timezoneWindows: "W. Europe Standard Time"
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Pull image
+        run: docker pull ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }}
       - name: Define variables
         run: |
-          echo "path_repo=$(pwd)" >> $GITHUB_ENV
-          echo "path_export=$(grep -E '^output_dir' _site.yml | sed 's/^output_dir://' | xargs)" >> $GITHUB_ENV
-          echo "path_export_canvas=$(grep -E '^output_dir' _site.yml | sed 's/^output_dir://' | xargs)-canvas" >> $GITHUB_ENV
-          echo "repo=$(echo $GITHUB_REPOSITORY | sed 's/.*\///')" >> $GITHUB_ENV
-      - name: Check R installation and variables
+          echo "PATH_REPO=$(pwd)" >> $GITHUB_ENV
+          echo "PATH_EXPORT=$(grep -E '^output_dir' _site.yml | sed 's/^output_dir://' | xargs)" >> $GITHUB_ENV
+          echo "PATH_EXPORT_CANVAS=$(grep -E '^output_dir' _site.yml | sed 's/^output_dir://' | xargs)-canvas" >> $GITHUB_ENV
+          echo "REPO=$(basename ${GITHUB_REPOSITORY})" >> $GITHUB_ENV
+      - name: Check docker and variables
         run: |
-          Rscript -e "print('R installation works!')"
-          echo "Github actor: ${GITHUB_ACTOR}"
-          echo "Github repository: ${GITHUB_REPOSITORY}"
-          echo "Github repo: ${repo}"
-          echo "Path repo: ${path_repo}"
-          echo "Path export: ${path_export}"
-          echo "Path export canvas: ${path_export_canvas}"
+          docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }} Rscript -e "print('R in docker works!')"
+          echo "GITHUB_ACTOR: ${GITHUB_ACTOR}"
+          echo "GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}"
+          echo "REPO: ${REPO}"
+          echo "env.PATH_REPO: ${{ env.PATH_REPO }}"
+          echo "PATH_EXPORT: ${PATH_EXPORT}"
+          echo "PATH_EXPORT_CANVAS: ${PATH_EXPORT_CANVAS}"
+          echo "GITHUB_REPOSITORY_OWNER: ${GITHUB_REPOSITORY_OWNER}"
           echo "---------------------------------------------------------------------------------------------"
           echo "Folder contents at $(pwd):"
           ls -lh
           echo "---------------------------------------------------------------------------------------------"
           printenv
           sed --version
-      - name: Print R packages
-        run: |
-          # parse r packages from Rmd amd md files
-          # grep -Eroh "(^library\([A-Za-z0-9]+\))|(^require\([A-Za-z0-9]+\))" *md | sed 's/library\|(\|)//g' | sort | uniq > r-packages.txt
-          # parse r packages from _site.yml
-          awk '/^[^ ]/{ f=/^packages:/; next } f{ if (sub(/:$/,"")) pkg=$2; else print pkg, $2 }' _site.yml | uniq | sort > r-packages.txt
-          cat r-packages.txt
-      - name: Cache R packages
-        id: cache-rp
-        uses: actions/cache@v2
-        with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ matrix.config.r }}-${{ hashFiles('r-packages.txt') }}
-      - name: Install R dependencies
-        if: steps.cache-rp.outputs.cache-hit != 'true'
-        env:
-          GITHUB_PAT: ${{ secrets.PAT }}
-        run: |
-          Rscript -e 'install.packages(c("yaml","pagedown"))'
-          Rscript -e 'if(!is.null(yaml::read_yaml("_site.yml")$packages$packages_cran_repo)) install.packages(yaml::read_yaml("_site.yml")$packages$packages_cran_repo)'
-          Rscript -e 'if(!is.null(yaml::read_yaml("_site.yml")$packages$packages_cran_student)) install.packages(yaml::read_yaml("_site.yml")$packages$packages_cran_student)'
-          Rscript -e 'if(!is.null(yaml::read_yaml("_site.yml")$packages$packages_bioc_repo)) BiocManager::install(as.character(yaml::read_yaml("_site.yml")$packages$packages_bioc_repo))'
-          Rscript -e 'if(!is.null(yaml::read_yaml("_site.yml")$packages$packages_bioc_student)) BiocManager::install(as.character(yaml::read_yaml("_site.yml")$packages$packages_bioc_student))'
-          Rscript -e 'if(!is.null(yaml::read_yaml("_site.yml")$packages$packages_github_repo)) remotes::install_github(yaml::read_yaml("_site.yml")$packages$packages_github_repo)'
-          Rscript -e 'if(!is.null(yaml::read_yaml("_site.yml")$packages$packages_github_student)) remotes::install_github(yaml::read_yaml("_site.yml")$packages$packages_github_student)'
-          Rscript -e 'installed.packages()'
-          echo "R directory size:"
-          du -sh ${{ env.R_LIBS_USER }}
       - name: Render site, pull repo, copy folder, create index, push repo
         run: |
           # Add git info
@@ -83,34 +52,32 @@ jobs:
 
           # Render website
           echo "Rendering Rmd files ..."
-          Rscript -e "rmarkdown::render_site()"
+          docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }}
+          # Generate PDFs
+          # cd ${PATH_EXPORT}
+          # echo "Generating PDFs ..."
+          # docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }} Rscript -e "l <- list.files('.',pattern='slide.+html');if(length(l)!=0) {for(i in 1:length(l)) pagedown::chrome_print(l[i],extra_args=c('--no-sandbox','--disable-dev-shm-usage','--disable-gpu'))};"
+          # cd ..
 
           # Render canvas version of website by changing _site.yml
           # suffixes output directory with -canvas
           # adds current output directory to be ignored
           # renames navbar block to remove menu
           # removes footer
-          #sed 's/\(output_dir:\)\s*\([0-9]*\)/\1 \2-canvas/' _site.yml | \
-          #sed "/^exclude/s/\]$/,\"${path_export}\"]/" | \
-          #sed 's/navbar:/unused:/' | \
-          #sed 's/include://' | \
-          #sed 's/after_body: assets\/footer-lab.html//' > _site.yml
 
           echo "Removing lab and slide links from schedule for canvas ..."
-          Rscript -e 'x <- read.csv("schedule.csv",sep=";",header=T,stringsAsFactors=F);x$link_lab <- x$link_slide <- NA;write.table(x,"schedule.csv",sep=";",col.names=T,row.names=F,quote=FALSE,na="")'
+          docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }} Rscript -e 'x <- read.csv("schedule.csv",sep=";",header=T,stringsAsFactors=F);x$link_youtube <- x$link_lab <- x$link_slide <- NA;write.table(x,"schedule.csv",sep=";",col.names=T,row.names=F,quote=FALSE,na="")'
           echo "Fixing _site.yml for canvas ..."
-          Rscript -e 'y <- yaml::read_yaml("_site.yml"); y$exclude <- append(y$exclude,y$output_dir); y$output_dir <- paste0(y$output_dir,"-canvas"); y$navbar <- NULL; y$output$`bookdown::html_document2`$include <- NULL; yaml::write_yaml(y,"_site.yml");'
+          docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }} Rscript -e 'y <- yaml::read_yaml("_site.yml"); y$exclude <- append(y$exclude,y$output_dir); y$output_dir <- paste0(y$output_dir,"-canvas"); y$navbar <- NULL; yaml::write_yaml(y,"_site.yml");'
 
           ls -lh
           cat _site.yml
           echo "Rendering Rmd files for canvas ..."
-          Rscript -e "rmarkdown::render_site()"
+          docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }}
 
-          # Modify links and generate PDFs
-          cd ${path_export}
-          Rscript -e "l <- list.files('.',pattern='slide.+html');if(length(l)!=0) {for(i in 1:length(l)) pagedown::chrome_print(l[i])};"
-          cd ..
-          cd ${path_export_canvas}
+          cd ${PATH_EXPORT_CANVAS}
+
+          # Fix links
           # all links (except those starting with #) in all html files are set to open in new tab
           # all links (those pointing within website and ends with html) in all html files are removed
           # display links to be fixed
@@ -131,47 +98,43 @@ jobs:
               rm -rf temp
           done
           echo "Generating PDFs for HTML slides ..."
-          Rscript -e "l <- list.files('.',pattern='slide.+html');if(length(l)!=0) {for(i in 1:length(l)) pagedown::chrome_print(l[i])};"
+          docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }} Rscript -e "l <- list.files('.',pattern='slide.+html');if(length(l)!=0) {for(i in 1:length(l)) {cat(paste('Running',l[i],'\n')); pagedown::chrome_print(l[i],extra_args=c('--no-sandbox','--disable-dev-shm-usage','--disable-gpu'));}}"
           cd ..
 
           # List folder contents
           echo "Folder contents at $(pwd):"
           ls -lh
-          echo "Folder contents at ${path_repo}/${path_export}:"
-          ls -lh ${path_repo}/${path_export}
-          echo "Folder contents at ${path_repo}/${path_export_canvas}:"
-          ls -lh ${path_repo}/${path_export_canvas}
+          echo "Folder contents at ${PATH_REPO}/${PATH_EXPORT}:"
+          ls -lh ${PATH_REPO}/${PATH_EXPORT}
+          echo "Folder contents at ${PATH_REPO}/${PATH_EXPORT_CANVAS}:"
+          ls -lh ${PATH_REPO}/${PATH_EXPORT_CANVAS}
 
           # Clone gh-pages branch and remove old directories with same name
           git clone --single-branch --branch gh-pages "https://${{ secrets.TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" tmprepo
           cd tmprepo
-          if [ -d $path_export ]; then
-            echo "Directory ${path_export} already exists. Removing the directory."
-            git rm -r $path_export
-            git commit -m "Old directory ${path_export} deleted."
+          if [ -d ${PATH_EXPORT} ]; then
+            echo "Directory ${PATH_EXPORT} already exists. Removing the directory."
+            git rm -r ${PATH_EXPORT}
+            git commit -m "Old directory ${PATH_EXPORT} deleted."
           fi
-          if [ -d $path_export_canvas ]; then
-            echo "Directory ${path_export_canvas} already exists. Removing the directory."
-            git rm -r $path_export_canvas
-            git commit -m "Old directory ${path_export_canvas} deleted."
+          if [ -d ${PATH_EXPORT}_canvas ]; then
+            echo "Directory ${PATH_EXPORT_CANVAS} already exists. Removing the directory."
+            git rm -r ${PATH_EXPORT}_canvas
+            git commit -m "Old directory ${PATH_EXPORT_CANVAS} deleted."
           fi
 
-          cd ${path_repo}
-          cp -r $path_export tmprepo/
-          cp -r $path_export_canvas tmprepo/
+          cd ${PATH_REPO}
+          cp -r ${PATH_EXPORT} tmprepo/
+          cp -r ${PATH_EXPORT_CANVAS} tmprepo/
           cd tmprepo
           echo "Folder contents at $(pwd):"
           ls -lh
 
-          # Create index file
-          echo "Full repo: ${GITHUB_REPOSITORY}"
-          str_user=$(dirname ${GITHUB_REPOSITORY})
-          echo "User: $str_user"
-          repo=$(basename ${GITHUB_REPOSITORY})
-          echo "Repo: $repo"
-
-          printf "All current and previous workshop materials are listed below.\n\n" > index.md
-          printf "<div style='display:block;'><p style='line-height:2;'>\n" >> index.md
+          # create index file
+          printf "<div class='wrapper-logo'><img class='logo' src='assets/logo.svg'></div>" > index.md
+          printf "The current and all previous instances of this workshop are listed below.\n" >> index.md
+          printf "<hr>\n" >> index.md
+          printf "<div class='workshop-list'>\n" >> index.md
           dirs=$(ls -d [0-9]*[0-9]/ | sed 's/\///' | tac)
           for i in ${dirs[@]}
           do
@@ -182,9 +145,11 @@ jobs:
           	printf -v thedate '2000-01-01 +%s years +%s months -1 day' "${i:0:2}" "${i:2}"
           	dt=$(date -d "$thedate" +'%Y %b')
           	echo $dt
-          	printf "<span style='display:block;'><a href='https://${str_user}.github.io/${repo}/${i}/'>$dt</a></span>" >> index.md
+            printf "<h3><a href='https://${GITHUB_REPOSITORY_OWNER}.github.io/${REPO}/${i}/'>$dt</a></h3>" >> index.md
           done
-          printf "</p></div>" >> index.md
+          printf "</div>\n" >> index.md
+          # printf "<hr>\n" >> index.md
+          # printf "$(date +'%Y') • NBIS\n" >> index.md
 
           echo "Folder contents at $(pwd):"
           ls -lh

.gitignore

@@ -1,4 +1,9 @@
 .Rhistory
+.Rproj.user
+[0-9]*
+data/.DS*
+*.Rproj
 libs/
 *_files/
 *.html
+.DS_Store

Dockerfile

@@ -0,0 +1,42 @@
+# DOCKER FILE FOR WORKSHOP-RNASEQ
+# 2023 Roy Francis
+
+FROM rocker/verse:4.2.3
+LABEL Description="Docker image for NBIS workshop-rnaseq"
+LABEL Maintainer="[email protected]"
+LABEL org.opencontainers.image.source="https://github.com/NBISweden/workshop-rnaseq"
+
+RUN apt-get update -y \
+  && apt-get install --no-install-recommends -y \
+  libxml2-dev \
+  libssl-dev \
+  libcurl4-openssl-dev \
+  libudunits2-dev \
+  libopenblas-base \
+  libgdal-dev \
+  && wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \
+  && apt-get install -y ./google-chrome-stable_current_amd64.deb \
+  && rm -rf google-chrome-stable_current_amd64.deb \
+  && rm -rf /var/lib/apt/lists/* \
+  && Rscript -e 'install.packages(c("bookdown","knitr","remotes","fontawesome","formattable","kableExtra","leaflet","pagedown","lubridate","markdown","xaringan","yaml"),repos = "https://packagemanager.posit.co/cran/__linux__/jammy/latest");' \
+  && mkdir /rmd
+
+RUN Rscript -e 'install.packages(c("BiocManager","dplyr","enrichR","ggplot2","gridExtra","pheatmap","pvclust","rafalib","stringr","tidyr","locfit"),repos = "https://packagemanager.posit.co/cran/__linux__/jammy/latest");'
+RUN Rscript -e 'BiocManager::install(c("DESeq2","edgeR","fgsea","biomaRt","GEOquery"));'
+WORKDIR /rmd
+CMD Rscript -e "rmarkdown::render_site()"
+
+# build container
+# docker build -t ghcr.io/nbisweden/workshop-rnaseq:1.0.0 .
+# docker tag ghcr.io/nbisweden/workshop-rnaseq:1.0.0 ghcr.io/nbisweden/workshop-rnaseq:latest
+
+# push to ghcr
+# docker login ghcr.io
+# docker push ghcr.io/nbisweden/workshop-rnaseq:1.0.0
+# docker push ghcr.io/nbisweden/workshop-rnaseq:latest
+
+# run container
+# render all files
+# docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ghcr.io/nbisweden/workshop-rnaseq:latest
+# render one file
+# docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ghcr.io/nbisweden/workshop-rnaseq:latest Rscript -e 'rmarkdown::render("index.Rmd")'

README.md

@@ -24,14 +24,37 @@ If you are not added as a collaborator, first fork this repo to your account, th
 
 :exclamation: Do not push any rendered .html files or intermediates.
 
+### Local build/preview using Docker
+
+You can preview changes and build the whole website locally without a local installation of R or dependency packages by using the pre-built Docker image.
+
+:exclamation: **Note:** Large image size: 4.6GB.
+
+Clone the repo if not already done. Make sure you are standing in the repo directory.
+
+To build the complete site,
+
+```
+docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ghcr.io/nbisweden/workshop-rnaseq:latest
+```
+
+To build a single file (for example `index.Rmd`),
+
+```
+docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) -v ${PWD}:/rmd ghcr.io/nbisweden/workshop-rnaseq:latest Rscript -e 'rmarkdown::render("index.Rmd")'
+```
+
+See **Dockerfile** to build the image.
+
+:exclamation: Output files are for local preview only. Do not push any rendered .html files or intermediates.
+
 ## Repo organisation
 
 The 3-day course source material is located on the *master* branch (default). The 2-day course material is located on the *twoday* branch. The rendered material is located on the *gh-pages* branch. For most part, one only needs to update content in master. Changes pushed to the *master* or *twoday* branches are automatically rendered to the *gh-pages* branch.
 
-:exclamation: The first build can take around 40 mins depending on the number of R packages. Subsequent builds take about 6-7 minutes since caching is enabled. Caches are removed after 7 days of last access. A push after that will require a full rebuild.
+:exclamation: Every push rebuilds the whole website using a docker image. Build takes about 6 mins.
 
-For more details about repo organisation, updating and modifying this repo, check out the [template repo](https://github.com/royfrancis/workshop-template-rmd-ga).
 
 ---
 
-**2021** • NBIS • SciLifeLab
+**2024** • NBIS • SciLifeLab

_site.yml

@@ -1,6 +1,6 @@
 name: Analysis of RNA-Seq data
 # output directory used on gh-pages branch. format: YYMM
-output_dir: 2303
+output_dir: 2403
 # used site-wide wherever uppmax project ids are used
 uppmax_project: g2020006
 uppmax_res_1: g2020006_1
@@ -24,24 +24,7 @@ navbar:
       href: home_content.html
     - text: Precourse
       href: home_precourse.html
-output:
-  bookdown::html_document2:
-    highlight: textmate
-    toc: true
-    toc_float:
-      collapsed: true
-      smooth_scroll: true
-      print: false
-    toc_depth: 4
-    number_sections: true
-    df_print: default
-    code_folding: none
-    self_contained: false
-    keep_md: false
-    encoding: 'UTF-8'
-    css: "assets/lab.css"
-    include:
-      after_body: assets/footer-lab.html
+
 # r packages needed for the workshop
 # used in home_precourse.Rmd
 # split into cran and bioc packages
@@ -69,7 +52,6 @@ packages:
  packages_github_student:
  packages_cran_repo:
   - bookdown
-  - captioner
   - formattable
   - kableExtra
   - knitr

assets/footer-lab.html

@@ -3,14 +3,14 @@
 <div class="container footer-container">
 <div class="row">
 
-<div class="col-sm-8" style="text-align:left;vertical-align:middle;">
-<p style="color:#bdbdbd;" class="small">
-<b>2021</b> • <a href="https://nbis.se/">NBIS</a> • <a href="https://www.scilifelab.se/">SciLifeLab</a>
+<div class="footer-left col-sm-6">
+<p class="small">
+<b>2024</b> • <a href="https://nbis.se/">NBIS</a> • <a href="https://www.scilifelab.se/">SciLifeLab</a>
 </p>
 </div>
 
-<div class="col-sm-4" style="text-align:right;vertical-align:middle;">
-<p style="color:#bdbdbd;">
+<div class="footer-right col-sm-6">
+<p>
 <span>
   <span class="footericon" style="padding-right:4px; padding-left:4px">
     <a href="https://nbis.se/">