.gitlab-ci.yml

image: $CI_REGISTRY/mouse-informatics/docker:latest

variables:
   # When using dind service we need to instruct docker, to talk with the
   # daemon started inside of the service. The daemon is available with
   # a network connection instead of the default /var/run/docker.sock socket.
   #
   # The 'docker' hostname is the alias of the service container as described at
   # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services
   #
   # Note that if you're using the Kubernetes executor, the variable should be set to
   # tcp://localhost:2375/ because of how the Kubernetes executor connects services
   # to the job container
   # DOCKER_HOST: tcp://localhost:2375/
   #
   # For non-Kubernetes executors, we use tcp://docker:2375/
   DOCKER_HOST: tcp://docker:2375/
   # When using dind, it's wise to use the overlayfs driver for
   # improved performance.
   DOCKER_DRIVER: overlay2
   
   # Since the docker:dind container and the runner container don’t share their root
   # filesystem, the job’s working directory can be used as a mount point for children
   # containers. For example, if you have files you want to share with a child container,
   # you may create a subdirectory under /builds/$CI_PROJECT_PATH and use it as your
   # mount point.
   MOUNT_POINT: /builds/$CI_PROJECT_PATH/mnt
   
   # For EBI you need to override the definition of CI_REGISTRY to remove the port number
   CI_REGISTRY: dockerhub.ebi.ac.uk
   CI_REGISTRY_IMAGE: $CI_REGISTRY/$CI_PROJECT_PATH

   #NOW: $(date '+%Y-%m-%d-%H-%M-%S')
   #NOW: $(date '+%Y-%m-%d')
   
   # To solve the issue with the Docker in Docker 19.03 service.
   # Logged as: GitLab.com CI jobs failing if using docker:stable-dind image
   # see: https://gitlab.com/gitlab-com/gl-infra/production/issues/982
   DOCKER_TLS_CERTDIR: ""
   SCAN_KUBERNETES_MANIFESTS: "true"


# Use this command to look at your docker environment
# Note: This step can be overwritten by before_script sections in specific jobs.
#
#before_script:
#   - docker info

include:
  - template: Security/SAST.gitlab-ci.yml
  - template: Security/Secret-Detection.gitlab-ci.yml


stages:
   - download
   - build
   - test
   - dev-deploy
   - dev-test
   - production-deploy
   - production-test


#
# Data download stage
#



Human_data:
    stage: download
    before_script:
        - apk add --update curl && rm -rf /var/cache/apk/*
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        # Fetch data files for the service
        
        # IDG        
        # - cp "$CI_PROJECT_DIR"/data/idg_out.txt "$MOUNT_POINT"/idg_out.txt
        
        # ClinGen
        # Note:2024-09 changed the link from download to downloadall
        - curl -sSLN -o gene-dosage.csv https://search.clinicalgenome.org/kb/gene-dosage/downloadall
        
        # DepMap Achilles_gene_effect.csv - DepMap Public 20Q4v2
        - curl -sSLN -o achilles_gene_effect.csv https://ndownloader.figshare.com/files/25770029
        
        # Process the Achilles_gene_effect.csv file
        - awk -f "$CI_PROJECT_DIR"/scripts/transpose_rows_to_cols.awk achilles_gene_effect.csv > achilles_gene_effect.col.tsv
        
        # Format the achilles file for loading
        # In the main body sed replaces parenthesis and spaces around the entrez acc id with delimiters, and removes any trailing tab and space characters.
        # In the header row that describes the cell lines the column heading DepMap_ID is removed with sed so the names can be loaded into a table.
        - cat achilles_gene_effect.col.tsv | sed -e 's/ (/|/g' | sed -e 's/[)]./|/g' | sed -e 's/[[:space:]]\{1,\}$//g' | sed -e 's/DepMap_ID.//g' > achilles_gene_effect.col.formatted.tsv
        

    artifacts:
        paths:
            - "$MOUNT_POINT/"



Pharos:
    stage: download
    image: $CI_REGISTRY/mouse-informatics/mysql:latest
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        # IDG        
        - mysql -h tcrd.kmc.io -u tcrd tcrd540 -e 'select t.name, t.tdl, t.fam, p.sym, p.uniprot, p.chr from target t, t2tc, protein p where t.id = t2tc.target_id and t2tc.protein_id = p.id' > pharos_out.txt
        

    artifacts:
        paths:
            - "$MOUNT_POINT/"



IDG_GitHub:
    stage: download
    image: $CI_REGISTRY/mouse-informatics/ubuntu:latest
    before_script:
        - apt-get update && apt-get install -y jq curl python3 && apt-get clean && rm -rf /var/lib/apt/lists/*
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        - source "$CI_PROJECT_DIR"/scripts/fetch_idg_data.sh
        # This is required if we switch to process IDG files from multiple years
        # It gathers the data into a single file 
        # containing unique entries for the database.
        # - python3 "$CI_PROJECT_DIR"/scripts/IDGPreProcessor.py      

    artifacts:
        paths:
            - "$MOUNT_POINT/"



IMPC_Viability:
    image: $CI_REGISTRY/mouse-informatics/ubuntu:latest
    stage: download
    before_script:
        - apt-get update && apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/*
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        - source "$CI_PROJECT_DIR"/scripts/extract_impc_viability_data.sh
        

    artifacts:
        paths:
            - "$MOUNT_POINT/"



IMPC_Phenotypes:
    image: $CI_REGISTRY/mouse-informatics/ubuntu:latest
    stage: download
    before_script:
        - apt-get update && apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/*
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        - source "$CI_PROJECT_DIR"/scripts/extract_impc_phenotype_data.sh
        

    artifacts:
        paths:
            - "$MOUNT_POINT/"



IMPC_Stats:
    image: $CI_REGISTRY/mouse-informatics/ubuntu:latest
    stage: download
    before_script:
        - apt-get update && apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/*
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        - source "$CI_PROJECT_DIR"/scripts/extract_impc_stats_data.sh
        

    artifacts:
        paths:
            - "$MOUNT_POINT/"



gnomAD_pLoF:
    image: $CI_REGISTRY/mouse-informatics/ubuntu:latest
    stage: download
    before_script:
        # - apt-get update && apt-get install -y autoconf automake make gcc perl zlib1g-dev libbz2-dev liblzma-dev libcurl4-gnutls-dev libssl-dev git curl && apt-get clean && rm -rf /var/lib/apt/lists/*
        - apt-get update && apt-get install -y tabix curl && apt-get clean && rm -rf /var/lib/apt/lists/*
        # - git clone https://github.com/samtools/htslib.git
        # - cd htslib/
        # - git submodule update --init --recursive
        # - autoheader && autoconf && ./configure && make && make install && cd ../
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        # gnomAD URL updated 20210126 to reflect gnomAD web site change
        - curl -sSLN -O https://storage.googleapis.com/gcp-public-data--gnomad/release/2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_gene.txt.bgz
        - bgzip -d gnomad.v2.1.1.lof_metrics.by_gene.txt.bgz
        - mv gnomad.* gnomad.lof_metrics.by_gene.txt
        

    artifacts:
        paths:
            - "$MOUNT_POINT/"




orthology_db_dump:
    image: $CI_REGISTRY/mouse-informatics/postgres:11-alpine
    stage: download
    before_script:
        - apk add --update --no-cache curl
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        - source "$CI_PROJECT_DIR"/scripts/orthologydb_dump.sh
        - cp "$CI_PROJECT_DIR"/schema/essential_genes_schema_additions.sql .
        
        
    artifacts:
        paths:
            - "$MOUNT_POINT/"



build_image:
    stage: build
    services:
        - name: $CI_REGISTRY/mouse-informatics/dind:latest
          alias: docker
    before_script:
        - echo "${CI_REGISTRY_PASSWORD}" | docker login -u "${CI_REGISTRY_USER}" --password-stdin  ${CI_REGISTRY}
        - NOW=$(date '+%Y-%m-%d-%H-%M-%S')
        - echo "export NOW=${NOW}" > ${MOUNT_POINT}/datetime.env
    script:

        - export
        - source ${MOUNT_POINT}/datetime.env
        - echo ${NOW}
        - sed -i "s|FROM postgres|FROM ${LOCAL_GITLAB_POSTGRES_IMAGE}|g" Dockerfile


        - docker build -t batchdownload-db .  | tee ${MOUNT_POINT}/build.log
        - docker run --name batchdbcontainer -v "$MOUNT_POINT:/mnt" -d batchdownload-db

        # Time is required to load the data
        # Hence the long pause period. 
        #
        # An alternative would be to copy the data required into to the image
        # when it is built, but this is not trivial in this build system, and a downside  would be
        # that every time the container starts there would always be a lag while the data was loaded.

        # Build using a debian based container for Bash
        
        - sleep "${DB_LOAD_TIME}"

        - docker exec batchdbcontainer sh /usr/local/data/postgres_processing_time.sh
        - docker cp batchdbcontainer:/usr/local/data/database.sql "$MOUNT_POINT"/database.sql
        - docker exec batchdbcontainer sh -c "rm -r /usr/local/data"
        - docker stop -t 120 batchdbcontainer


        # Package for production using an alpine based container

        - sed -i "s|FROM postgres|FROM ${LOCAL_GITLAB_POSTGRES_IMAGE}|g" Dockerfile-production
        - docker build -t prod-batch-db -f Dockerfile-production .  | tee ${MOUNT_POINT}/build-production.log
        - docker run --name prodbatchdbcontainer -v "$MOUNT_POINT:/mnt" -d prod-batch-db
        - sleep 120

        - echo "${CI_REGISTRY_IMAGE}":"${NOW}"
        
        # Required to make sure there is enough time for the db to shut down.
        - docker stop -t 120 prodbatchdbcontainer
        - docker commit prodbatchdbcontainer "${CI_REGISTRY_IMAGE}":"${NOW}"

        - rm "$MOUNT_POINT"/database.sql
        
        - docker tag "${CI_REGISTRY_IMAGE}":"${NOW}" "${CI_REGISTRY_IMAGE}":latest
        - docker push "${CI_REGISTRY_IMAGE}":"${NOW}"  | tee ${MOUNT_POINT}/push.log
        - docker push "${CI_REGISTRY_IMAGE}":latest  | tee ${MOUNT_POINT}/push.log

        - docker logout ${CI_REGISTRY}

        # PUSH THE IMAGE TO DOCKERHUB
        - echo "${DOCKER_HUB_PWD}" | docker login -u "${DOCKER_HUB_USER}" --password-stdin
 
        - docker tag "${CI_REGISTRY_IMAGE}":"${NOW}" "${DOCKER_HUB_USER}"/"${DOCKER_HUB_REPO}":"${NOW}"
        - docker push "${DOCKER_HUB_USER}"/"${DOCKER_HUB_REPO}":"${NOW}"  | tee ${MOUNT_POINT}/dockerhub-push-latest.log

        - docker tag "${CI_REGISTRY_IMAGE}":"${NOW}" "${DOCKER_HUB_USER}"/"${DOCKER_HUB_REPO}":latest
        - docker push "${DOCKER_HUB_USER}"/"${DOCKER_HUB_REPO}":latest  | tee ${MOUNT_POINT}/dockerhub-push-latest.log

        - docker logout

        # Reduce the number of files saved as artifacts
        - rm ${MOUNT_POINT}/*.tsv
        - rm ${MOUNT_POINT}/*.csv
        


    dependencies:
        - Human_data
        - Pharos
        - IDG_GitHub
        - IMPC_Viability
        - IMPC_Phenotypes
        - IMPC_Stats
        - gnomAD_pLoF
        - orthology_db_dump
    artifacts:
        name: "database-${CI_JOB_ID}"
        paths:
            - ${MOUNT_POINT}
    



sast:
  stage: test
  script:
    - echo "Running SAST scan ..."

  artifacts:
    reports:
      container_scanning: gl-sast-report.json


secret_detection:
  stage: test
  script:
    - echo "Running secret detection scan ..."

  artifacts:
    reports:
      container_scanning: gl-secret-detection-report.json



trivy_container_scanning:
  stage: test

  services:
    - name: $CI_REGISTRY/mouse-informatics/dind:latest
      alias: docker
  
  rules:
    - if: '$CI_COMMIT_REF_NAME == "master"'
      when: on_success
      allow_failure: true
  
  before_script:
    - export TRIVY_VERSION=$(wget -qO - "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/')
    - echo $TRIVY_VERSION
    - wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxvf -
    - echo "${CI_REGISTRY_PASSWORD}" | docker login -u "${CI_REGISTRY_USER}" --password-stdin  ${CI_REGISTRY}
    
    - source ${MOUNT_POINT}/datetime.env
    - echo ${NOW}

  script:
    # Build report
    - ./trivy --cache-dir .trivycache/ image --exit-code 0 --no-progress --format template --template "@contrib/gitlab.tpl" -o gl-container-scanning-report.json "${CI_REGISTRY_IMAGE}":"${NOW}"
    # Print report
    - ./trivy --cache-dir .trivycache/ image --exit-code 0 --no-progress --severity HIGH "${CI_REGISTRY_IMAGE}":"${NOW}"
    # Fail on critical vulnerability
    - ./trivy --cache-dir .trivycache/ image --exit-code 1 --severity CRITICAL --no-progress "${CI_REGISTRY_IMAGE}":"${NOW}"

    - docker logout ${CI_REGISTRY}

  cache:
    paths:
      - .trivycache/

  artifacts:
    reports:
      container_scanning: gl-container-scanning-report.json




    



hh-dev-deploy:
  stage: dev-deploy
  # Use an image with helm v2.14.3, kubectl v1.15.2, alpine 3.10
  # rancher/hyperkube:v1.15.3-rancher1 installed on the hh cluster
  image: $CI_REGISTRY/mouse-informatics/helm-kubectl-alpine:latest
  only:
    refs:
      - master
  script:
    - source ${MOUNT_POINT}/datetime.env
    - echo ${NOW}
    #
    - kubectl config set-cluster local --server="${HH_KUBERNETES_ENDPOINT}"
    - kubectl config set clusters.local.certificate-authority-data "${HH_KUBERNETES_CERTIFICATE_AUTHORITY_DATA}"
    - kubectl config set-credentials ${HH_KUBERNETES_DEV_USER} --token="${HH_KUBERNETES_DEV_USER_TOKEN}"
    - kubectl config set-context "${HH_KUBERNETES_DEV_NAMESPACE}" --cluster=local --user=${HH_KUBERNETES_DEV_USER} --namespace="${HH_KUBERNETES_DEV_NAMESPACE}"
    - kubectl config use-context "${HH_KUBERNETES_DEV_NAMESPACE}"
    - kubectl version
    #
    - sed -i "s/latest/${NOW}/g" kube/wp/dev/database/essential-genes-database-deployment.yaml
    - sed -i "s/STRING_REPLACED_DURING_REDEPLOY/$(date)/g" kube/wp/dev/database/essential-genes-database-deployment.yaml
    
    - |
      if kubectl apply -f kube/wp/dev/database/essential-genes-database-deployment.yaml --record | grep -q unchanged; then
          echo "=> Patching deployment to force image update."
          kubectl patch -f kube/wp/dev/database/essential-genes-database-deployment.yaml --record -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"ci-last-updated\":\"$(date +'%s')\"}}}}}"
      else
          echo "=> Deployment apply has changed the object, no need to force image update."
      fi


    - kubectl rollout status -f kube/wp/dev/database/essential-genes-database-deployment.yaml
    - kubectl get pod,deployment,rs



hx-dev-deploy:
  stage: dev-deploy
  # Use an image with helm v2.14.3, kubectl v1.15.2, alpine 3.10
  # rancher/hyperkube:v1.15.3-rancher1 installed on the hh cluster
  image: $CI_REGISTRY/mouse-informatics/helm-kubectl-alpine:latest
  only:
    refs:
      - master
  script:
    - source ${MOUNT_POINT}/datetime.env
    - echo ${NOW}
    #
    - kubectl config set-cluster local --server="${HX_KUBERNETES_ENDPOINT}"
    - kubectl config set clusters.local.certificate-authority-data "${HX_KUBERNETES_CERTIFICATE_AUTHORITY_DATA}"
    - kubectl config set-credentials ${HX_KUBERNETES_DEV_USER} --token="${HX_KUBERNETES_DEV_USER_TOKEN}"
    - kubectl config set-context "${HX_KUBERNETES_DEV_NAMESPACE}" --cluster=local --user=${HX_KUBERNETES_DEV_USER} --namespace="${HX_KUBERNETES_DEV_NAMESPACE}"
    - kubectl config use-context "${HX_KUBERNETES_DEV_NAMESPACE}"
    - kubectl version
    #
    - sed -i "s/latest/${NOW}/g" kube/wp/dev/database/essential-genes-database-deployment.yaml
    - sed -i "s/STRING_REPLACED_DURING_REDEPLOY/$(date)/g" kube/wp/dev/database/essential-genes-database-deployment.yaml
    
    - |
      if kubectl apply -f kube/wp/dev/database/essential-genes-database-deployment.yaml --record | grep -q unchanged; then
          echo "=> Patching deployment to force image update."
          kubectl patch -f kube/wp/dev/database/essential-genes-database-deployment.yaml --record -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"ci-last-updated\":\"$(date +'%s')\"}}}}}"
      else
          echo "=> Deployment apply has changed the object, no need to force image update."
      fi


    - kubectl rollout status -f kube/wp/dev/database/essential-genes-database-deployment.yaml
    - kubectl get pod,deployment,rs



dev-test:
    image: $CI_REGISTRY/mouse-informatics/ubuntu:latest
    stage: dev-test
    before_script:
        - apt-get update && apt-get install -y curl && apt-get clean && rm -rf /var/lib/apt/lists/*
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        - source "$CI_PROJECT_DIR"/scripts/service_test.sh  | tee ${MOUNT_POINT}/dev-test.log
 
        # Reduce the number of files saved as artifacts
        - rm ${MOUNT_POINT}/*.tsv
        - rm ${MOUNT_POINT}/*.csv
       

    artifacts:
        paths:
            - "$MOUNT_POINT/"



hh-production-deploy:
  stage: production-deploy
  # Use an image with helm v2.14.3, kubectl v1.15.2, alpine 3.10
  # rancher/hyperkube:v1.15.3-rancher1 installed on the hh cluster
  image: $CI_REGISTRY/mouse-informatics/helm-kubectl-alpine:latest
  only:
    refs:
      - master
  script:
    - source ${MOUNT_POINT}/datetime.env
    - echo ${NOW}
    #
    - kubectl config set-cluster local --server="${HH_KUBERNETES_ENDPOINT}"
    - kubectl config set clusters.local.certificate-authority-data "${HH_KUBERNETES_CERTIFICATE_AUTHORITY_DATA}"
    - kubectl config set-credentials ${HH_KUBERNETES_USER} --token="${HH_KUBERNETES_USER_TOKEN}"
    - kubectl config set-context "${HH_KUBERNETES_NAMESPACE}" --cluster=local --user=${HH_KUBERNETES_USER} --namespace="${HH_KUBERNETES_NAMESPACE}"
    - kubectl config use-context "${HH_KUBERNETES_NAMESPACE}"
    - kubectl version
    #
    - sed -i "s/latest/${NOW}/g" kube/wp/production/database/essential-genes-database-deployment.yaml
    - sed -i "s/STRING_REPLACED_DURING_REDEPLOY/$(date)/g" kube/wp/production/database/essential-genes-database-deployment.yaml
    
    - |
      if kubectl apply -f kube/wp/production/database/essential-genes-database-deployment.yaml --record | grep -q unchanged; then
          echo "=> Patching deployment to force image update."
          kubectl patch -f kube/wp/production/database/essential-genes-database-deployment.yaml --record -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"ci-last-updated\":\"$(date +'%s')\"}}}}}"
      else
          echo "=> Deployment apply has changed the object, no need to force image update."
      fi


    - kubectl rollout status -f kube/wp/production/database/essential-genes-database-deployment.yaml
    - kubectl get pod,deployment,rs



hx-production-deploy:
  stage: production-deploy
  # Use an image with helm v2.14.3, kubectl v1.15.2, alpine 3.10
  # rancher/hyperkube:v1.15.3-rancher1 installed on the hh cluster
  image: $CI_REGISTRY/mouse-informatics/helm-kubectl-alpine:latest
  only:
    refs:
      - master
  script:
    - source ${MOUNT_POINT}/datetime.env
    - echo ${NOW}
    #
    - kubectl config set-cluster local --server="${HX_KUBERNETES_ENDPOINT}"
    - kubectl config set clusters.local.certificate-authority-data "${HX_KUBERNETES_CERTIFICATE_AUTHORITY_DATA}"
    - kubectl config set-credentials ${HX_KUBERNETES_USER} --token="${HX_KUBERNETES_USER_TOKEN}"
    - kubectl config set-context "${HX_KUBERNETES_NAMESPACE}" --cluster=local --user=${HX_KUBERNETES_USER} --namespace="${HX_KUBERNETES_NAMESPACE}"
    - kubectl config use-context "${HX_KUBERNETES_NAMESPACE}"
    - kubectl version
    #
    - sed -i "s/latest/${NOW}/g" kube/wp/production/database/essential-genes-database-deployment.yaml
    - sed -i "s/STRING_REPLACED_DURING_REDEPLOY/$(date)/g" kube/wp/production/database/essential-genes-database-deployment.yaml
    
    - |
      if kubectl apply -f kube/wp/production/database/essential-genes-database-deployment.yaml --record | grep -q unchanged; then
          echo "=> Patching deployment to force image update."
          kubectl patch -f kube/wp/production/database/essential-genes-database-deployment.yaml --record -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"ci-last-updated\":\"$(date +'%s')\"}}}}}"
      else
          echo "=> Deployment apply has changed the object, no need to force image update."
      fi


    - kubectl rollout status -f kube/wp/production/database/essential-genes-database-deployment.yaml
    - kubectl get pod,deployment,rs



production-test:
    image: $CI_REGISTRY/mouse-informatics/ubuntu:latest
    stage: production-test
    before_script:
        - apt-get update && apt-get install -y curl && apt-get clean && rm -rf /var/lib/apt/lists/*
    script:
        
        # Establish a data directory
        - mkdir -p "$MOUNT_POINT"
        - cd "$MOUNT_POINT"
        
        - source "$CI_PROJECT_DIR"/scripts/service_test.sh -p  | tee ${MOUNT_POINT}/production-test.log
        
        # Reduce the number of files saved as artifacts
        - rm ${MOUNT_POINT}/*.tsv
        - rm ${MOUNT_POINT}/*.csv

    artifacts:
        paths:
            - "$MOUNT_POINT/"