From 8955836a3815e1071743cadbdc4d0ecfa9645f8b Mon Sep 17 00:00:00 2001 From: Ino de Bruijn Date: Fri, 31 Jul 2020 14:25:29 -0400 Subject: [PATCH] Make docker compose more seamless (#7693) - Add docker compose files to https://github.com/cBioPortal/cbioportal-docker-compose, initial work by @nr23730 & the Hyve - thanks! I figured it might be cleaner to keep the docker compose out of this repo, since it is using containers from various other repos (e.g. session-service) - Update docs to recommend to use that docker compose repo to set up a local instance - Update entrypoint for web-and-data image to include a check for database liveness and migrate the db automatically --- .dockerignore | 4 +- README.md | 2 +- docker/web-and-data/Dockerfile | 4 + docker/web-and-data/docker-entrypoint.sh | 86 ++++++++ docs/Deploy-Using-Docker.md | 171 ---------------- docs/docker/README.md | 189 ++++++------------ docs/docker/example_commands.md | 60 ++---- docs/docker/import_data.md | 47 +---- docs/docker/using-keycloak.md | 20 +- .../update_recommended_docker_version.sh | 55 ----- 10 files changed, 186 insertions(+), 452 deletions(-) create mode 100755 docker/web-and-data/docker-entrypoint.sh delete mode 100644 docs/Deploy-Using-Docker.md delete mode 100755 docs/scripts/update_recommended_docker_version.sh diff --git a/.dockerignore b/.dockerignore index fb495f0d19d..c722aa46de9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,8 @@ .DS_Store .github target -docker docs */target/* +docker +# don't ignore entrypoint scripts +!docker/web-and-data/docker-entrypoint.sh diff --git a/README.md b/README.md index 898d3c2004e..90206cde397 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ See [https://docs.cbioportal.org](https://docs.cbioportal.org) See [LICENSE](./LICENSE) ## 💻 Run Backend -cBioPortal consists of several components, please read the [Architecture docs](https://docs.cbioportal.org/2.1-deployment/architecture-overview) to figure out what repo would be relevant to edit. If you e.g. only want to make frontend changes, one can directly edit [the frontend repo](https://github.com/cbioportal/cbioportal-frontend) instead. Read the instructions in that repo for more info on how to do frontend development. This repo only contains the backend part. Before editing the backend, it's good to read the [backend code organization](docs/Backend-Code-Organization.md). For development of the backend repo one should first set up a database. Please follow the [Docker deployment documentation](https://docs.cbioportal.org/2.1.1-deploy-with-docker-recommended/docker) to do so. In step 6 instead of using the `latest` image, you can create your own image and use that: +cBioPortal consists of several components, please read the [Architecture docs](https://docs.cbioportal.org/2.1-deployment/architecture-overview) to figure out what repo would be relevant to edit. If you e.g. only want to make frontend changes, one can directly edit [the frontend repo](https://github.com/cbioportal/cbioportal-frontend) instead. Read the instructions in that repo for more info on how to do frontend development. This repo only contains the backend part. Before editing the backend, it's good to read the [backend code organization](docs/Backend-Code-Organization.md). For development of the backend repo one should first set up a database. Please follow the [Docker deployment documentation](https://docs.cbioportal.org/2.1.1-deploy-with-docker-recommended/docker) to do so. Change the [docker-compose](https://github.com/cBioPortal/cbioportal-docker-compose/blob/5da068f0eb9b4f42db52ab5e91321b26a1826d7a/docker-compose.yml#L6) file to use your image instead: ``` docker build -t cbioportal/cbioportal:my-dev-cbioportal-image -f docker/web-and-data/Dockerfile . diff --git a/docker/web-and-data/Dockerfile b/docker/web-and-data/Dockerfile index c7a6b4de0b2..9546b240914 100644 --- a/docker/web-and-data/Dockerfile +++ b/docker/web-and-data/Dockerfile @@ -74,3 +74,7 @@ ENV PORTAL_HOME=/cbioportal RUN mkdir -p $PORTAL_WEB_HOME RUN unzip /app.war -d $PORTAL_WEB_HOME + +# add entrypoint +COPY --from=build /cbioportal/docker/web-and-data/docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/web-and-data/docker-entrypoint.sh b/docker/web-and-data/docker-entrypoint.sh new file mode 100755 index 00000000000..e80ffc48df9 --- /dev/null +++ b/docker/web-and-data/docker-entrypoint.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -eo pipefail +shopt -s nullglob + +BAKED_IN_WAR_CONFIG_FILE=/cbioportal-webapp/WEB-INF/classes/portal.properties +CUSTOM_PROPERTIES_FILE=/cbioportal/portal.properties + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +parse_db_params_from_command_line() { + echo $@ | sed 's/-D/\n-D/g' | grep -- '-Ddb' | sed 's/-D//g' | grep db. +} + +parse_db_params_from_config_and_command_line() { + if [[ -f $CUSTOM_PROPERTIES_FILE ]]; then + PROPERTIES_FILE=$CUSTOM_PROPERTIES_FILE + else + PROPERTIES_FILE=$BAKED_IN_WAR_CONFIG_FILE + fi + + for param in db.host db.user db.portal_db_name db.password; do + if $(parse_db_params_from_command_line $@ | grep -q $param); then + parse_db_params_from_command_line $@ | grep "^$param" + else + grep -v '^#' $PROPERTIES_FILE | grep "^$param" + fi + done +} + +check_db_connection() { + eval "$(parse_db_params_from_config_and_command_line $@ | sed 's/^db\./db_/g')" + POTENTIAL_DB_PARAMS=$@ + + while ! mysqladmin ping -s -h$(echo ${db_host} | cut -d: -f1) -u${db_user} -p${db_password}; + do + sleep 5s; + if [ -n "$SHOW_DEBUG_INFO" ] && [ "$SHOW_DEBUG_INFO" != "false" ]; then + echo mysqladmin ping -s -h$(echo ${db_host} | cut -d: -f1) -u${db_user} -p${db_password} + fi + echo "Database not available yet (first time can take a few minutes to load seed database)... Attempting reconnect..." + done + echo "Database connection success" +} + +migrate_db() { + echo "Migrating database if necessary..." + POTENTIAL_DB_PARAMS=$@ + + if [[ -f $CUSTOM_PROPERTIES_FILE ]]; then + python3 /cbioportal/core/src/main/scripts/migrate_db.py -y -p $CUSTOM_PROPERTIES_FILE -s /cbioportal/db-scripts/src/main/resources/migration.sql + else + python3 /cbioportal/core/src/main/scripts/migrate_db.py -y -p <(parse_db_params_from_config_and_command_line $POTENTIAL_DB_PARAMS) -s /cbioportal/db-scripts/src/main/resources/migration.sql + fi +} + +_main() { + # when running the webapp, check db and do migration first + # check if command is something like "java -jar webapp-runner.jar" + if [[ "$@" == *java* ]] && [[ "$@" == *-jar* ]] && [[ "$@" == *webapp-runner.jar* ]]; then + # Parse database config. Use command line parameters (e.g. -Ddb.host) if + # available, otherwise use portal.properties + if [ -n "$SHOW_DEBUG_INFO" ] && [ "$SHOW_DEBUG_INFO" != "false" ]; then + echo "Using database config:" + parse_db_params_from_config_and_command_line $@ + fi + + check_db_connection $@ + migrate_db $@ + + if [ -n "$SHOW_DEBUG_INFO" ] && [ "$SHOW_DEBUG_INFO" != "false" ]; then + echo Running: "$@" + fi + fi + exec "$@" +} + +# If we are sourced from elsewhere, don't perform any further actions +if ! _is_sourced; then + _main "$@" +fi diff --git a/docs/Deploy-Using-Docker.md b/docs/Deploy-Using-Docker.md deleted file mode 100644 index e3898c3cae5..00000000000 --- a/docs/Deploy-Using-Docker.md +++ /dev/null @@ -1,171 +0,0 @@ -# Deploy Using Docker - -:warning: Please make sure to review [Docker prerequisites](Docker-Prerequisites.md) before continuing with deployment. - -## 1. Create a docker network - -Because MySQL and cBioPortal are running on separate containers, docker needs to know how to link them. Using Docker's legacy `--link` flag tends to be fragile since it will break if the MySQL container is restarted. - -To get around this, we can use the newer `Docker networks` feature by typing the following command in a docker terminal: - -#### Template - -```bash -docker network create ${DOCKER_NETWORK_NAME} -``` - -Where: -- **${DOCKER_NETWORK_NAME}** is the name of the network that cBioPortal and the cBioPortal DB are going to be accessible. _i.e If the network is called **"cbio-net"** the command should be:_ - -#### Example - -```bash -docker network create "cbio-net" -``` - -#### Useful Resources -- [Docker container networking](https://docs.docker.com/engine/userguide/networking/). -- [Docker network create](https://docs.docker.com/engine/reference/commandline/network_create/). - -## 2. Database Setup - -The cBioPortal software should run properly on MySQL version 5.7.x. Versions higher than 5.7.x can cause an issue -while loading the database schema. Minor versions lower than 5.7.x will cause issues with persistent cache invalidation. -The software can be found and downloaded from the [MySQL website](http://www.mysql.com/). - -There are two options to set up the cBioPortal Database: -2.1 Run MySQL on the host. -2.2 Run MySQL as a Docker container. - -### 2.1 Run MySQL on the host - -To install MySQL 5.7, kindly follow the vendor’s official detailed installation guide, available [here](http://dev.mysql.com/doc/refman/5.7/en/installing.html). - -### 2.2 Run MySQL as a docker container - -#### 2.2.1 Launch MySQL docker container - -In a docker terminal type the following command: - -##### Template - -```bash -docker run -d --name ${CONTAINER_NAME} \ ---restart=always \ ---net=${DOCKER_NETWORK_NAME} \ --p ${PREFERRED_EXTERNAL_PORT}:3306 \ --e MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} \ --e MYSQL_USER=${MYSQL_USER} \ --e MYSQL_PASSWORD=${MYSQL_PASSWORD} \ --e MYSQL_DATABASE=${MYSQL_DATABASE} \ --v ${PATH_TO_MYSQL_DATA}:/var/lib/mysql \ --v ${PATH_TO_SEED_DATABASE}/cgds.sql:/docker-entrypoint-initdb.d/cgds.sql:ro \ --v ${PATH_TO_SEED_DATABASE}/seed-cbioportal_no-pdb_hg19.sql.gz:/docker-entrypoint-initdb.d/seed_part1.sql.gz:ro \ --v ${PATH_TO_SEED_DATABASE}/seed-cbioportal_only-pdb.sql.gz:/docker-entrypoint-initdb.d/seed_part2.sql.gz:ro \ -mysql -``` - -Where: -- **${CONTAINER_NAME}**: The name of your container instance _i.e **cbio-DB**_. -- **${DOCKER_NETWORK-NAME}**: The name of your network _i.e **cbio-net**_. -- **${PREFERRED_EXTERNAL_PORT}**: The port that the container internal port will be mapped to _i.e **8306**_. -- **${MYSQL_ROOT_PASSWORD}**: The root password for the MySQL installation. For password restrictions please read carefully this [link](http://dev.mysql.com/doc/refman/5.7/en/user-names.html). -- **${MYSQL_USER}: The username for the cbioportal MySQL user _i.e **cbio**_. -- **${MYSQL_PASSWORD}**: The MySQL user password for the MySQL installation. For password restrictions please read carefully this [link](http://dev.mysql.com/doc/refman/5.7/en/user-names.html). -- **${MYSQL_DATABASE}**: The Database to create _i.e **cbioportal**_. -- **${PATH_TO_MYSQL_DATA}**: The MySQL path were all MySQL Data are stored. -- **${PATH_TO_SEED_DATABASE}**: Path to the seed databases. - -Running the above command will create a MySQL docker container and will automatically import all Seed Databases. - -:warning: This process can take about 45 minutes. For much faster -loading, we can choose to not load the PDB data, by removing the -line that loads `cbioportal-seed_only-pdb.sql.gz`. Please note that -your instance will be missing the 3D structure view feature (in the -mutations view) if you chose to leave this out. - -#### 2.2.2 MySQL Logs monitoring in Docker - -MySQL logs can easily be monitored by executing the following command on a terminal with docker. - -##### Template - -```bash -docker logs "{CONTAINER_NAME}" -``` - -Where: -- **{CONTAINER_NAME}**: The name of your container instance _i.e **cbio-DB**_. - -Learn more on [docker logs](https://docs.docker.com/engine/reference/commandline/logs/). - -#### 2.2.3 Access mysql shell on docker container - -To access the `mysql` shell on a docker container simply execute the following command: - -##### Template - -```bash -docker exec -it "{CONTAINER_NAME}" mysql -p"{MYSQL_ROOT_PASSWORD}" -``` - -Where: -- **{CONTAINER_NAME}**: The name of your container instance _i.e **cbio-DB**_. -- **{MYSQL_ROOT_PASSWORD}**: The root password for the MySQL installation. For password restrictions please read carefully this [link](http://dev.mysql.com/doc/refman/5.7/en/user-names.html). - -#### 2.2.4 Useful Resources -[MySQL Docker Hub](https://hub.docker.com/_/mysql/) -[MySQL Docker Github](https://github.com/docker-library/docs/tree/master/mysql) - -## 3. cBioPortal Setup - -### 3.1 Run the cBioPortal docker container - -In a docker terminal type the following command: - -##### Template - -```bash -docker run -d --name ${CONTAINER_NAME} \ - --restart=always \ - --net=${DOCKER_NETWORK_NAME} \ - -p ${PREFERRED_EXTERNAL_PORT}:8080 \ - -v ${PATH_TO_portal.properties}:/cbioportal/src/main/resources/portal.properties:ro \ - -v ${PATH_TO_log4j.properties}:/cbioportal/src/main/resources/log4j.properties:ro \ - -v ${PATH_TO_settings.xml}:/root/.m2/settings.xml:ro \ - -v ${PATH_TO_context.xml}:/usr/local/tomcat/conf/context.xml:ro \ - -v ${PATH_TO_CBIOPORTAL_LOGS}:/cbioportal_logs/ \ - -v ${PATH_TO_TOMCAT_LOGS}:/usr/local/tomcat/logs/ \ - -v ${PATH_TO_STUDIES}:/cbioportal_studies/:ro \ - cbioportal/cbioportal:${TAG} -``` - -Where: -- **${CONTAINER_NAME}**: The name of your container instance, _i.e **cbioportal**_. -- **${DOCKER_NETWORK_NAME}**: The name of your network, _i.e **cbio-net**_. -- **${PREFERRED_EXTERNAL_PORT}**: The port that the container internal port will be mapped to, _i.e **8306**_. -- **${PATH_TO_portal.properties}**: The external path were portal.properties are stored. -- **${PATH_TO_log4j.properties}**: The external path were log4j.properties are stored. -- **${PATH_TO_settings.xml}**: The external path were settings.xml is stored. -- **${PATH_TO_context.xml}**: The external path were context.xml is stored. -- **${PATH_TO_CUSTOMIZATION}**: The external path were customization files are stored. -- **${PATH_TO_CBIOPORTAL_LOGS}**: The external path where you want cBioPortal Logs to be stored. -- **${PATH_TO_TOMCAT_LOGS}**: The external path where you want Tomcat Logs to be stored. -- **${PATH_TO_STUDIES}**: The external path where cBioPortal studies are stored. -- **${TAG}**: The cBioPortal Version that you would like to run, _i.e **latest**_. - -### 3.2 Run DB Migrations - -Update the seeded database schema to match the cBioPortal version in the image. - -:warning: Running the following docker command will most likely make your database irreversibly incompatible with older versions of the portal code. - -##### Template - -```bash -docker exec -it ${CONTAINER_NAME} \ -migrate_db.py -p /cbioportal/src/main/resources/portal.properties -s /cbioportal/db-scripts/src/main/resources/migration.sql -``` - -Where: -- **${CONTAINER_NAME}**: The name of your network, _i.e **cbioportal**_. \ No newline at end of file diff --git a/docs/docker/README.md b/docs/docker/README.md index 7c8a618dab7..002be2f954d 100644 --- a/docs/docker/README.md +++ b/docs/docker/README.md @@ -10,138 +10,109 @@ Make sure that you have the latest version of Docker installed on your machine. [Notes for non-Linux systems](notes-for-non-linux.md) ## Usage instructions ## +In this example we use [Docker Compose](https://docs.docker.com/compose/) to spin up all the different containers for cBioPortal. + +### Quick Start -### Step 1 - Setup network ### -Create a network in order for the cBioPortal container and mysql database to communicate. ``` -docker network create cbio-net +git clone https://github.com/cBioPortal/cbioportal-docker-compose.git +cd cbioportal-docker-compose +./init.sh +docker-compose up ``` -### Step 2 - Run mysql with seed database ### -Start a MySQL server. The command below stores the database in a folder named -`//db_files/`. This should be an absolute path. - -Download the SQL schema (cgds.sql) and the seed database (seed-cbioportal__.sql.gz) from the -[cBioPortal Datahub](https://github.com/cBioPortal/datahub/blob/master/seedDB/README.md), -and use the command below to upload both to the server started above. +You should now be able to see the cBioPortal website at http://localhost:8080 -Make sure to replace -`//seed-cbioportal__.sql.gz` -with the path and name of the downloaded seed database. Again, this should be -an absolute path. +Import studies with: ``` -docker run -d --restart=always \ - --name=cbioDB \ - --net=cbio-net \ - -e MYSQL_ROOT_PASSWORD='P@ssword1' \ - -e MYSQL_USER=cbio \ - -e MYSQL_PASSWORD='P@ssword1' \ - -e MYSQL_DATABASE=cbioportal \ - -v //db_files/:/var/lib/mysql/ \ - -v //cgds.sql:/docker-entrypoint-initdb.d/cgds.sql:ro \ - -v //seed-cbioportal__.sql.gz:/docker-entrypoint-initdb.d/seed_part1.sql.gz:ro \ - mysql:5.7 +docker-compose run cbioportal metaImport.py -u http://cbioportal:8080 -s study/lgg_ucsf_2014/ -o ``` -Follow the logs of this step to ensure that no errors occur. If any error -occurs, make sure to check it. A common cause is pointing the `-v` parameters -above to folders or files that do not exist. +Clear persistent data volumes with: + +``` +docker compose down -v +``` -### Step 3 - Set up a portal.properties file ### +### Comprehensive Start -Copy the -[`portal.properties.EXAMPLE`](../../src/main/resources/portal.properties.EXAMPLE) -and change it according to your wishes. See the [full -reference](../portal.properties-Reference.md) and the [skin -properties](../Customizing-your-instance-of-cBioPortal.md) for more information on -the relevant properties. +#### Step 1 - Run Docker Compose -Make sure to at least provide the database parameters from step 1, which are -required for the next step: +Download the git repo that has the Docker compose file and go to the root of that folder: ``` -db.user=cbio -db.password=P@ssword1 -db.host=cbioDB -db.portal_db_name=cbioportal -db.connection_string=jdbc:mysql://cbioDB/ +git clone https://github.com/cBioPortal/cbioportal-docker-compose.git +cd cbioportal-docker-compose ``` -If you are using an external database change the `cbioDB` hostname to the -hostname of the MySQL database. If it requires an SSL connection use: - +Then download all necessary files (seed data, example config and example study from datahub) with the init script: ``` -db.use_ssl=true +./init.sh ``` -If you would like to enable OncoKB see [OncoKB data access](../OncoKB-Data-Access.md) for -how to obtain a data access token. After obtaining a valid token use: +Then run: ``` -show.oncokb=true -oncokb.token= +docker-compose up ``` -### Step 4 - Migrate database to latest version ### +This will start all four containers (services) defined [here](https://github.com/cBioPortal/cbioportal-docker-compose/blob/master/docker-compose.yml). That is: + +- the mysql database, which holds most of the cBioPortal data +- the cBioPortal Java web app, this serves the React frontend as well as the REST API +- the session service Java web app. This service has a REST API and stores session information (e.g. what genes are being queried) and user specific data (e.g. saved cohorts) in a separate mongo database +- the mongo database that persists the data for the session service + +It will take a few minutes the first time to import the seed database and perform migrations if necessary. Each container outputs logs to the terminal. For each log you'll see the name of the container that outputs it (e.g. `cbioportal_container` or `cbioportal_session_database_container`). If all is well you won't see any significant errors (maybe some warnings, that's fine to ignore). If all went well you should be able to visit the cBioPortal homepage on http://localhost:8080. You'll notice that no studies are shown on the homepage yet. Go to the next step to see how to import studies. -Update the seeded database schema to match the cBioPortal version in the image, -by running the following command. Note that this will most likely make your -database irreversibly incompatible with older versions of the portal code. +##### Notes on detached mode + +If you prefer to run the services in detached mode (i.e. not logging everything to your terminal), you can run ``` -docker run --rm -it --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - cbioportal/cbioportal:3.4.2 \ - migrate_db.py -p /cbioportal/portal.properties -s /cbioportal/db-scripts/src/main/resources/migration.sql +docker-compose up -d ``` -### Step 5 - Run Session Service containers -First, create the mongoDB database: +In this mode, you'll have to check the logs of each container manually using e.g.: ``` -docker run -d --name=mongoDB --net=cbio-net \ - -e MONGO_INITDB_DATABASE=session_service \ - mongo:3.6.6 +docker logs -f cbioportal_container ``` -Finally, create a container for the Session Service, adding the link to the mongoDB database using `-Dspring.data.mongodb.uri`: +You can list all containers running on your system with ``` -docker run -d --name=cbio-session-service --net=cbio-net \ - -e SERVER_PORT=5000 \ - -e JAVA_OPTS="-Dspring.data.mongodb.uri=mongodb://mongoDB:27017/session-service" \ - cbioportal/session-service:latest +docker ps -a ``` -### Step 6 - Run the cBioPortal web server ### +#### Step 2 - Import Studies +To import studies you can run: -Add any cBioPortal configuration in `portal.properties` as appropriate—see -the documentation on the -[main properties](https://github.com/cBioPortal/cbioportal/blob/master/docs/portal.properties-Reference.md) -and the -[skin properties](https://github.com/cBioPortal/cbioportal/blob/master/docs/Customizing-your-instance-of-cBioPortal.md). -Then start the web server as follows. +``` +docker-compose run cbioportal metaImport.py -u http://cbioportal:8080 -s study/lgg_ucsf_2014/ -o +``` +This will import the [lgg_ucsf_2014 study](https://www.cbioportal.org/patient?studyId=lgg_ucsf_2014) into your local database. It will take a few minutes to import. After importing, restart the cbioportal web container: ``` -docker run -d --restart=always \ - --name=cbioportal-container \ - --net=cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - -e JAVA_OPTS=' - -Xms2g - -Xmx4g - -Dauthenticate=noauthsessionservice - -Dsession.service.url=http://cbio-session-service:5000/api/sessions/my_portal/ - ' \ - -p 8081:8080 \ - cbioportal/cbioportal:3.4.2 \ - /bin/sh -c 'java ${JAVA_OPTS} -jar webapp-runner.jar /cbioportal-webapp' +docker-compose restart cbioportal ``` -To read more about the various ways to use authentication and `webapp-runner` -see the relevant [backend deployment -documentation](../Deploying.md#run-the-cbioportal-backend). +You can visit http://localhost:8080 again and you should be able to see the new study. + +All public studies can be downloaded from https://www.cbioportal.org/datasets, or https://github.com/cBioPortal/datahub/. You can add any of them to the `./study` folder and import them. There's also a script (`./study/init.sh`) to download multiple studies. You can set `DATAHUB_STUDIES` to any public study id (e.g. `lgg_ucsf_2014`) and run `./init.sh`. + +#### Step 3 - Customize your portal.properties file ### + +The properties file can be found in `./config/portal.properties`. Which was set up when running `init.sh`. + +This properties file allows you to customize your instance of cBioPortal with e.g. custom logos, or point the cBioPortal container to e.g. use an external mysql database. See for a the [properties](https://docs.cbioportal.org/2.3-customization/portal.properties-reference) documentation for a comprehensive overview. + +If you would like to enable OncoKB see [OncoKB data access](../OncoKB-Data-Access.md) for +how to obtain a data access token. After obtaining a valid token use: + +#### Step 4 - Customize cBioPortal setup +To read more about the various ways to use authentication and parameters for running the cBioPortal web app see the relevant [backend deployment documentation](../Deploying.md#run-the-cbioportal-backend). On server systems that can easily spare 4 GiB or more of memory, set the `-Xms` and `-Xmx` options to the same number. This should increase performance of @@ -150,15 +121,7 @@ co-expression tab. If you are using MacOS or Windows, make sure to take a look at [these notes](notes-for-non-linux.md) to allocate more memory for the virtual machine in which all Docker processes are running. -cBioPortal can now be reached at - -Activity of Docker containers can be seen with: - -``` -docker ps -a -``` - -## Data loading & more commands ## +## More commands ## For documentation on how to import a study, see [this tutorial](import_data.md) For more uses of the cBioPortal image, see [this file](example_commands.md) @@ -166,30 +129,6 @@ To Dockerize a Keycloak authentication service alongside cBioPortal, see [this file](using-keycloak.md). ## Uninstalling cBioPortal ## -First we stop the Docker containers. -``` -docker stop cbioDB -docker stop cbioportal-container -docker stop mongoDB -docker stop cbio-session-service -``` - -Then we remove the Docker containers. -``` -docker rm cbioDB -docker rm cbioportal-container -docker rm mongoDB -docker rm cbio-session-service -``` - -Cached Docker images can be seen with: -``` -docker images -``` - -Finally we remove the cached Docker images. ``` -docker rmi mysql:5.7 -docker rmi mongo:3.6.6 -docker rmi cbioportal/cbioportal:3.4.2 +docker compose down -v --rmi all ``` diff --git a/docs/docker/example_commands.md b/docs/docker/example_commands.md index 44600ff0d00..38040e480de 100644 --- a/docs/docker/example_commands.md +++ b/docs/docker/example_commands.md @@ -1,84 +1,59 @@ ### Importing gene panel ### Use this command to import a gene panel. Specify the gene panel file by replacing -`` with the absolute path to the gene panel file. +`` with the absolute path to the gene panel file. Another option is to add the +gene panel files in `./study` which is mounted inside the container on `/study/. ```shell -docker run -it --rm \ - --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ +docker-compose run \ -v :/gene_panels/gene_panel.txt:ro \ - cbioportal/cbioportal:3.4.2 \ + cbioportal bash -c 'cd /cbioportal/core/src/main/scripts/ && ./importGenePanel.pl --data /gene_panels/gene_panel.txt' ``` ### Importing data ### -Use this command to validate a dataset. Specify the study directory by replacing -`` with the absolute path to the study folder. The +Use this command to validate a dataset. Add the study to the `./study` folder. The command will connect to the web API of the container `cbioportal-container`, and import the study in its associated database. Make sure to replace `` with the absolute path were the html report of the validation will be saved. ```shell -docker run -it --rm --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - -v ":/study:ro" \ +docker-compose run \ -v ":/report" \ - cbioportal/cbioportal:3.4.2 \ - metaImport.py -u http://cbioportal-container:8080 -s /study --html=/report/report.html + cbioportal \ + metaImport.py -u http://cbioportal:8080 -s /study/name_of_study --html=/report/report.html ``` :warning: after importing a study, remember to restart `cbioportal-container` -to see the study on the home page. Run `docker restart cbioportal-container`. +to see the study on the home page. Run `docker-compose restart cbioportal`. #### Using cached portal side-data #### In some setups the data validation step may not have direct access to the web API, for instance when the web API is only accessible to authenticated browser sessions. You can use this command to generate a cached folder of files that the validation script can use instead. Make sure to replace `` with the absolute path where the cached folder is going to be generated. ```shell -docker run --rm --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ +docker-compose run \ -v "/portalinfo:/portalinfo" \ -w /cbioportal/core/src/main/scripts \ - cbioportal/cbioportal:3.4.2 \ + cbioportal \ ./dumpPortalInfo.pl /portalinfo ``` Then, grant the validation/loading command access to this folder and tell the script it to use it instead of the API: ```shell -docker run -it --rm --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - -v ":/study:ro" \ +docker-compose run \ -v ":/report" \ -v "/portalinfo:/portalinfo:ro" \ - cbioportal/cbioportal:3.4.2 \ + cbioportal \ metaImport.py -p /portalinfo -s /study --html=/report/report.html ``` ### Inspecting or adjusting the database ### -When creating the database container, you can map a port on the -local host to port 3306 of the container running the MySQL database, -by adding an option such as `-p 127.0.0.1:8306:3306` to the `docker -run` command before the name of the image (`mysql:5.7`). You can then -connect to this port (port 8306 in this example) using [MySQL -Workbench](https://www.mysql.com/products/workbench/) or another -MySQL client. - -If you have not opened a port, the following command can still -connect a command-line client to the container (`cbioDB` here) -using the `--net` option: - ```shell -docker run -it --rm \ - --net=cbio-net \ - -e MYSQL_HOST=cbioDB \ - -e MYSQL_USER=cbio \ - -e MYSQL_PASSWORD=P@ssword1 \ - -e MYSQL_DATABASE=cbioportal \ - mysql:5.7 \ - sh -c 'mysql -h"$MYSQL_HOST" -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DATABASE"' +docker-compose run cbioportal_database \ + sh -c 'mysql -hcbioportal_database -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DATABASE"' ``` ### Deleting a study ### @@ -86,9 +61,8 @@ docker run -it --rm \ To remove a study, run: ```shell -docker run -it --rm --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - cbioportal/cbioportal:3.4.2 \ +docker-compose run \ + cbioportal \ cbioportalImporter.py -c remove-study -id study_id ``` diff --git a/docs/docker/import_data.md b/docs/docker/import_data.md index 1cb305e6160..8b5b5babfd7 100644 --- a/docs/docker/import_data.md +++ b/docs/docker/import_data.md @@ -1,6 +1,6 @@ ## Import data instructions ## -This is an example to import a sample study: `study_es_0`. -When trying to import other studies, please follow the same routine: +This is an example to import a sample study: `study_es_0`. When trying to import other studies, please follow the same routine: + 1. import gene panels (if applicable, studies without gene panels are assumed to be whole exome/genome) 2. import study data @@ -10,18 +10,14 @@ To import gene panels for your study, please reference the example commands in [ These are the commands for importing `study_es_0` gene panels (`data_gene_panel_testpanel1` and `data_gene_panel_testpanel2`): ```shell -docker run -it --rm \ - --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - cbioportal/cbioportal:3.4.2 \ +docker-compose run \ + cbioportal \ bash -c 'cd /cbioportal/core/src/main/scripts/ && ./importGenePanel.pl --data /cbioportal/core/src/test/scripts/test_data/study_es_0/data_gene_panel_testpanel1.txt' ``` ```shell -docker run -it --rm \ - --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - cbioportal/cbioportal:3.4.2 \ +docker-compose run \ + cbioportal \ bash -c 'cd /cbioportal/core/src/main/scripts/ && ./importGenePanel.pl --data /cbioportal/core/src/test/scripts/test_data/study_es_0/data_gene_panel_testpanel2.txt' ``` @@ -32,16 +28,13 @@ To import data for your study, please reference the example commands in [this fi Command for importing `study_es_0` data: ```shell -docker run -it --rm --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - cbioportal/cbioportal:3.4.2 \ - metaImport.py -u http://cbioportal-container:8080 -s /cbioportal/core/src/test/scripts/test_data/study_es_0 +docker-compose run cbioportal metaImport.py -u http://cbioportal:8080 -s /cbioportal/core/src/test/scripts/test_data/study_es_0 -o ``` :warning: after importing a study, remember to restart `cbioportal-container` to see the study on the home page. Run `docker restart cbioportal-container`. -You have now imported the test study `study_es_0`. This study is included inside the cbioportal container. The process for adding a study that is outside of the container is almost the same. The difference is that one needs to add the `-v` parameter to mount the folder with the study data inside the container. See [How to import a real study](import_data.md#how-to-import-a-real-study) section for an example. +You have now imported the test study `study_es_0`. Note that this study is included inside the cbioportal container. The process for adding a study that is outside of the container is similar. Just make sure to add the data files in the `./study` folder. This folder is mounted as `/study/` inside of the container. ## Frequently Asked Questions @@ -54,9 +47,7 @@ please follow the first step to import gene panels (e.g. import `data_gene_panel ### Error occurred during validation step -Please make sure this line was included when [setting up the database](README.md#step-2-run-mysql-with-seed-database): - -`-v //seed-cbioportal__.sql.gz:/docker-entrypoint-initdb.d/seed_part1.sql.gz:ro \`. +Please make sure the seed database was correctly imported. ### Study imported correctly, but got error when trying to query something @@ -64,23 +55,3 @@ Remember to restart the `cbioportal-container` after data imported. ```shell docker restart cbioportal-container ``` - -### How to import a real study - -You can find all public studies at [cbioportal.org](http://www.cbioportal.org/data_sets.jsp) a zipped folder with staging files from each study can be downloaded. These zip files are compressed versions of the study folders in [datahub](https://github.com/cBioPortal/datahub). - -Download and extract the zip file for study `Cholangiocarcinoma (TCGA, PanCancer Atlas)`. Study `Cholangiocarcinoma (TCGA, PanCancer Atlas)` is a whole exome study. You can therefore skip [step 1 from the import data instructions section](import_data.md#step-1-import-gene-panels): the default gene panel is whole exome. Proceed with the command from step2. - -Specify the study directory by replacing -`` with the absolute path to the study folder (e.g. path of folder `chol_tcga_pan_can_atlas_2018`). -```shell -docker run -it --rm --net cbio-net \ - -v //portal.properties:/cbioportal/portal.properties:ro \ - -v ":/study:ro" \ - cbioportal/cbioportal:3.4.2 \ - metaImport.py -u http://cbioportal-container:8080 -s /study -o --html=/report/report.html -``` -after data imported successfully: -```shell -docker restart cbioportal-container -``` diff --git a/docs/docker/using-keycloak.md b/docs/docker/using-keycloak.md index 8d7e89b9ccb..4f77388046a 100644 --- a/docs/docker/using-keycloak.md +++ b/docs/docker/using-keycloak.md @@ -56,21 +56,5 @@ documentation](../Authenticating-and-Authorizing-Users-via-keycloak.md#configure Remember to specify port 8180 for the Keycloak server, wherever the guide says 8080. -After configuring Keycloak, set up cBioPortal containers [as specified in the documentation](README.md). Make sure to replace -the `JAVA_OPTS` argument in Step 6 to include the `-Dauthenticate=saml` flag: - -
-docker run -d --restart=always \
-    --name=cbioportal-container \
-    --net=cbio-net \
-    -v /<path_to_config_file>/portal.properties:/cbioportal/portal.properties:ro \
-    -e JAVA_OPTS='
-        -Xms2g
-        -Xmx4g
-        -Dauthenticate=saml
-        -Dsession.service.url=http://cbio-session-service:5000/api/sessions/my_portal/
-    ' \
-    -p 8081:8080 \
-    cbioportal/cbioportal:3.4.2 \
-    /bin/sh -c 'java ${JAVA_OPTS} -jar webapp-runner.jar /cbioportal-webapp'
-
+After configuring Keycloak, set up cBioPortal containers [as specified in the documentation](README.md). Make sure to update the `-Dauthenticate` +in the [docker-compose file](https://github.com/cBioPortal/cbioportal-docker-compose/blob/5da068f0eb9b4f42db52ab5e91321b26a1826d7a/docker-compose.yml#L20) to `-Dauthenticate=saml`. diff --git a/docs/scripts/update_recommended_docker_version.sh b/docs/scripts/update_recommended_docker_version.sh deleted file mode 100755 index 0152cb4bcab..00000000000 --- a/docs/scripts/update_recommended_docker_version.sh +++ /dev/null @@ -1,55 +0,0 @@ -#/bin/bash -# update the recommended docker version to install -# use like e.g. ./update_recommended_docker_version.sh 3.4.2 -set -e -HELPDOC=$( cat <&2 - echo "$HELPDOC" - exit 1 - ;; - esac -done - -# Parse arguments -if [ "$#" -ne "1" ] -then - echo "Invalid number of arguments: 1 needed but $# supplied" >&2 - echo "$HELPDOC" - exit 1 -fi - -NEW_VERSION=$1 -# strip off "v" -NEW_VERSION=${NEW_VERSION#v} -echo "Setting docs to use docker image cbioportal/cbioportal:$NEW_VERSION" - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd $SCRIPT_DIR/../docker - -for f in $(git grep 'cbioportal/cbioportal:' | gcut -f1 -d: | uniq); - do gsed -i "s|cbioportal/cbioportal:[a-z0-9.]*|cbioportal/cbioportal:${NEW_VERSION}|g" $f; -done