From 8232fc6f2943bb587fe623e507b910bdb5666959 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 29 May 2018 08:50:13 +0200 Subject: [PATCH 001/194] Fix zookeeper healthcheck --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 118ec1336..c482a1f7e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -83,7 +83,7 @@ services: ZOOKEEPER_SYNC_LIMIT: 2 ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 healthcheck: - test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-1 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] + test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-2 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] interval: 1m30s timeout: 5s retries: 3 @@ -105,7 +105,7 @@ services: ZOOKEEPER_SYNC_LIMIT: 2 ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 healthcheck: - test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-1 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] + test: ["CMD", "/bin/bash", "-c", "[ $$(echo dump | nc zookeeper-3 2181 | head -c1 | wc -c) -gt 0 ] || exit 1"] interval: 1m30s timeout: 5s retries: 3 From a22613b5a7dea3023c0f306b0b2cd6326d009418 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 29 May 2018 10:27:03 +0200 Subject: [PATCH 002/194] Increased HDFS health timeout --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index c482a1f7e..b3ad95504 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -88,7 +88,6 @@ services: timeout: 5s retries: 3 - zookeeper-3: image: confluentinc/cp-zookeeper:4.1.0 networks: @@ -361,7 +360,7 @@ services: healthcheck: test: ["CMD", "hdfs", "dfs", "-ls", "/"] interval: 1m - timeout: 5s + timeout: 10s retries: 3 hdfs-datanode-2: @@ -377,7 +376,7 @@ services: healthcheck: test: ["CMD", "hdfs", "dfs", "-ls", "/"] interval: 1m - timeout: 5s + timeout: 10s retries: 3 hdfs-namenode: @@ -395,7 +394,7 @@ services: healthcheck: test: ["CMD", "hdfs", "dfs", "-ls", "/"] interval: 1m - timeout: 5s + timeout: 10s retries: 3 #---------------------------------------------------------------------------# From 53112097ebe95167925555de39117b7e5f49cf04 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Jun 2018 13:27:07 +0200 Subject: [PATCH 003/194] Added all zookeeper instances to zookeeper-connect --- .../radar-cp-hadoop-stack/docker-compose.yml | 14 +++++++------- .../etc/rest-api/radar.yml.template | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 118ec1336..c64bcfe85 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -127,7 +127,7 @@ services: restart: always environment: KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181" + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 @@ -153,7 +153,7 @@ services: restart: always environment: KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181" + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_HOURS: 730 @@ -230,7 +230,7 @@ services: - schema-registry-1 restart: always environment: - KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181 + KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 KAFKA_REST_LISTENERS: http://0.0.0.0:8082 KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry-1:8081 KAFKA_REST_HOST_NAME: rest-proxy-1 @@ -258,7 +258,7 @@ services: volumes: - ./etc/schema:/schema/conf environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 KAFKA_SCHEMA_REGISTRY: http://schema-registry-1:8081 KAFKA_NUM_BROKERS: 3 RADAR_NUM_PARTITIONS: 3 @@ -448,7 +448,7 @@ services: CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" CONNECT_REST_ADVERTISED_HOST_NAME: "radar-mongodb-connector" - CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181 + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 CONNECT_CONSUMER_MAX_POLL_RECORDS: 500 CONNECT_CONSUMER_MAX_POLL_INTERVAL_MS: 300000 CONNECT_CONSUMER_SESSION_TIMEOUT_MS: 10000 @@ -502,7 +502,7 @@ services: CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" - CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181 + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" KAFKA_BROKERS: 3 @@ -739,7 +739,7 @@ services: volumes: - ./etc/schema:/schema/conf environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 KAFKA_SCHEMA_REGISTRY: http://schema-registry-1:8081 KAFKA_NUM_BROKERS: 3 RADAR_NUM_PARTITIONS: 3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template index 3a97db6e5..044c54ef3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-api/radar.yml.template @@ -27,4 +27,4 @@ management_portal_config: # timeout should be specified as the ISO-8601 duration format {@code PnDTnHnMn.nS}. source-type-connection-timeout: android_phone: PT2H - empatica_e4: PT1H \ No newline at end of file + empatica_e4: PT1H From 0e0ca082df312fc3dcad3a2dbc79bbd5cdd3270f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Jun 2018 13:27:43 +0200 Subject: [PATCH 004/194] Compress data by default, with explicit Kafka version --- .../radar-cp-hadoop-stack/docker-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index c64bcfe85..f5ac94848 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -133,6 +133,10 @@ services: KAFKA_LOG_RETENTION_HOURS: 730 KAFKA_MESSAGE_MAX_BYTES: 4000048 KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + KAFKA_COMPRESSION_TYPE: lz4 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: "1.1" + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "1.1" + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/1 || exit 1"] interval: 1m30s @@ -159,6 +163,10 @@ services: KAFKA_LOG_RETENTION_HOURS: 730 KAFKA_MESSAGE_MAX_BYTES: 4000048 KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + KAFKA_COMPRESSION_TYPE: lz4 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: "1.1" + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "1.1" + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/2 || exit 1"] interval: 1m30s @@ -185,6 +193,10 @@ services: KAFKA_LOG_RETENTION_HOURS: 730 KAFKA_MESSAGE_MAX_BYTES: 4000048 KAFKA_LOG4J_LOGGERS: kafka.producer.async.DefaultEventHandler=INFO,kafka.controller=INFO,state.change.logger=INFO + KAFKA_COMPRESSION_TYPE: lz4 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: "1.1" + KAFKA_LOG_MESSAGE_FORMAT_VERSION: "1.1" + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/3 || exit 1"] interval: 1m30s From 32193c988f6c98122c8703bee90a1fab44e1918a Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Jun 2018 13:28:03 +0200 Subject: [PATCH 005/194] Install script fix with Zsh --- dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh index c477e0de0..b64450ecc 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh @@ -63,7 +63,7 @@ inline_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mo # Set topics if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then - COMBINED_AGG_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_aggregated.sh 2>/dev/null) + COMBINED_AGG_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_aggregated.sh 2>/dev/null | tail -n 1) if [ -n "${RADAR_AGG_TOPIC_LIST}" ]; then COMBINED_AGG_TOPIC_LIST="${RADAR_AGG_TOPIC_LIST},${COMBINED_AGG_TOPIC_LIST}" fi @@ -72,7 +72,7 @@ inline_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/mongodb-connector/sin echo "==> Configuring HDFS Connector" if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then - COMBINED_RAW_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_raw.sh 2>/dev/null) + COMBINED_RAW_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_raw.sh 2>/dev/null | tail -n 1) if [ -n "${RADAR_RAW_TOPIC_LIST}" ]; then COMBINED_RAW_TOPIC_LIST="${RADAR_RAW_TOPIC_LIST},${COMBINED_RAW_TOPIC_LIST}" fi From 614a08e708e8dee7242071d07037d5ebc63aee01 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Jun 2018 15:45:22 +0200 Subject: [PATCH 006/194] Fixed gitignore --- dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index 8d3a50299..189a8c8f2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -8,6 +8,6 @@ /rest-api/mp_info.yml /redcap-integration/radar.yml /managementportal/config/keystore.jks -/managementportal/config/liquibase/oauth_client_details.csv +/managementportal/config/oauth_client_details.csv /schema/commons/ /schema/specifications/ From a261b049ee5dfa9c4d36e394f75892ffd8c30a2e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Jun 2018 16:37:45 +0200 Subject: [PATCH 007/194] Add compression to the REST proxy --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index f5ac94848..e7b93ce87 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -246,6 +246,7 @@ services: KAFKA_REST_LISTENERS: http://0.0.0.0:8082 KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry-1:8081 KAFKA_REST_HOST_NAME: rest-proxy-1 + KAFKA_REST_COMPRESSION_TYPE: lz4 healthcheck: test: ["CMD", "curl", "-f", "localhost:8082/topics"] interval: 1m30s From 1da7b4c0785550d2f16816187100b1e4758e5053 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 6 Jun 2018 16:24:28 +0200 Subject: [PATCH 008/194] Update Postgres --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 38617b38e..b44d086e1 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -678,7 +678,7 @@ services: managementportal-postgresql: - image: postgres:9.6.2 + image: postgres:9.6-alpine volumes: - "${MP_POSTGRES_DIR}/data/:/var/lib/postgresql/data/" - "./postgres-backup/backups/postgresql:/backups/database/postgresql/" From 5bd713f726639b62f9a58356d57de8c23889c1c0 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 7 Jun 2018 19:46:53 +0100 Subject: [PATCH 009/194] Add hash backup for more unified backups --- .../hash-backup/README.md | 45 +++++++++++++ .../hash-backup/backup.conf | 31 +++++++++ .../hash-backup/dest.conf | 19 ++++++ .../hash-backup/initialize-hb.sh | 66 +++++++++++++++++++ .../hash-backup/run-backup.sh | 32 +++++++++ .../systemd/radar-hashbackup.service.template | 10 +++ .../systemd/radar-hashbackup.timer.template | 9 +++ 7 files changed, 212 insertions(+) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.service.template create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.timer.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md new file mode 100644 index 000000000..a4ff083a7 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md @@ -0,0 +1,45 @@ +# HASH BACKUPS + +This directory contains a unified solution to create backups for different paths (or directories) in the system. For a quickstart, the postgres path (configured in .env file) is already included by default. +- + +- First configure all the parameters in the `backup.conf` file. Please note that the key and passphrase should be provided and be kept safe and backed up elsewhere. +- Then configure the remote destinations (if any) to send the backups to in the `dest.conf` file. Please look at the hashbackup documentation for more info on this. For a start FTP and Amazon s3 examples are included. Please note to leave the `dir` at any value since this will be eventually be replaced by the script based on the `ROOT_REMOTE_PATH` and `INPUTS` specified in the `backup.conf` file. +- Then run the initialization scripts +```shell +sudo bash initialize-hb.sh +``` +- This should initialize the hashbackup output directories with the specified key and passphrase and apply any configurations. +- If the `SET_UP_TIMER` parameter in `backup.conf` is set to `true` then the above command automatically configures a `systemd timer` to run the backups (`./run-bakup.sh` script) daily at 3am. This can be changed in `/etc/systemd/system/radar-hashbackup.timer`. +- Alternatively, If you want to run this via CRON job just add the following to the crontab - +``` +00 03 * * * root sudo bash //dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh +``` + +If the `systemd` timer is set to run the backups, then the backup should be controlled via `systemctl`. +```shell +# query the latest status and logs of the backup service +sudo systemctl status radar-hashbackup + +# Stop backup timer +sudo systemctl stop radar-hashbackup.timer + +# Restart backup timer +sudo systemctl reload radar-hashbackup.timer + +# Start backup timer +sudo systemctl start radar-hashbackup.timer + +# Full radar-hashbackup system logs +sudo journalctl -u radar-hashbackup +``` +The CRON job should preferably not be used if `systemd` is used. To remove `systemctl` integration, run +``` +sudo systemctl disable radar-hashbackup +``` + + +**Note**: If you want to run the backups once or manually, instead of using `systemd` or `CRON` you can just run the run backup script like - +```shell +sudo bash run-backup.sh +``` diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf new file mode 100644 index 000000000..52f63618a --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf @@ -0,0 +1,31 @@ + +. ../.env + +# Comma separated list of absolute paths to backup. Default is set to postgres db path from .env file +INPUTS=${MP_POSTGRES_DIR} + +# Absolute path on which to create backup from the inputs. It is recommended to use a separate disk in production +OUTPUT=/hashbackup + +# Deduplicate memory to use while backing up. Higher value means better deduplication. Default is 1gb +DEDUPLICATE_MEMORY=-D1g + +# Retention options. Default is the last 30 days of backups + one every month for the last 12 months +RETAIN=-s30d12m + +# Common key to use for encrypting all the backups. Please make sure to keep this safe and backed up. +KEY= + +# Common passphrase to use for all the backups. Please make sure to keep this safe and backed up. +PASSPHRASE= + +# Local backup size. It is recommended to set cache-size-limit as high as reasonable. Leave at default if not using a remote backup. +# The default is -1, meaning to keep a copy of all backup data in the local backup directory. +LOCAL_SIZE= + +# Set to true if want to set up a systemd unit and timer to run the backup scripts. Default time is set to run daily at 3am but you +# can configure it in ./systemd/radar-hashbackup.timer +SET_UP_TIMER=false + +# The remote path in which to create backups +ROOT_REMOTE_PATH=/hash-backups diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf new file mode 100644 index 000000000..fcb935c08 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf @@ -0,0 +1,19 @@ +# Remote destinations information according to hash backup documentation + +destname myftp +type ftp +host ftp.myserver.com +port 21 +userid myuser +password mypass +dir + + +destname myS3 +type s3 +location US +accesskey myaccesskey +secretkey mysecretkey +bucket myaccesskey-hashbackup +dir +class ia diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh new file mode 100644 index 000000000..fece98c77 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +#. ../.env +. ./backup.conf +. ../util.sh > /dev/null + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +IFS=',' read -r -a inputs <<< "$INPUTS" + +HB_VERSION="2115" +# install hash backup if it does not exist +if hash hb 2>/dev/null +then + echo "Hash backup detected. Proceeding..." +else + echo "Installing Hash Backup..." + mkdir tmp + wget -q -P tmp http://www.hashbackup.com/download/hb-"${HB_VERSION}"-linux-64bit.tar.gz + tar -xzf tmp/hb-*.tar.gz -C tmp + sudo-linux cp tmp/hb-"${HB_VERSION}"/hb /usr/local/bin + rm -r tmp + echo "Hash Backup installed...." +fi + +# initialize a backup directory for each input path and set up remote destinations +for element in "${inputs[@]}" +do + if [[ ! -d $element ]] + then + echo "The input path ${element} is not a directory." + exit 1 + fi + + echo "Initializing backup for input: ${element}" + backupSubpath=$(basename "${element}") + finalPath="${OUTPUT}/${backupSubpath}" + + # Only init if the directory does not exist + if [[ ! -d $finalPath ]] + then + hb init -c ${finalPath} -k "${KEY}" + if [ ! -z ${LOCAL_SIZE} ] + then + hb config -c ${finalPath} cache-size-limit ${LOCAL_SIZE} + fi + cp dest.conf "${finalPath}"/dest.conf + # Set up remote directory according to input path and remote root dir + sed -i -e "s~dir.*~dir ${ROOT_REMOTE_PATH}/${backupSubpath}/~g" "${finalPath}"/dest.conf + else + echo "Output Directory ${finalPath} already exists, Skipping initializing it..." + fi +done + +if [[ ${SET_UP_TIMER} -eq true ]] +then + check_command_exists systemctl + copy_template_if_absent /etc/systemd/system/radar-hashbackup.service systemd/radar-hashbackup.service.template + copy_template_if_absent /etc/systemd/system/radar-hashbackup.timer systemd/radar-hashbackup.timer.template + + inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-hashbackup.service + inline_variable 'ExecStart=' "$DIR/run-backup.sh" /etc/systemd/system/radar-hashbackup.service + + sudo systemctl daemon-reload + sudo systemctl enable radar-hashbackup.timer + sudo systemctl start radar-hashbackup.timer +fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh new file mode 100644 index 000000000..ec10fe185 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh @@ -0,0 +1,32 @@ +#!/bin/bash +. ./backup.conf + +# lock file +lockfile=.LOCKFILE + +if [ ! -f $lockfile ]; then + echo "Creating lock ..." + touch $lockfile + IFS=',' read -r -a inputs <<< "$INPUTS" + + for element in "${inputs[@]}" + do + if [[ ! -d $element ]] + then + echo "The input path ${element} is not a directory." + exit 1 + fi + + echo "Running backup for input: ${element}" + backupSubpath=$(basename "${element}") + finalPath="${OUTPUT}/${backupSubpath}" + hb log backup -c ${finalPath} ${element} ${DEDUPLICATE_MEMORY} + hb log retain -c ${finalPath} ${RETAIN} -x3m + hb log selftest -c ${finalPath} -v4 --inc 1d/30d + done + echo "Removing lock ..." + rm $lockfile +else + echo "Another instance is already running ... " +fi +echo "### DONE ###" diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.service.template b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.service.template new file mode 100644 index 000000000..2c39dd371 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.service.template @@ -0,0 +1,10 @@ +[Unit] +Description=RADAR-Docker hashbackup service + +[Service] + +WorkingDirectory= + +ExecStart= + +NotifyAccess=all diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.timer.template b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.timer.template new file mode 100644 index 000000000..f928be987 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/systemd/radar-hashbackup.timer.template @@ -0,0 +1,9 @@ +[Unit] +Description=RADAR-Docker hash backup timer. Default set to daily +[Timer] +# Daily at 3am +OnCalendar=*-*-* 03:00:00 +Persistent=true +Unit=radar-hashbackup.service +[Install] +WantedBy=multi-user.target From b9d347684b99014fb8517222e53bc147da916dca Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 7 Jun 2018 20:05:27 +0100 Subject: [PATCH 010/194] cd into bash source for cron jobs --- .../radar-cp-hadoop-stack/hash-backup/dest.conf | 2 +- .../radar-cp-hadoop-stack/hash-backup/initialize-hb.sh | 9 ++++----- .../radar-cp-hadoop-stack/hash-backup/run-backup.sh | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf index fcb935c08..598732459 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/dest.conf @@ -15,5 +15,5 @@ location US accesskey myaccesskey secretkey mysecretkey bucket myaccesskey-hashbackup -dir class ia +dir diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh index fece98c77..06d38bc86 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh @@ -1,10 +1,8 @@ #!/bin/bash +cd "$( dirname "${BASH_SOURCE[0]}" )" +. "./backup.conf" +. "../util.sh" > /dev/null -#. ../.env -. ./backup.conf -. ../util.sh > /dev/null - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" IFS=',' read -r -a inputs <<< "$INPUTS" HB_VERSION="2115" @@ -57,6 +55,7 @@ then copy_template_if_absent /etc/systemd/system/radar-hashbackup.service systemd/radar-hashbackup.service.template copy_template_if_absent /etc/systemd/system/radar-hashbackup.timer systemd/radar-hashbackup.timer.template + DIR="$( pwd )" inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-hashbackup.service inline_variable 'ExecStart=' "$DIR/run-backup.sh" /etc/systemd/system/radar-hashbackup.service diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh index ec10fe185..196ced1a9 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh @@ -1,5 +1,6 @@ #!/bin/bash -. ./backup.conf +cd "$( dirname "${BASH_SOURCE[0]}" )" +. "./backup.conf" # lock file lockfile=.LOCKFILE From 44fa2ceb1d9ea81447e2e50c038eab7d3dc41d2a Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 8 Jun 2018 11:28:58 +0100 Subject: [PATCH 011/194] Add execute permission on run backup script --- .../radar-cp-hadoop-stack/hash-backup/initialize-hb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh index 06d38bc86..23c940c59 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh @@ -56,6 +56,7 @@ then copy_template_if_absent /etc/systemd/system/radar-hashbackup.timer systemd/radar-hashbackup.timer.template DIR="$( pwd )" + sudo chmod +x $DIR/run-backup.sh inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-hashbackup.service inline_variable 'ExecStart=' "$DIR/run-backup.sh" /etc/systemd/system/radar-hashbackup.service From 4a2e8350b6afec3bc335a8c4cb2f4569d65660e6 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 8 Jun 2018 12:12:20 +0100 Subject: [PATCH 012/194] Use passphrase from env var --- .../radar-cp-hadoop-stack/hash-backup/initialize-hb.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh index 23c940c59..e37972fa1 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh @@ -36,7 +36,8 @@ do # Only init if the directory does not exist if [[ ! -d $finalPath ]] then - hb init -c ${finalPath} -k "${KEY}" + export HBPASS=${PASSPHRASE} + hb init -c ${finalPath} -k "${KEY}" -p "env" if [ ! -z ${LOCAL_SIZE} ] then hb config -c ${finalPath} cache-size-limit ${LOCAL_SIZE} From e26a393f8981dcbc501da809721c1304e2e1a31a Mon Sep 17 00:00:00 2001 From: Amos Folarin Date: Mon, 11 Jun 2018 13:27:43 +0100 Subject: [PATCH 013/194] recommended systemd timer --- dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md index a4ff083a7..120aee74a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md @@ -11,7 +11,7 @@ sudo bash initialize-hb.sh ``` - This should initialize the hashbackup output directories with the specified key and passphrase and apply any configurations. - If the `SET_UP_TIMER` parameter in `backup.conf` is set to `true` then the above command automatically configures a `systemd timer` to run the backups (`./run-bakup.sh` script) daily at 3am. This can be changed in `/etc/systemd/system/radar-hashbackup.timer`. -- Alternatively, If you want to run this via CRON job just add the following to the crontab - +- systemd timer is recommended but you may alternatively, run this via CRON job just add the following to the crontab - ``` 00 03 * * * root sudo bash //dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh ``` From 8407f039e0f32ef6877fe4beaecbaa1a146780b9 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 12 Jun 2018 17:55:27 +0100 Subject: [PATCH 014/194] Changes based on comments and additional documentation --- .../radar-cp-hadoop-stack/hash-backup/README.md | 16 +++++++++++++++- .../hash-backup/backup.conf | 11 +++++++++++ .../hash-backup/initialize-hb.sh | 5 ++++- .../hash-backup/run-backup.sh | 4 ++-- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md index a4ff083a7..1c55dc9f0 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md @@ -4,6 +4,7 @@ This directory contains a unified solution to create backups for different paths - - First configure all the parameters in the `backup.conf` file. Please note that the key and passphrase should be provided and be kept safe and backed up elsewhere. +- The passphrase will be taken from the environment variable HBPASS whenever access to the backup is requested. A passphrase secures backup data: - for users in hosted or managed environments, like a VPS - when the backup directory is on USB thumb drives - when the backup directory is on mounted storage like Google Drive, Amazon Cloud Drive, Dropbox, etc. additional details here - [HashBackup Security](http://www.hashbackup.com/technical/security) - Then configure the remote destinations (if any) to send the backups to in the `dest.conf` file. Please look at the hashbackup documentation for more info on this. For a start FTP and Amazon s3 examples are included. Please note to leave the `dir` at any value since this will be eventually be replaced by the script based on the `ROOT_REMOTE_PATH` and `INPUTS` specified in the `backup.conf` file. - Then run the initialization scripts ```shell @@ -39,7 +40,20 @@ sudo systemctl disable radar-hashbackup ``` -**Note**: If you want to run the backups once or manually, instead of using `systemd` or `CRON` you can just run the run backup script like - +**Notes**: +If you want to run the backups once or manually, instead of using `systemd` or `CRON` you can just run the run backup script like - ```shell sudo bash run-backup.sh ``` + +Also remember to upgrade hash backup frequently (~ every 3 months) since it is stated in documentation that - `The compatibility goal is that backups created less than a year ago should be accessible with the latest version.` + +Currently, the hashbackups are configured to use input paths but for systems like databases, you should prefer first creating dump of the database on a filepath and then using that path in the hashbackup configuration. +This can be easily done using a cron job for example - +This is for creating a dump of the postgres db running inside a docker container on a directory on the host named `/localpostgresdump` every night at 12 - + +``` +00 00 * * * docker exec pg_dump > /localpostgresdump/backup +``` + +You can then add the path `/localpostgresdump` in the `backup.conf` file in `INPUTS` which will create a backup of SQL dumps. diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf index 52f63618a..e72eb4983 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/backup.conf @@ -1,6 +1,9 @@ . ../.env +# Hash Backup version to use. The version will not be compatible after 1 year so please make sure to update it. +HB_VERSION=2115 + # Comma separated list of absolute paths to backup. Default is set to postgres db path from .env file INPUTS=${MP_POSTGRES_DIR} @@ -13,6 +16,10 @@ DEDUPLICATE_MEMORY=-D1g # Retention options. Default is the last 30 days of backups + one every month for the last 12 months RETAIN=-s30d12m +# Retention options for the files that are deleted. Default is the files for the last 3 months. +# Any files that are deleted will be removed from backups after 3 months. +DELETED_RETAIN=-x3m + # Common key to use for encrypting all the backups. Please make sure to keep this safe and backed up. KEY= @@ -23,6 +30,10 @@ PASSPHRASE= # The default is -1, meaning to keep a copy of all backup data in the local backup directory. LOCAL_SIZE= +# Arc size limit. At least 2 x arc-size-limit bytes of free disk space will be required in the local backup directory. +# The default is 100, A practical limit is around 4GB (for huge files), because many storage services do not allow file sizes over 5GB. +ARC_SIZE= + # Set to true if want to set up a systemd unit and timer to run the backup scripts. Default time is set to run daily at 3am but you # can configure it in ./systemd/radar-hashbackup.timer SET_UP_TIMER=false diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh index e37972fa1..72dfa2e8d 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh @@ -5,7 +5,6 @@ cd "$( dirname "${BASH_SOURCE[0]}" )" IFS=',' read -r -a inputs <<< "$INPUTS" -HB_VERSION="2115" # install hash backup if it does not exist if hash hb 2>/dev/null then @@ -42,6 +41,10 @@ do then hb config -c ${finalPath} cache-size-limit ${LOCAL_SIZE} fi + if [ ! -z ${ARC_SIZE} ] + then + hb config -c ${finalPath} arc-size-limit ${ARC_SIZE} + fi cp dest.conf "${finalPath}"/dest.conf # Set up remote directory according to input path and remote root dir sed -i -e "s~dir.*~dir ${ROOT_REMOTE_PATH}/${backupSubpath}/~g" "${finalPath}"/dest.conf diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh index 196ced1a9..9ead2d089 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh @@ -21,8 +21,8 @@ if [ ! -f $lockfile ]; then echo "Running backup for input: ${element}" backupSubpath=$(basename "${element}") finalPath="${OUTPUT}/${backupSubpath}" - hb log backup -c ${finalPath} ${element} ${DEDUPLICATE_MEMORY} - hb log retain -c ${finalPath} ${RETAIN} -x3m + hb log backup -c ${finalPath} ${element} ${DEDUPLICATE_MEMORY} -X + hb log retain -c ${finalPath} ${RETAIN} ${DELETED_RETAIN} -v hb log selftest -c ${finalPath} -v4 --inc 1d/30d done echo "Removing lock ..." From cdf28c140ee204656417c70479af81fa8755f1e0 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 13 Jun 2018 12:05:04 +0100 Subject: [PATCH 015/194] Optimise selftest and add upgrade info --- .../radar-cp-hadoop-stack/hash-backup/README.md | 15 ++++++++++++++- .../hash-backup/run-backup.sh | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md index 828ac679b..9cd5b9cfe 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/README.md @@ -53,7 +53,20 @@ This can be easily done using a cron job for example - This is for creating a dump of the postgres db running inside a docker container on a directory on the host named `/localpostgresdump` every night at 12 - ``` -00 00 * * * docker exec pg_dump > /localpostgresdump/backup +00 00 * * * docker exec pg_dumpall > /localpostgresdump/backup ``` You can then add the path `/localpostgresdump` in the `backup.conf` file in `INPUTS` which will create a backup of SQL dumps. + + +## Important INFO +Quoting the Hashbackup Docs from the download page - +``` +Beta versions of HashBackup expire quarterly on the 15th of January, April, July, and October. Use hb upgrade to get the latest version and extend the expiration date. +IMPORTANT: You can always access your backup data after the expiration date: everything continues to work except the backup command. +``` + +This means you will need to upgrade Hashbackup regularly. You can easily set up a CRON job to accomplish this. The following example shows how to upgrade every week at 1 AM on a Sunday. +``` +0 1 * * 0 root hb upgrade +``` diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh index 9ead2d089..2a05c35cf 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/run-backup.sh @@ -23,7 +23,7 @@ if [ ! -f $lockfile ]; then finalPath="${OUTPUT}/${backupSubpath}" hb log backup -c ${finalPath} ${element} ${DEDUPLICATE_MEMORY} -X hb log retain -c ${finalPath} ${RETAIN} ${DELETED_RETAIN} -v - hb log selftest -c ${finalPath} -v4 --inc 1d/30d + hb log selftest -c ${finalPath} -v4 --inc 1d/120d --sample 4 done echo "Removing lock ..." rm $lockfile From ec9a15bd61f0888f544ca8fb09b881f1d894b8aa Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 18 Jun 2018 13:40:30 +0200 Subject: [PATCH 016/194] Updates for gateway --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 8 ++++---- .../radar-cp-hadoop-stack/etc/gateway/gateway.yml | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index b44d086e1..479797ca4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -722,7 +722,7 @@ services: # RADAR Gateway # #---------------------------------------------------------------------------# gateway: - image: radarcns/radar-gateway:0.1.2 + image: radarbase/radar-gateway:0.2.0 networks: - api - kafka @@ -730,11 +730,11 @@ services: - rest-proxy-1 - managementportal-app volumes: - - ./etc/gateway/radar-is.yml:/etc/radar-gateway/radar-is.yml - - ./etc/gateway/classpath.xml:/usr/local/tomcat/conf/Catalina/localhost/radar-gateway.xml + - ./etc/gateway:/etc/radar-gateway + command: ["radar-gateway", "/etc/radar-gateway/gateway.yml"] healthcheck: # should give an unauthenticated response, rather than a 404 - test: ["CMD-SHELL", "wget --spider localhost:8080/radar-gateway 2>&1 | grep -q 401 || exit 1"] + test: ["CMD-SHELL", "wget --spider localhost:8080/radar-gateway/ 2>&1 | grep -q 401 || exit 1"] interval: 1m30s timeout: 5s retries: 3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml new file mode 100644 index 000000000..99993dead --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml @@ -0,0 +1,3 @@ +baseUri: http://0.0.0.0/radar-gateway/ +managementPortalUrl: http://managementportal-app:8080/managementportal +restProxyUrl: http://rest-proxy-1:8082 From 856b177766caf7fd458949cb9a3d430bb5511a95 Mon Sep 17 00:00:00 2001 From: nivethika Date: Wed, 20 Jun 2018 12:10:51 +0200 Subject: [PATCH 017/194] update rest-api and gateway versions --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 479797ca4..b22d0e523 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -301,7 +301,7 @@ services: # RADAR REST API # #---------------------------------------------------------------------------# rest-api: - image: radarbase/radar-restapi:0.2.0 + image: radarbase/radar-restapi:0.2.1 networks: - hotstorage - api @@ -722,7 +722,7 @@ services: # RADAR Gateway # #---------------------------------------------------------------------------# gateway: - image: radarbase/radar-gateway:0.2.0 + image: radarbase/radar-gateway:0.2.1 networks: - api - kafka From b8525e65e67c4ab961e5059d10921535489bec57 Mon Sep 17 00:00:00 2001 From: travis-ci Date: Wed, 20 Jun 2018 18:11:13 +0100 Subject: [PATCH 018/194] Better hdfs restructure --- .../hdfs-restructure/README.md | 29 +++++++++ .../hdfs_restructure.sh | 2 +- .../restructure_backup_hdfs.sh | 16 +++-- .../hdfs-restructure/util.sh | 43 +++++++++++++ .../install-systemd-wrappers.sh | 4 +- .../postgres-backup/README.md | 4 +- images/radar-hdfs-connector/Dockerfile | 27 -------- images/radar-hdfs-connector/README.md | 44 ------------- images/radar-hdfs-connector/kafka_status.sh | 63 ------------------- images/radar-hdfs-connector/launch | 53 ---------------- .../radar-hdfs-connector/sink-hdfs.properties | 8 --- scripts/README.md | 6 -- 12 files changed, 85 insertions(+), 214 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md rename dcompose-stack/radar-cp-hadoop-stack/{ => hdfs-restructure}/hdfs_restructure.sh (89%) rename {scripts => dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure}/restructure_backup_hdfs.sh (67%) mode change 100755 => 100644 create mode 100644 dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/util.sh delete mode 100644 images/radar-hdfs-connector/Dockerfile delete mode 100644 images/radar-hdfs-connector/README.md delete mode 100755 images/radar-hdfs-connector/kafka_status.sh delete mode 100755 images/radar-hdfs-connector/launch delete mode 100644 images/radar-hdfs-connector/sink-hdfs.properties diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md new file mode 100644 index 000000000..b5250b81d --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md @@ -0,0 +1,29 @@ +## Scripts + +This folder contains useful scripts to manage the extraction of data from HDFS in the RADAR-base Platform. + +- `hdfs_restructure.sh` + - This script uses the Restructure-HDFS-topic to extracts records from HDFS and converts them from AVRO to specified format + - By default, the format is CSV, compression is set to gzip and deduplication is enabled. + - To change configurations and for more info look at the [README here](https://github.com/RADAR-base/Restructure-HDFS-topic) + +- `restracture_backup_hdfs.sh` for running the above script in a controlled manner with rotating logs + - `logfile` is the log file where the script logs each operation + - `storage_directory` is the directory where the extracted data will be stored + - `lockfile` lock useful to check whether there is a previous instance still running + +- A systemd timer for this script can be installed by running the `../install-systemd-wrappers.sh`. Or you can add a cron job like below. + +To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u root` and add your task at the end of the file. The syntax is +```shell +* * * * * command to be executed +- - - - - +| | | | | +| | | | +----- day of week (0 - 6) (Sunday=0) +| | | +------- month (1 - 12) +| | +--------- day of month (1 - 31) +| +----------- hour (0 - 23) ++------------- min (0 - 59) +``` + +For example, `*/2 * * * * /absolute/path/to/script-name.sh` will execute `script-name.sh` every `2` minutes. diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh similarity index 89% rename from dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh rename to dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh index 345468ddf..acff2a866 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs_restructure.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh @@ -25,7 +25,7 @@ OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" # Internal docker directory to write output to HDFS_OUTPUT_DIR=/output # HDFS command to run -HDFS_COMMAND=(/usr/bin/java -jar "/${JAR}" hdfs://hdfs-namenode:8020 "$HDFS_FILE" "$HDFS_OUTPUT_DIR") +HDFS_COMMAND=(/usr/bin/java -Dorg.radarcns.compression=gzip -Dorg.radarcns.deduplicate=true -jar "/${JAR}" hdfs://hdfs-namenode:8020 "$HDFS_FILE" "$HDFS_OUTPUT_DIR") mkdir -p $OUTPUT_DIR sudo-linux docker run -i --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" -v "$PWD/lib/${JAR}:/${JAR}" openjdk:8-jre-alpine "${HDFS_COMMAND[@]}" diff --git a/scripts/restructure_backup_hdfs.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/restructure_backup_hdfs.sh old mode 100755 new mode 100644 similarity index 67% rename from scripts/restructure_backup_hdfs.sh rename to dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/restructure_backup_hdfs.sh index 369e1bcff..042ae247f --- a/scripts/restructure_backup_hdfs.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/restructure_backup_hdfs.sh @@ -1,16 +1,14 @@ #!/bin/bash +cd "$( dirname "${BASH_SOURCE[0]}" )" # log file -logfile= +logfile=hdfs_output.log -# working directory -working_directory= - -# landing folder -storage_directory= +# landing folder. Specify if not +storage_directory=output # lock file -lockfile= +lockfile=.LOCKFILE # involved HDFS directory hdfs_directory=/topicAndroidNew @@ -21,7 +19,7 @@ hdfs_directory=/topicAndroidNew if [ ! -f $lockfile ]; then log_info "Creating lock ..." touch $lockfile - (cd $working_directory && ./hdfs_restructure.sh $hdfs_directory $storage_directory >> $logfile 2>&1) + (./hdfs_restructure.sh $hdfs_directory $storage_directory >> $logfile 2>&1) log_info "Removing lock ..." rm $lockfile else @@ -30,4 +28,4 @@ fi log_info "### DONE ###" # check if log size exceeds the limit. If so, it rotates the log file -rolloverLog \ No newline at end of file +rolloverLog diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/util.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/util.sh new file mode 100644 index 000000000..e62a5447d --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/util.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin + +# maximum file size in byte to rotate log +minimumsize=10000000 + +# current time +timestamp=$(date '+%Y-%m-%d %H:%M:%S'); + +# Write message in the log file +log_info() { + echo "$timestamp - $@" >> $logfile 2>&1 +} + +# Remove old lock +checkLock() { + uptime=$( $logfile + fi +} + +# Entry point +touch $logfile +log_info "### $timestamp ###" +log_info "Checking lock ..." +checkLock diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh index c443bb459..eb13bc0da 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh @@ -17,8 +17,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-docker.service inline_variable 'ExecStart=' "$DIR/lib/systemd/start-radar-stack.sh" /etc/systemd/system/radar-docker.service -inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-output.service -inline_variable 'ExecStart=' "$DIR/hdfs_restructure.sh /topicAndroidNew output" /etc/systemd/system/radar-output.service +inline_variable 'WorkingDirectory=' "$DIR/hdfs-restructure" /etc/systemd/system/radar-output.service +inline_variable 'ExecStart=' "$DIR/hdfs-restructure/restructure_backup_hdfs.sh" /etc/systemd/system/radar-output.service inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-check-health.service inline_variable 'ExecStart=' "$DIR/check-health.sh" /etc/systemd/system/radar-check-health.service diff --git a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md index f97246685..be89dbe23 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/postgres-backup/README.md @@ -1,5 +1,7 @@ # POSTGRES Backup Scripts +**Note These Scripts Have been Deprecated. Please use the unified backup solution provided in `hash-backup` folder. This folder will be removed in the future.** + The `scripts` directory contains a script for running roatated backups of a running postgres instance. In this case these scripts are mounted to the postgres container (using bind mounts) and then run to create backups. These scripts are taken from the [postgres wiki](https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux) The backups can be configured to create daily, weekly and monthly backups by configuring the `pg_backup.config` configuration file. Please note that if you change the `BACKUP_DIR` in the config file then make sure to change the bind mount in the postgres container in docker-compose.yml file as well. @@ -13,4 +15,4 @@ Example cron job for daily running this script on postgres container at midnight 00 00 * * * docker exec radarcphadoopstack_managementportal-postgresql_1 ./backup-scripts/pg_backup_rotated.sh >> ~/pg_backup.log 2>&1 ``` -This also logs the output to a file. \ No newline at end of file +This also logs the output to a file. diff --git a/images/radar-hdfs-connector/Dockerfile b/images/radar-hdfs-connector/Dockerfile deleted file mode 100644 index bffddce29..000000000 --- a/images/radar-hdfs-connector/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -FROM confluentinc/cp-kafka-connect:3.2.1 - -MAINTAINER Nivethika M , Joris B - -LABEL description="RADAR-base Backend- HDFS Sink Connector" - -# Deploy RADAR-Backend - HDFS Sink Connector -ADD https://github.com/RADAR-base/RADAR-HDFS-Sink-Connector/releases/download/v0.1/radar-hdfs-sink-connector-0.1-sources.jar /etc/kafka-connect/jars/ - -# Load topics validator -COPY ./kafka_status.sh /home/kafka_status.sh - -# Load modified launcher -COPY launch /etc/confluent/docker/launch diff --git a/images/radar-hdfs-connector/README.md b/images/radar-hdfs-connector/README.md deleted file mode 100644 index 34feb3f8c..000000000 --- a/images/radar-hdfs-connector/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Dockerised RADAR-HDFS-Connector - -It runs the Confluent HDFS Connector 3.1.2 using a custom [RecordWriterProvider](https://github.com/RADAR-base/RADAR-Backend/blob/dev/src/main/java/org/radarcns/sink/hdfs/AvroRecordWriterProviderRadar.java) to support RADAR-base Avro schemas. For more details about Confluent HDFS Connector click [here](http://docs.confluent.io/3.1.2/connect/connect-hdfs/docs/index.html). - -Create the docker image: -``` -$ docker build -t radarcns/radar-hdfs-connector ./ -``` - -Or pull from dockerhub: -``` -$ docker pull radarcns/radar-hdfs-connector:0.1 -``` - -## Configuration - -This image has to be extended with a volume with appropriate `sink.properties` - -Sample HDFS sink.properties -``` -name=radar-hdfs-sink-android-15000 -connector.class=io.confluent.connect.hdfs.HdfsSinkConnector -tasks.max=4 -topics=topic1, topic2, ... -flush.size=15000 -hdfs.url=hdfs://namenode:8020 -format.class=org.radarcns.sink.hdfs.AvroFormatRadar -topics.dir=topicAndroidNew -``` - -## Runtime environment variables - -This container requires two environment variable: - -- `KAFKA_REST_PROXY`: a valid Rest-Proxy instance -- `TOPIC_LIST`: a comma separated list containing all required topic names - -Before starting the streams, it waits until all topics inside TOPIC_LIST are available. This check is performed using the /topic Rest-Proxy API, for more details click here. - -Note that connector's REST_PORT must be different from the one used by Rest-Proxy. - -## How to run - -For a complete use case scenario, check the RADAR-base `docker-compose` file available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) diff --git a/images/radar-hdfs-connector/kafka_status.sh b/images/radar-hdfs-connector/kafka_status.sh deleted file mode 100755 index e488b1eef..000000000 --- a/images/radar-hdfs-connector/kafka_status.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Check if variables exist -if [ -z "$CONNECT_ZOOKEEPER_CONNECT" ]; then - echo "CONNECT_ZOOKEEPER_CONNECT is not defined" - exit 2 -fi - -if [ -z "$CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL" ]; then - echo "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL is not defined" - exit 4 -fi - -KAFKA_BROKERS=${KAFKA_BROKERS:-3} - -max_timeout=32 - -tries=10 -timeout=1 -while true; do - ZOOKEEPER_CHECK=$(zookeeper-shell $CONNECT_ZOOKEEPER_CONNECT <<< "ls /brokers/ids") - ZOOKEEPER_CHECK="${ZOOKEEPER_CHECK##*$'\n'}" - ZOOKEEPER_CHECK="$(echo -e "${ZOOKEEPER_CHECK}" | tr -d '[:space:]' | tr -d '[' | tr -d ']')" - - IFS=',' read -r -a array <<< $ZOOKEEPER_CHECK - LENGTH=${#array[@]} - if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then - echo "Kafka brokers available." - break - fi - - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED: KAFKA BROKERs NOT READY." - exit 5 - fi - echo "Expected $KAFKA_BROKERS brokers but found only $LENGTH. Waiting $timeout second before retrying ..." - sleep $timeout - if [ $timeout -lt $max_timeout ]; then - timeout=$((timeout * 2)) - fi -done - -tries=10 -timeout=1 -while true; do - if wget --spider -q "${CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL}/subjects" 2>/dev/null; then - break - fi - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED TO REACH SCHEMA REGISTRY." - exit 6 - fi - echo "Failed to reach schema registry. Retrying in ${timeout} seconds." - sleep $timeout - if [ $timeout -lt $max_timeout ]; then - timeout=$((timeout * 2)) - fi -done - - -echo "Kafka is available. Ready to go!" diff --git a/images/radar-hdfs-connector/launch b/images/radar-hdfs-connector/launch deleted file mode 100755 index d3333db15..000000000 --- a/images/radar-hdfs-connector/launch +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2016 Confluent Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. -if [ -z "$KAFKA_JMX_OPTS" ]; then - export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " -fi - -# The JMX client needs to be able to connect to java.rmi.server.hostname. -# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. -# For host n/w, this is the IP that the hostname on the host resolves to. - -# If you have more that one n/w configured, hostname -i gives you all the IPs, -# the default is to pick the first IP (or network). -export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} - -if [ "$KAFKA_JMX_PORT" ]; then - # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. - export JMX_PORT=$KAFKA_JMX_PORT - export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" -fi - -# Busy waiting loop that waits untill all topic are available -echo "===> Wait for infrastructure ..." -./home/kafka_status.sh -radar_check=$? -if [ "$radar_check" -ne 0 ]; then - exit $radar_check -fi - -echo "===> Launching ${COMPONENT} ... new" -# Add external jars to the classpath -# And this also makes sure that the CLASSPATH does not start with ":/etc/..." -# because this causes the plugin scanner to scan the entire disk. -export CLASSPATH="/etc/${COMPONENT}/jars/*" -echo $CLASSPATH - -# execute connector in standalone mode -exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}".properties $(find /etc/"${COMPONENT}"/ -type f -name "${CONNECTOR_PROPERTY_FILE_PREFIX}*.properties") diff --git a/images/radar-hdfs-connector/sink-hdfs.properties b/images/radar-hdfs-connector/sink-hdfs.properties deleted file mode 100644 index 7a8649520..000000000 --- a/images/radar-hdfs-connector/sink-hdfs.properties +++ /dev/null @@ -1,8 +0,0 @@ -name=radar-hdfs-sink-android-15000 -connector.class=io.confluent.connect.hdfs.HdfsSinkConnector -tasks.max=4 -topics=android_empatica_e4_electrodermal_activity,android_empatica_e4_inter_beat_interval,android_empatica_e4_temperature -flush.size=15000 -hdfs.url=hdfs://namenode:8020 -format.class=org.radarcns.sink.hdfs.AvroFormatRadar -topics.dir=topicAndroidNew \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index 1aca271d3..62e94b85a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,12 +8,6 @@ This folder contains useful scripts to manage the server where the RADAR-base Pl - `logfile` is the log file where the script logs each operation - `url` is the web site used to check the connectivity -- `restracture_backup_hdfs.sh` - - `logfile` is the log file where the script logs each operation - - `working_directory` is the directory where the `hdfs_restructure.sh` script is located. - - `storage_directory` is the directory where the extracted data will be stored - - `lockfile` lock usefull to check whether there is a previous instance still running - To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u root` and add your task at the end of the file. The syntax is ```shell * * * * * command to be executed From 234965dfe3b69764fd8936658506e18b2c09ceb3 Mon Sep 17 00:00:00 2001 From: travis-ci Date: Wed, 20 Jun 2018 18:30:24 +0100 Subject: [PATCH 019/194] Use latest versions --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 8 ++++---- dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh | 4 +--- .../radar-cp-hadoop-stack/kafka-radarinit/Dockerfile | 3 +-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 38617b38e..4c791acb2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -301,7 +301,7 @@ services: # RADAR REST API # #---------------------------------------------------------------------------# rest-api: - image: radarbase/radar-restapi:0.2.0 + image: radarbase/radar-restapi:0.2.1 networks: - hotstorage - api @@ -428,7 +428,7 @@ services: # RADAR mongo connector # #---------------------------------------------------------------------------# radar-mongodb-connector: - image: radarbase/kafka-connect-mongodb-sink:0.2.1 + image: radarbase/kafka-connect-mongodb-sink:0.2.2 restart: on-failure volumes: - ./etc/mongodb-connector/sink-mongo.properties:/etc/kafka-connect/sink.properties @@ -480,7 +480,7 @@ services: # RADAR HDFS connector # #---------------------------------------------------------------------------# radar-hdfs-connector: - image: radarbase/radar-hdfs-connector-auto:0.1 + image: radarbase/radar-hdfs-connector-auto:0.2.0 restart: on-failure volumes: - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties @@ -645,7 +645,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarbase/management-portal:0.3.6 + image: radarbase/management-portal:0.3.7 networks: - default - api diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh b/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh index 3e0262921..515dfa9a5 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh @@ -19,7 +19,7 @@ topics=($($LS_CMD)) #printf "%s\n" "${topics[@]}" # consumer command to run -KACC_CMD="kafka-avro-console-consumer --zookeeper zookeeper-1:2181 --property schema.registry.url=http://schema-registry-1:8081 --property print.key=true --topic ${args[0]} ${args[@]:1}" +KACC_CMD="kafka-avro-console-consumer --broker-list kafka-1:9092,kafka-2:9092,kafka-3:9092 --property schema.registry.url=http://schema-registry-1:8081 --property print.key=true --topic ${args[0]} ${args[@]:1}" DOCKER_CMD="docker-compose exec schema-registry-1" # check if is valid topic @@ -46,5 +46,3 @@ fi # run consumer echo $DOCKER_CMD $KACC_CMD exec $DOCKER_CMD $KACC_CMD - - diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile index 4a84d7f31..ce955c5e3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile @@ -25,7 +25,7 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.3.2 +ENV RADAR_SCHEMAS_VERSION=0.3.4 RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 @@ -41,4 +41,3 @@ RUN chmod +x /usr/bin/*.sh ENTRYPOINT ["init.sh"] CMD ["topic_init.sh"] - From c0b40dcfec4e4ad2c294dc03c54dc008fa50b5c6 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 4 Jul 2018 17:05:08 +0100 Subject: [PATCH 020/194] Add script for hdfs data delete based on date and topic --- scripts/README.md | 38 +++++++++- .../hdfs_data_retention.sh | 75 +++++++++++++++++++ .../hdfs_get_relevant_files.pig | 34 +++++++++ .../hdfs-data-retention/topics_to_remove.txt | 2 + 4 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 scripts/hdfs-data-retention/hdfs_data_retention.sh create mode 100644 scripts/hdfs-data-retention/hdfs_get_relevant_files.pig create mode 100644 scripts/hdfs-data-retention/topics_to_remove.txt diff --git a/scripts/README.md b/scripts/README.md index 62e94b85a..133f56a3a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,7 +2,9 @@ This folder contains useful scripts to manage the server where the RADAR-base Platform is running. -- `check_radar_network.sh` checks if the machine is connected to internet. The script can be parametrised with +### `check_radar_network.sh` +**It checks if the machine is connected to internet. The script can be parametrised with the following** +- - `nic` is the internet gateway - `lockfile` lock usefull to check whether there is a previous instance still running - `logfile` is the log file where the script logs each operation @@ -23,3 +25,37 @@ To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u For example, `*/2 * * * * /absolute/path/to/script-name.sh` will execute `script-name.sh` every `2` minutes. Before deploying the task, make sure that all paths used by the script are absolute. Replace the relative path to `util.sh` with the absolute one. + + + +### `hdfs-data-retention/hdfs_data_retention.sh` +**It is a script for deleting records from hdfs based on name of the topic and the date. All the records for the current topics older than the specified date are deleted from HDFS.** + + - `OUTPUT_DIR` - the directory where FS image file and extracted data will be stored. Default is `./tmp` + - `date_time_to_remove_before` - All records for appropriate topics before this date will be removed from HDFS. + - `HDFS_NAME_NODE` - The url of the hdfs namenode to download the FS image file and delete files. + - `hdfs-data-retention/topics_to_remove.txt` - The file used by the above script to delete files from these topics. Please specify each topic on a new line. + +Usage: +To just get the FS image file and process it and list the sum of file sizes of all the relevant files using apache pig, run the command like - +```shell +cd hdfs-data-retention +sudo bash hdfs_data_retention.sh +``` +This will output the file sizes sum of the calculated paths like - +``` +(SUM OF FILES SIZES TO BE DELETED IN MB = 46555) +``` +and also store the finalised path meeting the conditions of topics and date in the `./tmp/final_paths/part_r_00000` + +To also delete the files listed by the command above, just run - +```shell +cd hdfs-data-retention +sudo bash hdfs_data_retention.sh delete +``` + +Info: +By default the script is set up to run against docker containers in the RADAR-base stack. +The script will use the hdfs.image and hdfs.txt files from `./tmp` folder if present. To get a new FS image file from namenode, delete these files first and then run the script. + +If you get JAVA_HOME not set error, please uncomment and specify the JAVA_HOME in the script. diff --git a/scripts/hdfs-data-retention/hdfs_data_retention.sh b/scripts/hdfs-data-retention/hdfs_data_retention.sh new file mode 100644 index 000000000..ee38426b9 --- /dev/null +++ b/scripts/hdfs-data-retention/hdfs_data_retention.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# All records for appropriate topics before this date will be removed from HDFS. +date_time_to_remove_before='2018-03-15 12:00' + +OUTPUT_DIR="./tmp" +OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" + +HDFS_NAME_NODE='hdfs://hdfs-namenode:8020' + +# HDFS command to get FS image file from hdfs name node +HDFS_COMMAND_IMAGE=(hdfs dfsadmin -fetchImage /fsimage_tmp/hdfs.image) +# Can also use (curl --silent "http://hdfs-namenode:50070/imagetransfer?getimage=1&txid=latest" -o /fsimage_tmp/hdfs.image) + +# HDFS command to create text file from FSImage file +HDFS_COMMAND_TEXT=(hadoop oiv -i /fsimage_tmp/hdfs.image -o /fsimage_tmp/hdfs.txt -p Delimited -delimiter ,) + +if [[ ! -d 'tmp' ]]; then + mkdir tmp +fi + +if [[ ! -f './tmp/hdfs.image' ]]; then + echo "Downloading a new FS image file at ./tmp and converting to txt." + docker run -i --rm --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 ${HDFS_COMMAND_IMAGE[@]} + + docker run -i --rm --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 ${HDFS_COMMAND_TEXT[@]} +else + echo "./tmp/hdfs.image already exists. Using the same FS image file. " + if [[ ! -f './tmp/hdfs.txt' ]]; then + docker run -i --rm --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 ${HDFS_COMMAND_TEXT[@]} + else + echo "./tmp/hdfs.txt already exists. Not generating a new one. " + fi +fi + + +# Set this if get JAVA_HOME not set error or set it in ~/.profile +#export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" + +# Download and configure apache pig +export PIG_HOME="$(pwd)"/pig-0.16.0 +export PATH=$PATH:"$(pwd)"/pig-0.16.0/bin + +if ! hash "pig" >/dev/null 2>&1; then + wget http://www-us.apache.org/dist/pig/pig-0.16.0/pig-0.16.0.tar.gz + tar -xzf pig-0.16.0.tar.gz + export PATH=$PATH:"$(pwd)"/pig-0.16.0/bin +fi + +# Write all the relevant file paths to file using apache pig +pig -x local -param inputFile=./tmp/hdfs.txt -param outputFile=./tmp/final_paths -param topics=./topics_to_remove.txt -param time="${date_time_to_remove_before}" ./hdfs_get_relevant_files.pig + +FINAL_PATH='./tmp/final_paths/part-r-00000' +NUMOFLINES=$(wc -l < "${FINAL_PATH}") +# If delete is passed as an argument, only then delete the files from the HDFS. +if [[ "$1" = "delete" ]]; then + docker run -i -d --name "hdfs-delete" --network hadoop -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 /bin/bash + # Wait for the container to start up + sleep 30 + if [[ -f "${FINAL_PATH}" ]]; then + echo "READING AND REMOVING RELEVANT PATHS" + let "curr_prog = 0" + while IFS='' read -r line || [[ -n "$line" ]]; do + docker exec hdfs-delete hdfs dfs -rm "$line" > /dev/null + echo "Deleted from HDFS file: $line" + let "curr_prog += 1" + let "perc_prog = ((curr_prog*100)/NUMOFLINES)" + echo "Progress: $perc_prog %" + done < "${FINAL_PATH}" + fi + echo "Complete deletion" + docker rm -f "hdfs-delete" +fi + +rm -r ./tmp/final_paths/ diff --git a/scripts/hdfs-data-retention/hdfs_get_relevant_files.pig b/scripts/hdfs-data-retention/hdfs_get_relevant_files.pig new file mode 100644 index 000000000..00d4b26d9 --- /dev/null +++ b/scripts/hdfs-data-retention/hdfs_get_relevant_files.pig @@ -0,0 +1,34 @@ + -- Load all of the fields from the file + A = LOAD '$inputFile' USING PigStorage(',') AS (path:chararray, + replication:int, + modTime:chararray, + accessTime:chararray, + blockSize:long, + numBlocks:int, + fileSize:long, + NamespaceQuota:int, + DiskspaceQuota:int, + perms:chararray, + username:chararray, + groupname:chararray); + + + -- Grab just the path, size and modDate(in milliseconds) + B = FOREACH A GENERATE path, fileSize, org.apache.pig.builtin.ToMilliSeconds(org.apache.pig.builtin.ToDate(modTime, 'yyyy-MM-dd HH:mm', '+00:00')) as modTime:long; + C = FILTER B BY ((modTime < org.apache.pig.builtin.ToMilliSeconds(org.apache.pig.builtin.ToDate('$time', 'yyyy-MM-dd HH:mm', '+00:00'))) AND (path matches '^((?!tmp).)*.avro')); + -- DUMP C; + -- Load topics from the provided file + D = LOAD '$topics' USING PigStorage() AS (topic:chararray); + + C_0 = FOREACH C GENERATE path, fileSize; + C_1 = CROSS C_0, D; + -- DUMP C_1; + E = FILTER C_1 BY (path matches SPRINTF('.*%s.*', topic)); + + -- Calculate total file size + S = FOREACH (GROUP E ALL) GENERATE CONCAT('SUM OF FILES SIZES TO BE DELETED IN MB = ', (chararray)(org.apache.pig.builtin.SUM(E.fileSize) / 1024 / 1024)); + DUMP S; + F = FOREACH E GENERATE path; + -- Save results + -- DUMP F; + STORE F INTO '$outputFile'; diff --git a/scripts/hdfs-data-retention/topics_to_remove.txt b/scripts/hdfs-data-retention/topics_to_remove.txt new file mode 100644 index 000000000..d4e5a20f2 --- /dev/null +++ b/scripts/hdfs-data-retention/topics_to_remove.txt @@ -0,0 +1,2 @@ +android_phone_acceleration +android_phone_gyroscope From 07cac5af4df8f23fb125d0827d1df104dfba4b87 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 4 Jul 2018 17:12:46 +0100 Subject: [PATCH 021/194] Add config to hdfs to prevent failures --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 4c791acb2..ff2d7534e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -370,6 +370,8 @@ services: environment: CORE_CONF_fs_defaultFS: hdfs://hdfs-namenode:8020 HDFS_CONF_dfs_replication: 2 + HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_enable: "false" + HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_policy: "NEVER" healthcheck: test: ["CMD", "hdfs", "dfs", "-ls", "/"] interval: 1m @@ -386,6 +388,8 @@ services: environment: CORE_CONF_fs_defaultFS: hdfs://hdfs-namenode:8020 HDFS_CONF_dfs_replication: 2 + HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_enable: "false" + HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_policy: "NEVER" healthcheck: test: ["CMD", "hdfs", "dfs", "-ls", "/"] interval: 1m @@ -404,6 +408,8 @@ services: environment: CLUSTER_NAME: radar-cns HDFS_CONF_dfs_namenode_name_dir: file:///hadoop/dfs/name/1,file:///hadoop/dfs/name/2 + HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_enable: "false" + HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_policy: "NEVER" healthcheck: test: ["CMD", "hdfs", "dfs", "-ls", "/"] interval: 1m From b0acbf11079aff9afd69a47f23d308a6637c97d0 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 5 Jul 2018 10:37:07 +0000 Subject: [PATCH 022/194] add postgres migration script --- .../migrate-postgres-data.sh | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh b/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh new file mode 100755 index 000000000..e0647d712 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +NEW_VERSION=10.4-alpine + + +. ./.env + +POSTGRES_NEW_DIR="${MP_POSTGRES_DIR}/data-${NEW_VERSION}" + +echo "Migrating ManagementPortal database to ${NEW_VERSION}" +if [ -e "${POSTGRES_NEW_DIR}" ]; then + echo "Please remove old temporary directory $POSTGRES_NEW_DIR before proceeding" + exit 1 +fi + +POSTGRES_NEW=$(docker run -d -v "${POSTGRES_NEW_DIR}/:/var/lib/postgresql/data" --env-file ./.env postgres:"${NEW_VERSION}") +sleep 5 + +docker-compose exec managementportal-postgresql pg_dumpall -U "${POSTGRES_USER}" \ + | docker exec -i ${POSTGRES_NEW} psql -U "${POSTGRES_USER}" + +docker rm -vf "${POSTGRES_NEW}" + +echo "Stopping postgres..." +docker-compose stop managementportal-postgresql +docker-compose rm -vf managementportal-postgresql + +echo "Moving dumped data to new volume" +mv "${MP_POSTGRES_DIR}/data/" "${MP_POSTGRES_DIR}/data-backup-$(date +%FT%TZ)/" +mv "${POSTGRES_NEW_DIR}" "${MP_POSTGRES_DIR}/data/" + +# change postgres version +sed -i "s| image: postgres:.*| image: postgres:${NEW_VERSION}|" docker-compose.yml + +echo "Starting postgres..." + +docker-compose up -d managementportal-postgresql + + From b734376b6fd79f74cefb3f020b0fe2ec06cf347f Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 5 Jul 2018 12:57:09 +0200 Subject: [PATCH 023/194] update schema version --- dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile index 4a84d7f31..f32682370 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile @@ -25,7 +25,7 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.3.2 +ENV RADAR_SCHEMAS_VERSION=0.3.4 RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 From c59d61b94ee058f3ad00d77447a68447426493be Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 5 Jul 2018 14:35:00 +0200 Subject: [PATCH 024/194] update postgres version with an env variable --- .../radar-cp-hadoop-stack/docker-compose.yml | 2 +- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 1 + .../radar-cp-hadoop-stack/migrate-postgres-data.sh | 10 ++++------ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index b22d0e523..168cc6407 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -678,7 +678,7 @@ services: managementportal-postgresql: - image: postgres:9.6-alpine + image: postgres:${POSTGRES_NEW_VERSION} volumes: - "${MP_POSTGRES_DIR}/data/:/var/lib/postgresql/data/" - "./postgres-backup/backups/postgresql:/backups/database/postgresql/" diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index becd3c8f4..a9246b534 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -21,3 +21,4 @@ MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT=false HEALTHCHECK_HIPCHAT_NOTIFY=no HEALTHCHECK_HIPCHAT_ROOM_ID="" HEALTHCHECK_HIPCHAT_TOKEN="" +POSTGRES_NEW_VERSION=10.4-alpine diff --git a/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh b/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh index e0647d712..37ebb5a5a 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh @@ -2,20 +2,18 @@ set -e -NEW_VERSION=10.4-alpine - . ./.env -POSTGRES_NEW_DIR="${MP_POSTGRES_DIR}/data-${NEW_VERSION}" +POSTGRES_NEW_DIR="${MP_POSTGRES_DIR}/data-${POSTGRES_NEW_VERSION}" -echo "Migrating ManagementPortal database to ${NEW_VERSION}" +echo "Migrating ManagementPortal database to ${POSTGRES_NEW_VERSION}" if [ -e "${POSTGRES_NEW_DIR}" ]; then echo "Please remove old temporary directory $POSTGRES_NEW_DIR before proceeding" exit 1 fi -POSTGRES_NEW=$(docker run -d -v "${POSTGRES_NEW_DIR}/:/var/lib/postgresql/data" --env-file ./.env postgres:"${NEW_VERSION}") +POSTGRES_NEW=$(docker run -d -v "${POSTGRES_NEW_DIR}/:/var/lib/postgresql/data" --env-file ./.env postgres:"${POSTGRES_NEW_VERSION}") sleep 5 docker-compose exec managementportal-postgresql pg_dumpall -U "${POSTGRES_USER}" \ @@ -32,7 +30,7 @@ mv "${MP_POSTGRES_DIR}/data/" "${MP_POSTGRES_DIR}/data-backup-$(date +%FT%TZ)/" mv "${POSTGRES_NEW_DIR}" "${MP_POSTGRES_DIR}/data/" # change postgres version -sed -i "s| image: postgres:.*| image: postgres:${NEW_VERSION}|" docker-compose.yml +sed -i "s| image: postgres:.*| image: postgres:${POSTGRES_NEW_VERSION}|" docker-compose.yml echo "Starting postgres..." From 1ebd3417b1961c7217aea01287a9ca54c9247195 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 10 Jul 2018 16:40:14 +0100 Subject: [PATCH 025/194] Add command line parser and changes based on comments --- scripts/README.md | 27 ++-- .../hdfs_data_retention.sh | 117 +++++++++++++----- .../hdfs_get_relevant_files.pig | 28 ++--- 3 files changed, 114 insertions(+), 58 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 133f56a3a..8fe725001 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,7 +4,7 @@ This folder contains useful scripts to manage the server where the RADAR-base Pl ### `check_radar_network.sh` **It checks if the machine is connected to internet. The script can be parametrised with the following** -- + - `nic` is the internet gateway - `lockfile` lock usefull to check whether there is a previous instance still running - `logfile` is the log file where the script logs each operation @@ -30,17 +30,13 @@ Before deploying the task, make sure that all paths used by the script are absol ### `hdfs-data-retention/hdfs_data_retention.sh` **It is a script for deleting records from hdfs based on name of the topic and the date. All the records for the current topics older than the specified date are deleted from HDFS.** - - - `OUTPUT_DIR` - the directory where FS image file and extracted data will be stored. Default is `./tmp` - - `date_time_to_remove_before` - All records for appropriate topics before this date will be removed from HDFS. - - `HDFS_NAME_NODE` - The url of the hdfs namenode to download the FS image file and delete files. - - `hdfs-data-retention/topics_to_remove.txt` - The file used by the above script to delete files from these topics. Please specify each topic on a new line. + - `hdfs-data-retention/topics_to_remove.txt` - The default file used by the above script to delete files from these topics. Please specify each topic on a new line. Usage: To just get the FS image file and process it and list the sum of file sizes of all the relevant files using apache pig, run the command like - ```shell cd hdfs-data-retention -sudo bash hdfs_data_retention.sh +sudo bash hdfs_data_retention.sh --date "2018-03-15 12:00" ``` This will output the file sizes sum of the calculated paths like - ``` @@ -48,14 +44,21 @@ This will output the file sizes sum of the calculated paths like - ``` and also store the finalised path meeting the conditions of topics and date in the `./tmp/final_paths/part_r_00000` -To also delete the files listed by the command above, just run - -```shell -cd hdfs-data-retention -sudo bash hdfs_data_retention.sh delete +To also delete the files and other options see below - +``` +Usage: ./hdfs_data_retention.sh --date [Options...] +Options: ** means required + + -d|--delete: enable delete for the data. If not specified, the size of selected files is displayed. + -st|--skip-trash: Enables skipTrash option for . To be used with -d|--delete option. +* -u|--url: The HDFS namenode Url to connect to. Default is hdfs://hdfs-namenode:8020 +* -tf|--topics-file: The path of the file containing the newline-separated list of topics to remove the files from. Default is ./topics_to_remove.txt +** -dt|--date: All the files modified before this date time will be selected. Format is (yyyy-MM-dd HH:mm) ``` +Recommended use of the script for large filesystems is via a Cron job or a Screen session as it may take some time to delete all the files. Info: By default the script is set up to run against docker containers in the RADAR-base stack. -The script will use the hdfs.image and hdfs.txt files from `./tmp` folder if present. To get a new FS image file from namenode, delete these files first and then run the script. +The script will use the hdfs.image and hdfs.txt files from `./tmp` folder if delete is specified and the files are not older than a day. If you get JAVA_HOME not set error, please uncomment and specify the JAVA_HOME in the script. diff --git a/scripts/hdfs-data-retention/hdfs_data_retention.sh b/scripts/hdfs-data-retention/hdfs_data_retention.sh index ee38426b9..01ac1a146 100644 --- a/scripts/hdfs-data-retention/hdfs_data_retention.sh +++ b/scripts/hdfs-data-retention/hdfs_data_retention.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash -# All records for appropriate topics before this date will be removed from HDFS. -date_time_to_remove_before='2018-03-15 12:00' +PIG_VERSION="0.16.0" OUTPUT_DIR="./tmp" OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" +TOPICS_FILE="./topics_to_remove.txt" HDFS_NAME_NODE='hdfs://hdfs-namenode:8020' +DELETE="false" +SKIP_TRASH="" # HDFS command to get FS image file from hdfs name node HDFS_COMMAND_IMAGE=(hdfs dfsadmin -fetchImage /fsimage_tmp/hdfs.image) @@ -14,62 +16,113 @@ HDFS_COMMAND_IMAGE=(hdfs dfsadmin -fetchImage /fsimage_tmp/hdfs.image) # HDFS command to create text file from FSImage file HDFS_COMMAND_TEXT=(hadoop oiv -i /fsimage_tmp/hdfs.image -o /fsimage_tmp/hdfs.txt -p Delimited -delimiter ,) +DOCKER_COMMAND=(docker run -i --rm --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2) + + if [[ ! -d 'tmp' ]]; then mkdir tmp fi -if [[ ! -f './tmp/hdfs.image' ]]; then - echo "Downloading a new FS image file at ./tmp and converting to txt." - docker run -i --rm --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 ${HDFS_COMMAND_IMAGE[@]} +while [[ $# -gt 0 ]] +do +key="$1" - docker run -i --rm --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 ${HDFS_COMMAND_TEXT[@]} -else - echo "./tmp/hdfs.image already exists. Using the same FS image file. " - if [[ ! -f './tmp/hdfs.txt' ]]; then - docker run -i --rm --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 ${HDFS_COMMAND_TEXT[@]} - else - echo "./tmp/hdfs.txt already exists. Not generating a new one. " - fi +case $key in + -d|--delete) + DELETE="true" + shift # past argument + ;; + -st|--skip-trash) + SKIP_TRASH="-skipTrash" + shift # past argument + ;; + -u|--url) + HDFS_NAME_NODE="$2" + shift # past argument + shift # past value + ;; + -tf|--topics-file) + TOPICS_FILE="$2" + shift # past argument + shift # past value + ;; + -dt|--date) + if [[ "$2" =~ ^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])[[:space:]]([0-1][0-9]|2[0-3]):([0-5][0-9])$ ]]; then + # All records for appropriate topics before this date will be removed from HDFS. + date_time_to_remove_before="$2" + else + echo "Invalid date. Please use -h or --help for more information." + exit 1 + fi + shift # past argument + shift # past value + ;; + -h|--help) + echo "Usage: ./hdfs_data_retention.sh -u -tf -dt -d" + echo "Options: * means required" + echo " -d|--delete: enable delete for the data. If not specified, the size of selected files is displayed." + echo " -st|--skip-trash: Enables skipTrash option for . To be used with -d|--delete option." + echo " -h|--help: Displays this help." + echo "* -u|--url: The HDFS namenode Url to connect to. Default is hdfs://hdfs-namenode:8020" + echo "* -tf|--topics-file: The path of the file containing the newline-separated list of topics to remove the files from. Default is ./topics_to_remove.txt" + echo "* -dt|--date: All the files modified before this date time will be selected. Format is (yyyy-MM-dd HH:mm)" + exit 0 + ;; + *) # unknown option + echo "Unknown Option $1. Try again. Use -h or --help for more info." + exit 1 + ;; +esac +done + +if [[ -z "$date_time_to_remove_before" ]]; then + echo "Please specify a date and time. See -h or --help for more information." + exit 1 fi +if [[ "${DELETE}" = "true" ]] && [[ -f "./tmp/hdfs.image" ]]; then + if [[ $(find ./tmp/hdfs.image -mtime +1 -print) ]]; then + echo "./tmp/hdfs.image is older than a day. Downloading a new FS image file. " + ${DOCKER_COMMAND[@]} ${HDFS_COMMAND_IMAGE[@]} + ${DOCKER_COMMAND[@]} ${HDFS_COMMAND_TEXT[@]} + fi +else + echo "Downloading a new FS image file and converting to txt. " + ${DOCKER_COMMAND[@]} ${HDFS_COMMAND_IMAGE[@]} + ${DOCKER_COMMAND[@]} ${HDFS_COMMAND_TEXT[@]} +fi # Set this if get JAVA_HOME not set error or set it in ~/.profile #export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" # Download and configure apache pig -export PIG_HOME="$(pwd)"/pig-0.16.0 -export PATH=$PATH:"$(pwd)"/pig-0.16.0/bin +export PIG_HOME="$(pwd)"/pig-"${PIG_VERSION}" +export PATH=$PATH:"$(pwd)"/pig-"${PIG_VERSION}"/bin if ! hash "pig" >/dev/null 2>&1; then - wget http://www-us.apache.org/dist/pig/pig-0.16.0/pig-0.16.0.tar.gz - tar -xzf pig-0.16.0.tar.gz - export PATH=$PATH:"$(pwd)"/pig-0.16.0/bin + wget http://www-us.apache.org/dist/pig/pig-"${PIG_VERSION}"/pig-"${PIG_VERSION}".tar.gz + tar -xzf pig-"${PIG_VERSION}".tar.gz + export PATH=$PATH:"$(pwd)"/pig-"${PIG_VERSION}"/bin fi # Write all the relevant file paths to file using apache pig -pig -x local -param inputFile=./tmp/hdfs.txt -param outputFile=./tmp/final_paths -param topics=./topics_to_remove.txt -param time="${date_time_to_remove_before}" ./hdfs_get_relevant_files.pig +pig -x local -param inputFile=./tmp/hdfs.txt -param outputFile=./tmp/final_paths -param topics=${TOPICS_FILE} -param time="${date_time_to_remove_before}" ./hdfs_get_relevant_files.pig FINAL_PATH='./tmp/final_paths/part-r-00000' -NUMOFLINES=$(wc -l < "${FINAL_PATH}") +#NUMOFLINES=$(wc -l < "${FINAL_PATH}") # If delete is passed as an argument, only then delete the files from the HDFS. -if [[ "$1" = "delete" ]]; then - docker run -i -d --name "hdfs-delete" --network hadoop -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 /bin/bash +if [[ "${DELETE}" = "true" ]]; then + docker run -i -d --name "hdfs-delete" --network hadoop -v "${OUTPUT_DIR}:/fsimage_tmp" -e "CORE_CONF_fs_defaultFS=${HDFS_NAME_NODE}" uhopper/hadoop:2.7.2 /bin/bash # Wait for the container to start up sleep 30 if [[ -f "${FINAL_PATH}" ]]; then echo "READING AND REMOVING RELEVANT PATHS" - let "curr_prog = 0" - while IFS='' read -r line || [[ -n "$line" ]]; do - docker exec hdfs-delete hdfs dfs -rm "$line" > /dev/null - echo "Deleted from HDFS file: $line" - let "curr_prog += 1" - let "perc_prog = ((curr_prog*100)/NUMOFLINES)" - echo "Progress: $perc_prog %" - done < "${FINAL_PATH}" + docker exec hdfs-delete bash -c 'apt-get -y install pv && pv -ptI /fsimage_tmp/final_paths/part-r-00000 | xargs -n 100 hdfs dfs -rm ${SKIP_TRASH}' fi - echo "Complete deletion" - docker rm -f "hdfs-delete" + # Delete the image after delete operation is complete + rm -r ./tmp/hdfs.* + docker rm -f hdfs-delete fi rm -r ./tmp/final_paths/ diff --git a/scripts/hdfs-data-retention/hdfs_get_relevant_files.pig b/scripts/hdfs-data-retention/hdfs_get_relevant_files.pig index 00d4b26d9..94a8ecaf1 100644 --- a/scripts/hdfs-data-retention/hdfs_get_relevant_files.pig +++ b/scripts/hdfs-data-retention/hdfs_get_relevant_files.pig @@ -1,5 +1,5 @@ -- Load all of the fields from the file - A = LOAD '$inputFile' USING PigStorage(',') AS (path:chararray, + DATA = LOAD '$inputFile' USING PigStorage(',') AS (path:chararray, replication:int, modTime:chararray, accessTime:chararray, @@ -14,21 +14,21 @@ -- Grab just the path, size and modDate(in milliseconds) - B = FOREACH A GENERATE path, fileSize, org.apache.pig.builtin.ToMilliSeconds(org.apache.pig.builtin.ToDate(modTime, 'yyyy-MM-dd HH:mm', '+00:00')) as modTime:long; - C = FILTER B BY ((modTime < org.apache.pig.builtin.ToMilliSeconds(org.apache.pig.builtin.ToDate('$time', 'yyyy-MM-dd HH:mm', '+00:00'))) AND (path matches '^((?!tmp).)*.avro')); - -- DUMP C; + RELEVANT_FIELDS = FOREACH DATA GENERATE path, fileSize, ToMilliSeconds(ToDate(modTime, 'yyyy-MM-dd HH:mm', '+00:00')) as modTime:long; + RELEVANT_FILES = FILTER RELEVANT_FIELDS BY ((modTime < ToMilliSeconds(ToDate('$time', 'yyyy-MM-dd HH:mm', '+00:00'))) AND (path matches '^((?!tmp).)*.avro')); + -- DUMP RELEVANT_FILES; -- Load topics from the provided file - D = LOAD '$topics' USING PigStorage() AS (topic:chararray); + TOPICS = LOAD '$topics' USING PigStorage() AS (topic:chararray); - C_0 = FOREACH C GENERATE path, fileSize; - C_1 = CROSS C_0, D; - -- DUMP C_1; - E = FILTER C_1 BY (path matches SPRINTF('.*%s.*', topic)); + PATH_SIZE = FOREACH RELEVANT_FILES GENERATE path, fileSize; + PATH_SIZE_TOPIC = CROSS PATH_SIZE, TOPICS; + -- DUMP PATH_SIZE_TOPIC; + PATH_MATCHES_TOPIC = FILTER PATH_SIZE_TOPIC BY (path matches SPRINTF('.*%s.*', topic)); -- Calculate total file size - S = FOREACH (GROUP E ALL) GENERATE CONCAT('SUM OF FILES SIZES TO BE DELETED IN MB = ', (chararray)(org.apache.pig.builtin.SUM(E.fileSize) / 1024 / 1024)); - DUMP S; - F = FOREACH E GENERATE path; + SUM_FILE_SIZES = FOREACH (GROUP PATH_MATCHES_TOPIC ALL) GENERATE CONCAT('SUM OF FILES SIZES TO BE DELETED IN MB = ', (chararray)(SUM(PATH_MATCHES_TOPIC.fileSize) / 1024 / 1024)); + DUMP SUM_FILE_SIZES; + FINAL_PATHS = FOREACH PATH_MATCHES_TOPIC GENERATE path; -- Save results - -- DUMP F; - STORE F INTO '$outputFile'; + -- DUMP FINAL_PATH; + STORE FINAL_PATHS INTO '$outputFile'; From dd49828e8d280d28df50a7c698f14c792e8f69b2 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 10 Jul 2018 20:22:25 +0100 Subject: [PATCH 026/194] minor changes --- scripts/hdfs-data-retention/hdfs_data_retention.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/hdfs-data-retention/hdfs_data_retention.sh b/scripts/hdfs-data-retention/hdfs_data_retention.sh index 01ac1a146..ca1e3ac12 100644 --- a/scripts/hdfs-data-retention/hdfs_data_retention.sh +++ b/scripts/hdfs-data-retention/hdfs_data_retention.sh @@ -81,7 +81,7 @@ if [[ -z "$date_time_to_remove_before" ]]; then exit 1 fi -if [[ "${DELETE}" = "true" ]] && [[ -f "./tmp/hdfs.image" ]]; then +if [[ -f "./tmp/hdfs.image" ]]; then if [[ $(find ./tmp/hdfs.image -mtime +1 -print) ]]; then echo "./tmp/hdfs.image is older than a day. Downloading a new FS image file. " ${DOCKER_COMMAND[@]} ${HDFS_COMMAND_IMAGE[@]} @@ -118,7 +118,7 @@ if [[ "${DELETE}" = "true" ]]; then sleep 30 if [[ -f "${FINAL_PATH}" ]]; then echo "READING AND REMOVING RELEVANT PATHS" - docker exec hdfs-delete bash -c 'apt-get -y install pv && pv -ptI /fsimage_tmp/final_paths/part-r-00000 | xargs -n 100 hdfs dfs -rm ${SKIP_TRASH}' + docker exec hdfs-delete bash -c 'apt-get -y install pv && pv -pte /fsimage_tmp/final_paths/part-r-00000 | xargs -n 100 hdfs dfs -rm ${SKIP_TRASH}' fi # Delete the image after delete operation is complete rm -r ./tmp/hdfs.* From edd9f63e3ce849f71bbeaae561eed25113057369 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Jul 2018 15:39:10 +0200 Subject: [PATCH 027/194] Update to latest hdfs restructure script --- .../hdfs-restructure/hdfs_restructure.sh | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh index acff2a866..55ce2a785 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh @@ -8,14 +8,7 @@ fi . ./util.sh # HDFS restructure version -JAR_VERSION=0.3.2 -# HDFS restructure JAR -JAR="restructurehdfs-all-${JAR_VERSION}.jar" - -if [ ! -e "lib/${JAR}" ]; then - echo "Downloading HDFS restructuring JAR version ${JAR_VERSION}" - sudo-linux curl -L -# -o lib/${JAR} "https://github.com/RADAR-base/Restructure-HDFS-topic/releases/download/v${JAR_VERSION}/${JAR}" -fi +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.0 # HDFS filename to get HDFS_FILE=$1 @@ -25,7 +18,7 @@ OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" # Internal docker directory to write output to HDFS_OUTPUT_DIR=/output # HDFS command to run -HDFS_COMMAND=(/usr/bin/java -Dorg.radarcns.compression=gzip -Dorg.radarcns.deduplicate=true -jar "/${JAR}" hdfs://hdfs-namenode:8020 "$HDFS_FILE" "$HDFS_OUTPUT_DIR") +HDFS_COMMAND=(--compression gzip --deduplicate -u hdfs://hdfs-namenode:8020 -o "$HDFS_OUTPUT_DIR" "$HDFS_FILE" ) mkdir -p $OUTPUT_DIR -sudo-linux docker run -i --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" -v "$PWD/lib/${JAR}:/${JAR}" openjdk:8-jre-alpine "${HDFS_COMMAND[@]}" +sudo-linux docker run -t --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" $DOCKER_IMAGE "${HDFS_COMMAND[@]}" From 071e065e63bbe8e00b41cd64a13317d981956732 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 14:07:24 +0200 Subject: [PATCH 028/194] Refactor HDFS images --- .../radar-cp-hadoop-stack/README.md | 41 ++- .../{hdfs_extract.sh => bin/hdfs-extract} | 0 .../radar-cp-hadoop-stack/bin/hdfs-ha-init | 26 ++ .../hdfs-restructure} | 0 .../hdfs-restructure-process} | 2 +- .../radar-cp-hadoop-stack/bin/hdfs-upgrade | 45 +++ .../radar-cp-hadoop-stack/bin/radar-docker | 30 ++ .../{hdfs-restructure => bin}/util.sh | 0 .../docker-compose.hdfs.yml | 313 ++++++++++++++++++ .../radar-cp-hadoop-stack/docker-compose.yml | 122 +------ .../radar-cp-hadoop-stack/etc/env.template | 13 +- .../hdfs-namenode/Dockerfile | 4 - .../hdfs-namenode/run.sh | 24 -- .../hdfs-restructure/README.md | 29 -- .../images/hdfs/Dockerfile | 5 + .../images/hdfs/entrypoint.sh | 120 +++++++ .../images/hdfs/hdfs-site.xml.mustache | 110 ++++++ .../install-systemd-wrappers.sh | 4 +- 18 files changed, 703 insertions(+), 185 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/{hdfs_extract.sh => bin/hdfs-extract} (100%) create mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init rename dcompose-stack/radar-cp-hadoop-stack/{hdfs-restructure/hdfs_restructure.sh => bin/hdfs-restructure} (100%) rename dcompose-stack/radar-cp-hadoop-stack/{hdfs-restructure/restructure_backup_hdfs.sh => bin/hdfs-restructure-process} (88%) mode change 100644 => 100755 create mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade create mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker rename dcompose-stack/radar-cp-hadoop-stack/{hdfs-restructure => bin}/util.sh (100%) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/Dockerfile delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/run.sh delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md create mode 100644 dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile create mode 100755 dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh create mode 100644 dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 32bc096e4..c499c047f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -97,18 +97,18 @@ Otherwise, the following manual commands can be invoked. Raw data can be extracted from this setup by running: ```shell -./hdfs_extract.sh +hdfs/bin/hdfs-extract ``` This command will not overwrite data in the destination directory. CSV-structured data can be gotten from HDFS by running ```shell -./hdfs_restructure.sh /topicAndroidNew +hdfs/bin/hdfs-restructure /topicAndroidNew ``` This will put all CSV files in the destination directory, with subdirectory structure `ProjectId/SubjectId/SensorType/Date_Hour.csv`. -## Cerificate +## Certificate If systemd integration is enabled, the ssl certificate will be renewed daily. It can then be run directly by running ``` @@ -157,3 +157,38 @@ Also you will need to change the directory. So just add the following to the top cd "$( dirname "${BASH_SOURCE[0]}" )" ``` +### HDFS + +This folder contains useful scripts to manage the extraction of data from HDFS in the RADAR-base Platform. + +- `bin/hdfs-ha-init` + - This initializes the HDFS structure to use High Availability mode with two name nodes. This needs to be called after converting from + the older uhopper images. +- `bin/hdfs-upgrade VERSION` + - Perform an upgrade from an older version of the [Smizy HDFS base image](https://hub.docker.com/r/smizy/hadoop-base/) to a newer one. E.g. from `2.7.6-alpine`, which is compatible with the `uhopper` image, to `3.0.3-alpine`. +- `bin/hdfs-restructure` + - This script uses the Restructure-HDFS-topic to extracts records from HDFS and converts them from AVRO to specified format + - By default, the format is CSV, compression is set to gzip and deduplication is enabled. + - To change configurations and for more info look at the [README here](https://github.com/RADAR-base/Restructure-HDFS-topic) + +- `bin/hdfs-restructure-process` for running the above script in a controlled manner with rotating logs + - `logfile` is the log file where the script logs each operation + - `storage_directory` is the directory where the extracted data will be stored + - `lockfile` lock useful to check whether there is a previous instance still running + +- A systemd timer for this script can be installed by running the `../install-systemd-wrappers.sh`. Or you can add a cron job like below. + +To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u root` and add your task at the end of the file. The syntax is +```shell +* * * * * command to be executed +- - - - - +| | | | | +| | | | +----- day of week (0 - 6) (Sunday=0) +| | | +------- month (1 - 12) +| | +--------- day of month (1 - 31) +| +----------- hour (0 - 23) ++------------- min (0 - 59) +``` + +For example, `*/2 * * * * /absolute/path/to/script-name.sh` will execute `script-name.sh` every `2` minutes. + diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs_extract.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/hdfs_extract.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init new file mode 100755 index 000000000..37bc97262 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init @@ -0,0 +1,26 @@ +#!/bin/bash + +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) +STACK="${DIR}/bin/radar-docker" + +. "$DIR/.env" + +echo "Are you sure you want to initialize HDFS?" + +select yn in "Yes" "No"; do + case $yn in + Yes ) break;; + No ) exit;; + esac +done + +"$STACK" up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 +"$STACK" quit hdfs-namenode-1 hdfs-namenode-2 +"$STACK" run --rm hdfs-namenode-1 namenode-1 -initializeSharedEdits +"$STACK" up -d hdfs-namenode-1 +echo "Waiting for HDFS namenode to come online..." +while ! "$STACK" exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do + sleep 1 +done +"$STACK" run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force +"$STACK" hdfs \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/hdfs_restructure.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/restructure_backup_hdfs.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process old mode 100644 new mode 100755 similarity index 88% rename from dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/restructure_backup_hdfs.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process index 042ae247f..8041f2446 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/restructure_backup_hdfs.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process @@ -19,7 +19,7 @@ hdfs_directory=/topicAndroidNew if [ ! -f $lockfile ]; then log_info "Creating lock ..." touch $lockfile - (./hdfs_restructure.sh $hdfs_directory $storage_directory >> $logfile 2>&1) + (./hdfs-restructure $hdfs_directory $storage_directory >> $logfile 2>&1) log_info "Removing lock ..." rm $lockfile else diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade new file mode 100755 index 000000000..a6a20dde4 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade @@ -0,0 +1,45 @@ +#!/bin/bash + +if [ $# = 0 ]; then + echo "Usage: $0 VERSION" + exit 1 +fi + +VERSION=$1 +STACK_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) +STACK="${STACK_DIR}/bin/radar-docker" + +. "${STACK_DIR}/.env" +. "${STACK_DIR}/util.sh" + +echo "Are you sure you want to to upgrade HDFS from version ${HDFS_BASE_VERSION} to ${VERSION}?" + +select yn in "Yes" "No"; do + case $yn in + Yes ) break;; + No ) exit;; + esac +done + +"$STACK" quit hdfs-namenode-1 hdfs-namenode-2 radar-hdfs-connector + +sed_i "s/HDFS_BASE_VERSION=.*/HDFS_BASE_VERSION=${VERSION}/" "${STACK_DIR}/.env" + +"$STACK" build +"$STACK" up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 +sleep 10 +NAME_1=$("$STACK" run --rm -d --name hdfs-namenode-1 hdfs-namenode-1 namenode-1 -upgrade) + +echo "Waiting for the upgrade to finalize..." +while ! docker exec $NAME_1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do + sleep 1 +done + +"$STACK" run --no-deps --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby +"$STACK" up --no-deps -d hdfs-namenode-2 +sleep 10 +docker exec $NAME_1 hdfs dfsadmin -finalizeUpgrade + +docker rm -vf $NAME_1 +"$STACK" hdfs +"$STACK" up -d radar-hdfs-connector \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker new file mode 100755 index 000000000..8ef995eee --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -0,0 +1,30 @@ +#!/bin/bash +stack=(docker-compose -f docker-compose.yml -f docker-compose.hdfs.yml) + +if [ $# != 0 ]; then + CMD=$1 + shift + + case "${CMD}" in + "rebuild") + ${stack[@]} build "$@" && \ + ${stack[@]} stop "$@" && \ + ${stack[@]} rm -vf "$@" && \ + ${stack[@]} up -d "$@" && \ + exit 0 || exit 1 + ;; + "quit") + ${stack[@]} stop "$@" && \ + ${stack[@]} rm -vf "$@" && \ + exit 0 || exit 1 + ;; + "hdfs") + exec ${stack[@]} up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 + ;; + *) + exec ${stack[@]} "$CMD" "$@" + ;; + esac +else + exec ${stack[@]} +fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/util.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/util.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/util.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/util.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml new file mode 100644 index 000000000..bf924e8cc --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml @@ -0,0 +1,313 @@ +--- +version: '2.4' + +networks: + hadoop: + external: true + +services: + #---------------------------------------------------------------------------# + # RADAR Cold Storage # + #---------------------------------------------------------------------------# + hdfs-datanode-1: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: datanode + networks: + - hadoop + - zookeeper + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - hdfs-namenode-1 + - hdfs-namenode-2 + volumes: + - "${HDFS_DATA_DIR_1}:/hadoop/dfs/data" + restart: always + environment: + SERVICE_9866_NAME: datanode + SERVICE_9867_IGNORE: "true" + SERVICE_9864_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 + HADOOP_DFS_REPLICATION: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-datanode-2: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: datanode + networks: + - hadoop + - zookeeper + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - hdfs-namenode-1 + - hdfs-namenode-2 + volumes: + - "${HDFS_DATA_DIR_2}:/hadoop/dfs/data" + restart: always + environment: + SERVICE_9866_NAME: datanode + SERVICE_9867_IGNORE: "true" + SERVICE_9864_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 + HADOOP_DFS_REPLICATION: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-datanode-3: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: datanode + networks: + - hadoop + - zookeeper + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - hdfs-namenode-1 + - hdfs-namenode-2 + volumes: + - "${HDFS_DATA_DIR_3}:/hadoop/dfs/data" + restart: always + environment: + SERVICE_9866_NAME: datanode + SERVICE_9867_IGNORE: "true" + SERVICE_9864_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 + HADOOP_DFS_REPLICATION: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-namenode-1: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: namenode-1 + networks: + - hadoop + - zookeeper + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - hdfs-journalnode-1 + - hdfs-journalnode-2 + - hdfs-journalnode-3 + volumes: + - "${HDFS_NAME_DIR_1}:/hadoop/dfs/name" + restart: always + environment: + SERVICE_8020_NAME: namenode + SERVICE_9870_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 + HADOOP_QJOURNAL_ADDRESS: hdfs-journalnode-1:8485;hdfs-journalnode-2:8485;hdfs-journalnode-3:8485 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-namenode-2: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: namenode-2 + networks: + - hadoop + - zookeeper + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - hdfs-journalnode-1 + - hdfs-journalnode-2 + - hdfs-journalnode-3 + - hdfs-namenode-1 + volumes: + - "${HDFS_NAME_DIR_2}:/hadoop/dfs/name" + restart: always + environment: + SERVICE_8020_NAME: namenode + SERVICE_9870_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 + HADOOP_DFS_REPLICATION: 2 + HADOOP_QJOURNAL_ADDRESS: hdfs-journalnode-1:8485;hdfs-journalnode-2:8485;hdfs-journalnode-3:8485 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-journalnode-1: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: journalnode + networks: + - hadoop + - zookeeper + volumes: + - "${HDFS_JOURNAL_DIR_1}:/hadoop/dfs/journal" + restart: always + environment: + SERVICE_8485_NAME: journalnode + SERVICE_8480_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-journalnode-2: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: journalnode + networks: + - hadoop + - zookeeper + volumes: + - "${HDFS_JOURNAL_DIR_2}:/hadoop/dfs/journal" + restart: always + environment: + SERVICE_8485_NAME: journalnode + SERVICE_8480_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-journalnode-3: + build: + context: ./hdfs/images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarcns/hdfs:${HDFS_BASE_VERSION} + command: journalnode + networks: + - hadoop + - zookeeper + volumes: + - "${HDFS_JOURNAL_DIR_3}:/hadoop/dfs/journal" + restart: always + environment: + SERVICE_8485_NAME: journalnode + SERVICE_8480_IGNORE: "true" + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE_HA: nn1,nn2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-ls", "/"] + interval: 1m + timeout: 10s + retries: 3 + + #---------------------------------------------------------------------------# + # RADAR HDFS connector # + #---------------------------------------------------------------------------# + radar-hdfs-connector: + image: radarbase/radar-hdfs-connector-auto:0.2.0 + restart: on-failure + volumes: + - ../etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties + networks: + - zookeeper + - kafka + - hadoop + depends_on: + - zookeeper-1 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + - kafka-init + - hdfs-datanode-1 + - hdfs-datanode-2 + - hdfs-datanode-3 + - hdfs-namenode-1 + - hdfs-namenode-2 + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "default" + CONNECT_CONFIG_STORAGE_TOPIC: "default.config" + CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" + CONNECT_STATUS_STORAGE_TOPIC: "default.status" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + CONNECT_LOG4J_ROOT_LOGLEVEL: WARN + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" + healthcheck: + test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] + interval: 1m + timeout: 5s + retries: 3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index ff2d7534e..23c371c9b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -1,5 +1,5 @@ --- -version: '2.1' +version: '2.4' networks: zookeeper: @@ -11,8 +11,6 @@ networks: api: driver: bridge internal: true - hadoop: - external: true mail: driver: bridge internal: true @@ -357,65 +355,6 @@ services: timeout: 5s retries: 3 - #---------------------------------------------------------------------------# - # RADAR Cold Storage # - #---------------------------------------------------------------------------# - hdfs-datanode-1: - image: uhopper/hadoop-datanode:2.7.2 - networks: - - hadoop - volumes: - - "${HDFS_DATA_DIR_1}:/hadoop/dfs/data" - restart: always - environment: - CORE_CONF_fs_defaultFS: hdfs://hdfs-namenode:8020 - HDFS_CONF_dfs_replication: 2 - HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_enable: "false" - HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_policy: "NEVER" - healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-datanode-2: - image: uhopper/hadoop-datanode:2.7.2 - networks: - - hadoop - volumes: - - "${HDFS_DATA_DIR_2}:/hadoop/dfs/data" - restart: always - environment: - CORE_CONF_fs_defaultFS: hdfs://hdfs-namenode:8020 - HDFS_CONF_dfs_replication: 2 - HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_enable: "false" - HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_policy: "NEVER" - healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-namenode: - build: hdfs-namenode - image: radarcns/hdfs-namenode:2.7.2 - networks: - - hadoop - volumes: - - "${HDFS_NAME_DIR_1}:/hadoop/dfs/name/1" - - "${HDFS_NAME_DIR_2}:/hadoop/dfs/name/2" - restart: always - environment: - CLUSTER_NAME: radar-cns - HDFS_CONF_dfs_namenode_name_dir: file:///hadoop/dfs/name/1,file:///hadoop/dfs/name/2 - HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_enable: "false" - HDFS_CONF_dfs_client_block_write_replace___datanode___on___failure_policy: "NEVER" - healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] - interval: 1m - timeout: 10s - retries: 3 - #---------------------------------------------------------------------------# # Email server # #---------------------------------------------------------------------------# @@ -463,7 +402,7 @@ services: CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kcontext: ./images/hdfsonnect.json.JsonConverter" CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" CONNECT_REST_ADVERTISED_HOST_NAME: "radar-mongodb-connector" CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 @@ -481,60 +420,7 @@ services: timeout: 5s retries: 3 - - #---------------------------------------------------------------------------# - # RADAR HDFS connector # - #---------------------------------------------------------------------------# - radar-hdfs-connector: - image: radarbase/radar-hdfs-connector-auto:0.2.0 - restart: on-failure - volumes: - - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties - networks: - - zookeeper - - kafka - - hadoop - depends_on: - - zookeeper-1 - - kafka-1 - - kafka-2 - - kafka-3 - - schema-registry-1 - - rest-proxy-1 - - kafka-init - - hdfs-datanode-1 - - hdfs-datanode-2 - - hdfs-namenode - environment: - CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 - CONNECT_REST_PORT: 8083 - CONNECT_GROUP_ID: "default" - CONNECT_CONFIG_STORAGE_TOPIC: "default.config" - CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" - CONNECT_STATUS_STORAGE_TOPIC: "default.status" - CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" - CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" - CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" - CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" - CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 - CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" - KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" - KAFKA_BROKERS: 3 - CONNECT_LOG4J_ROOT_LOGLEVEL: WARN - CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" - # For backwards compatibility - TOPIC_LIST: "application_record_counts" - healthcheck: - test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] - interval: 1m - timeout: 5s - retries: 3 - - #---------------------------------------------------------------------------# + #------------------------------------------------------------------context: ./images/hdfs---# # RADAR backend streams # #---------------------------------------------------------------------------# radar-backend-stream: @@ -749,7 +635,7 @@ services: # Catalog server from radar-schemas # #---------------------------------------------------------------------------# catalog-server: - build: kafka-radarinit + build: images/kafka-radarinit image: radarcns/kafka-init networks: - management diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index becd3c8f4..591b8e1e8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -8,10 +8,6 @@ RADAR_AGG_TOPIC_LIST= HOTSTORAGE_USERNAME= HOTSTORAGE_PASSWORD=XXXXXXXX HOTSTORAGE_NAME= -HDFS_DATA_DIR_1=/usr/local/var/lib/docker/hdfs-data-1 -HDFS_DATA_DIR_2=/usr/local/var/lib/docker/hdfs-data-2 -HDFS_NAME_DIR_1=/usr/local/var/lib/docker/hdfs-name-1 -HDFS_NAME_DIR_2=/usr/local/var/lib/docker/hdfs-name-2 MONGODB_DIR=/usr/local/var/lib/docker/mongodb MP_POSTGRES_DIR=/usr/local/var/lib/docker/postgres POSTGRES_USER= @@ -21,3 +17,12 @@ MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT=false HEALTHCHECK_HIPCHAT_NOTIFY=no HEALTHCHECK_HIPCHAT_ROOM_ID="" HEALTHCHECK_HIPCHAT_TOKEN="" +HDFS_BASE_VERSION=3.0.3-alpine +HDFS_DATA_DIR_1=/usr/local/var/lib/docker/hdfs-data-1 +HDFS_DATA_DIR_2=/usr/local/var/lib/docker/hdfs-data-2 +HDFS_DATA_DIR_3=/usr/local/var/lib/docker/hdfs-data-3 +HDFS_NAME_DIR_1=/usr/local/var/lib/docker/hdfs-name-1 +HDFS_NAME_DIR_2=/usr/local/var/lib/docker/hdfs-name-2 +HDFS_JOURNAL_DIR_1=/usr/local/var/lib/docker/hdfs-journal-1 +HDFS_JOURNAL_DIR_2=/usr/local/var/lib/docker/hdfs-journal-2 +HDFS_JOURNAL_DIR_3=/usr/local/var/lib/docker/hdfs-journal-3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/Dockerfile deleted file mode 100644 index 82f947883..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM uhopper/hadoop-namenode:2.7.2 - -ADD run.sh /run.sh -RUN chmod a+x /run.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/run.sh b/dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/run.sh deleted file mode 100644 index c4f7bbb2e..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs-namenode/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -if [ -z "$CLUSTER_NAME" ]; then - echo "Cluster name not specified" - exit 2 -fi - -IFS=',' read -r -a namedirs <<< $(echo "$HDFS_CONF_dfs_namenode_name_dir" | sed -e 's#file://##g') - -for namedir in "${namedirs[@]}"; do - mkdir -p "$namedir" - if [ ! -d "$namedir" ]; then - echo "Namenode name directory not found: $namedir" - exit 2 - fi - - if [ -z "$(ls -A "$namedir")" ]; then - echo "Formatting namenode name directory: $namedir is not yet formatted" - $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME - break - fi -done - -$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode diff --git a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md b/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md deleted file mode 100644 index b5250b81d..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/hdfs-restructure/README.md +++ /dev/null @@ -1,29 +0,0 @@ -## Scripts - -This folder contains useful scripts to manage the extraction of data from HDFS in the RADAR-base Platform. - -- `hdfs_restructure.sh` - - This script uses the Restructure-HDFS-topic to extracts records from HDFS and converts them from AVRO to specified format - - By default, the format is CSV, compression is set to gzip and deduplication is enabled. - - To change configurations and for more info look at the [README here](https://github.com/RADAR-base/Restructure-HDFS-topic) - -- `restracture_backup_hdfs.sh` for running the above script in a controlled manner with rotating logs - - `logfile` is the log file where the script logs each operation - - `storage_directory` is the directory where the extracted data will be stored - - `lockfile` lock useful to check whether there is a previous instance still running - -- A systemd timer for this script can be installed by running the `../install-systemd-wrappers.sh`. Or you can add a cron job like below. - -To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u root` and add your task at the end of the file. The syntax is -```shell -* * * * * command to be executed -- - - - - -| | | | | -| | | | +----- day of week (0 - 6) (Sunday=0) -| | | +------- month (1 - 12) -| | +--------- day of month (1 - 31) -| +----------- hour (0 - 23) -+------------- min (0 - 59) -``` - -For example, `*/2 * * * * /absolute/path/to/script-name.sh` will execute `script-name.sh` every `2` minutes. diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile new file mode 100644 index 000000000..f4a33fdc9 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile @@ -0,0 +1,5 @@ +ARG BASE_VERSION=3.0.3-alpine +FROM smizy/hadoop-base:${BASE_VERSION} + +COPY ./hdfs-site.xml.mustache ${HDFS_CONF_DIR}/ +COPY ./entrypoint.sh /usr/local/bin/ diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh new file mode 100755 index 000000000..d66a7185e --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +set -eo pipefail + +wait_until() { + local hostname=${1?} + local port=${2?} + local retry=${3:-100} + local sleep_secs=${4:-2} + + local address_up=0 + + while [ ${retry} -gt 0 ] ; do + echo "Waiting until ${hostname}:${port} is up ... with retry count: ${retry}" + if nc -z ${hostname} ${port}; then + address_up=1 + break + fi + retry=$((retry-1)) + sleep ${sleep_secs} + done + + if [ $address_up -eq 0 ]; then + echo "GIVE UP waiting until ${hostname}:${port} is up! " + exit 1 + fi +} + +# apply template +for template in $(ls ${HADOOP_CONF_DIR}/*.mustache) +do + conf_file=${template%.mustache} + cat ${conf_file}.mustache | mustache.sh > ${conf_file} +done + +USAGE=$'Usage: $0 [CMD] ...\n\tCMD: journalnode|namenode-1|namenode-2|datanode|resourcemanager-1|nodemanager|historyserver-1' + +if [ "$#" == "0" ]; then + echo "$USAGE" + exit 1 +fi + +CMD=$1 +shift + +case $CMD in +"journalnode") + exec su-exec hdfs hdfs journalnode "$@" + ;; +"namenode-1") + if [ ! -e "${HADOOP_TMP_DIR}/dfs/name/current/VERSION" ]; then + su-exec hdfs hdfs namenode -format -force + if [ "${HADOOP_NAMENODE_HA}" != "" ]; then + su-exec hdfs hdfs zkfc -formatZK -force + fi + fi +# wait_until ${HADOOP_QJOURNAL_ADDRESS%%:*} 8485 + if [ "${HADOOP_NAMENODE_HA}" != "" ]; then + su-exec hdfs hdfs zkfc & + fi + exec su-exec hdfs hdfs namenode "$@" + ;; +"namenode-2") + if [ ! -e "${HADOOP_TMP_DIR}/dfs/name/current/VERSION" ]; then + wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 + su-exec hdfs hdfs namenode -bootstrapStandby + fi + su-exec hdfs hdfs zkfc & + exec su-exec hdfs hdfs namenode "$@" + ;; +"datanode") + wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 + exec su-exec hdfs hdfs datanode "$@" + ;; +"resourcemanager-1") + exec su-exec yarn yarn resourcemanager "$@" + ;; +"nodemanager") + wait_until ${YARN_RESOURCEMANAGER_HOSTNAME} 8031 + exec su-exec yarn yarn nodemanager "$@" + ;; +"historyserver-1") + wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 + + set +e -x + + su-exec hdfs hdfs dfs -ls /tmp > /dev/null 2>&1 + if [ $? -ne 0 ]; then + su-exec hdfs hdfs dfs -mkdir -p /tmp + su-exec hdfs hdfs dfs -chmod 1777 /tmp + fi + + su-exec hdfs hdfs dfs -ls /user > /dev/null 2>&1 + if [ $? -ne 0 ]; then + su-exec hdfs hdfs dfs -mkdir -p /user/hdfs + su-exec hdfs hdfs dfs -chmod 755 /user + fi + + su-exec hdfs hdfs dfs -ls ${YARN_REMOTE_APP_LOG_DIR} > /dev/null 2>&1 + if [ $? -ne 0 ]; then + su-exec yarn hdfs dfs -mkdir -p ${YARN_REMOTE_APP_LOG_DIR} + su-exec yarn hdfs dfs -chmod -R 1777 ${YARN_REMOTE_APP_LOG_DIR} + su-exec yarn hdfs dfs -chown -R yarn:hadoop ${YARN_REMOTE_APP_LOG_DIR} + fi + + su-exec hdfs hdfs dfs -ls ${YARN_APP_MAPRED_STAGING_DIR} > /dev/null 2>&1 + if [ $? -ne 0 ]; then + su-exec mapred hdfs dfs -mkdir -p ${YARN_APP_MAPRED_STAGING_DIR} + su-exec mapred hdfs dfs -chmod -R 1777 ${YARN_APP_MAPRED_STAGING_DIR} + su-exec mapred hdfs dfs -chown -R mapred:hadoop ${YARN_APP_MAPRED_STAGING_DIR} + fi + + set -e +x + + exec su-exec mapred mapred historyserver "$@" + ;; +*) + exec "$CMD" "$@" + ;; +esac diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache new file mode 100644 index 000000000..7c1804fc2 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache @@ -0,0 +1,110 @@ + + + + + + + + + + dfs.permissions.superusergroup + hadoop + + + + dfs.nameservices + {{HADOOP_CLUSTER_NAME}} + + + + dfs.replication + {{HADOOP_DFS_REPLICATION}} + + +{{#HADOOP_NAMENODE_HA}} + + dfs.ha.namenodes.{{HADOOP_CLUSTER_NAME}} + {{HADOOP_NAMENODE_HA}} + + + + dfs.namenode.rpc-address.{{HADOOP_CLUSTER_NAME}}.nn1 + {{HADOOP_NAMENODE1_HOSTNAME}}:8020 + + + + dfs.namenode.http-address.{{HADOOP_CLUSTER_NAME}}.nn1 + {{HADOOP_NAMENODE1_HOSTNAME}}:9870 + + + + dfs.namenode.rpc-address.{{HADOOP_CLUSTER_NAME}}.nn2 + {{HADOOP_NAMENODE2_HOSTNAME}}:8020 + + + + dfs.namenode.http-address.{{HADOOP_CLUSTER_NAME}}.nn2 + {{HADOOP_NAMENODE2_HOSTNAME}}:9870 + + + + dfs.namenode.shared.edits.dir + qjournal://{{HADOOP_QJOURNAL_ADDRESS}}/{{HADOOP_CLUSTER_NAME}} + + + + dfs.client.failover.proxy.provider.{{HADOOP_CLUSTER_NAME}} + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + + dfs.ha.fencing.methods + shell(/usr/local/bin/fence.sh) + + + + dfs.journalnode.edits.dir + {{HADOOP_TMP_DIR}}/dfs/journal/data + + + + dfs.ha.automatic-failover.enabled + true + + +{{/HADOOP_NAMENODE_HA}} + +{{^HADOOP_NAMENODE_HA}} + + dfs.namenode.rpc-address.{{HADOOP_CLUSTER_NAME}} + {{HADOOP_NAMENODE1_HOSTNAME}}:8020 + + + + dfs.namenode.http-address.{{HADOOP_CLUSTER_NAME}} + {{HADOOP_NAMENODE1_HOSTNAME}}:9870 + +{{/HADOOP_NAMENODE_HA}} + + + dfs.client.block.write.replace-datanode-on-failure.policy + NEVER + + + + dfs.client.block.write.replace-datanode-on-failure.enable + false + + diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh index eb13bc0da..4dee0f8a6 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh @@ -17,8 +17,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-docker.service inline_variable 'ExecStart=' "$DIR/lib/systemd/start-radar-stack.sh" /etc/systemd/system/radar-docker.service -inline_variable 'WorkingDirectory=' "$DIR/hdfs-restructure" /etc/systemd/system/radar-output.service -inline_variable 'ExecStart=' "$DIR/hdfs-restructure/restructure_backup_hdfs.sh" /etc/systemd/system/radar-output.service +inline_variable 'WorkingDirectory=' "$DIR/hdfs" /etc/systemd/system/radar-output.service +inline_variable 'ExecStart=' "$DIR/bin/hdfs-restructure-process" /etc/systemd/system/radar-output.service inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-check-health.service inline_variable 'ExecStart=' "$DIR/check-health.sh" /etc/systemd/system/radar-check-health.service From b1a48c9d640d03f5db9983d733d07ad9795ec947 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:09:34 +0200 Subject: [PATCH 029/194] Refactor script locations --- .../radar-cp-hadoop-stack/README.md | 43 ++-- .../{docker-prune.sh => bin/docker-prune} | 4 +- .../radar-cp-hadoop-stack/bin/hdfs-extract | 9 +- .../radar-cp-hadoop-stack/bin/hdfs-ha-init | 19 +- .../bin/hdfs-restructure | 6 +- .../bin/hdfs-restructure-process | 4 +- .../radar-cp-hadoop-stack/bin/hdfs-upgrade | 26 +-- .../radar-cp-hadoop-stack/bin/lock-util.sh | 41 ++++ .../radar-cert-renew} | 4 +- .../radar-cp-hadoop-stack/bin/radar-docker | 29 ++- .../{check-health.sh => bin/radar-health} | 20 +- .../radar-kafka-consumer} | 17 +- .../radar-systemd-wrappers} | 20 +- .../radar-cp-hadoop-stack/bin/util.sh | 201 +++++++++++++++--- .../docker-compose.hdfs.yml | 38 ++-- .../radar-cp-hadoop-stack/docker-compose.yml | 8 +- .../images/hdfs/entrypoint.sh | 40 ++-- .../images/hdfs/hdfs-site.xml.mustache | 12 +- .../radar-kafka-init}/Dockerfile | 0 .../radar-kafka-init}/init.sh | 0 .../radar-kafka-init}/list_aggregated.sh | 0 .../radar-kafka-init}/list_raw.sh | 0 .../radar-kafka-init}/topic_init.sh | 0 .../lib/perform-install.sh | 118 ++++++++++ .../lib/systemd/start-radar-stack.sh | 6 - .../reboot-radar-stack.sh | 6 - .../start-radar-stack.sh | 6 - .../radar-cp-hadoop-stack/stop-radar-stack.sh | 6 - dcompose-stack/radar-cp-hadoop-stack/util.sh | 184 ---------------- 29 files changed, 496 insertions(+), 371 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/{docker-prune.sh => bin/docker-prune} (92%) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/bin/lock-util.sh rename dcompose-stack/radar-cp-hadoop-stack/{renew_ssl_certificate.sh => bin/radar-cert-renew} (75%) rename dcompose-stack/radar-cp-hadoop-stack/{check-health.sh => bin/radar-health} (87%) rename dcompose-stack/radar-cp-hadoop-stack/{docker-kacc.sh => bin/radar-kafka-consumer} (88%) rename dcompose-stack/radar-cp-hadoop-stack/{install-systemd-wrappers.sh => bin/radar-systemd-wrappers} (69%) mode change 100644 => 100755 dcompose-stack/radar-cp-hadoop-stack/bin/util.sh rename dcompose-stack/radar-cp-hadoop-stack/{kafka-radarinit => images/radar-kafka-init}/Dockerfile (100%) rename dcompose-stack/radar-cp-hadoop-stack/{kafka-radarinit => images/radar-kafka-init}/init.sh (100%) rename dcompose-stack/radar-cp-hadoop-stack/{kafka-radarinit => images/radar-kafka-init}/list_aggregated.sh (100%) rename dcompose-stack/radar-cp-hadoop-stack/{kafka-radarinit => images/radar-kafka-init}/list_raw.sh (100%) rename dcompose-stack/radar-cp-hadoop-stack/{kafka-radarinit => images/radar-kafka-init}/topic_init.sh (100%) create mode 100755 dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/lib/systemd/start-radar-stack.sh delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/util.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index c499c047f..d4790a4e1 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -46,13 +46,13 @@ This docker-compose stack contains the full operational RADAR platform. Once con Run ```shell -./install-radar-stack.sh +bin/radar-docker install ``` -to start all the RADAR services. Use the `(start|stop|reboot)-radar-stack.sh` to start, stop or reboot it. Note: whenever `.env` or `docker-compose.yml` are modified, this script needs to be called again. To start a reduced set of containers, call `install-radar-stack.sh` with the intended containers as arguments. +to start all the RADAR services. Use the `bin/radar-docker start|down|restart` to start, stop or reboot it. Note: whenever `.env` or `docker-compose.yml` are modified, this script needs to be called again. To start a reduced set of containers, call `bin/radar-docker install` with the intended containers as arguments. To enable a `systemd` service to control the platform, run ```shell -./install-systemd-wrappers.sh +bin/radar-systemd-wrappers ``` After that command, the RADAR platform should be controlled via `systemctl`. ```shell @@ -82,11 +82,18 @@ sudo systemctl disable radar-renew-certificate To clear all data from the platform, run ``` sudo systemctl stop radar-docker -./docker-prune.sh +bin/docker-prune sudo systemctl start radar-docker ``` -## Data extraction +### Monitoring a topic + +To see current data coming out of a Kafka topic, run +```script +bin/radar-kafka-consumer TOPIC +``` + +### Data extraction If systemd integration is enabled, HDFS data will be extracted to the `./output` directory every hour. It can then be run directly by running ``` @@ -97,25 +104,25 @@ Otherwise, the following manual commands can be invoked. Raw data can be extracted from this setup by running: ```shell -hdfs/bin/hdfs-extract +bin/hdfs-extract ``` This command will not overwrite data in the destination directory. CSV-structured data can be gotten from HDFS by running ```shell -hdfs/bin/hdfs-restructure /topicAndroidNew +bin/hdfs-restructure /topicAndroidNew ``` This will put all CSV files in the destination directory, with subdirectory structure `ProjectId/SubjectId/SensorType/Date_Hour.csv`. -## Certificate +### Certificate If systemd integration is enabled, the ssl certificate will be renewed daily. It can then be run directly by running ``` sudo systemctl start radar-renew-certificate.service ``` Otherwise, the following manual commands can be invoked. -If `SELF_SIGNED_CERT=no` in `./.env`, be sure to run `./renew_ssl_certificate.sh` daily to ensure that your certificate does not expire. +If `SELF_SIGNED_CERT=no` in `./.env`, be sure to run `bin/radar-cert-renew` daily to ensure that your certificate does not expire. ### cAdvisor @@ -133,28 +140,30 @@ Portainer provides simple interactive UI-based docker management. If running loc The [kafka-manager](https://github.com/yahoo/kafka-manager) is an interactive web based tool for managing Apache Kafka. Kafka manager has beed integrated in the stack. It is accessible at `http:///kafkamanager/` ### Check Health -Each of the containers in the stack monitor their own health and show the output as healthy or unhealthy. A script called check-health.sh is used to check this output and send an email to the maintainer if a container is unhealthy. +Each of the containers in the stack monitor their own health and show the output as healthy or unhealthy. A script called `bin/radar-health` is used to check this output and send an email to the maintainer if a container is unhealthy. First check that the `MAINTAINER_EMAIL` in the .env file is correct. Then make sure that the SMTP server is configured properly and running. -If systemd integration is enabled, the check-health.sh script will check health of containers every five minutes. It can then be run directly by running if systemd wrappers have been installed +If systemd integration is enabled, the `radar-health` script will check health of containers every five minutes. It can then be run directly by running if systemd wrappers have been installed ``` sudo systemctl start radar-check-health.service ``` Otherwise, the following manual commands can be invoked. -Add a cron job to run the `check-health.sh` script periodically like - +Add a cron job to run the `radar-health` script periodically like - 1. Edit the crontab file for the current user by typing `$ crontab -e` 2. Add your job and time interval. For example, add the following for checking health every 5 mins - -```*/5 * * * * /home/ubuntu/RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/check-health.sh``` +``` +*/5 * * * * /home/ubuntu/RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/bin/radar-health +``` -You can check the logs of CRON by typing `$ grep CRON /var/log/syslog` -Also you will need to change the directory. So just add the following to the top of the check-health.sh script - +You can check the logs of CRON by typing `grep CRON /var/log/syslog` +Also you will need to change the directory. So just add the following to the top of the `radar-health` script - ```sh -cd "$( dirname "${BASH_SOURCE[0]}" )" +cd "$( dirname "${BASH_SOURCE[0]}" )/.." ``` ### HDFS @@ -176,7 +185,7 @@ This folder contains useful scripts to manage the extraction of data from HDFS i - `storage_directory` is the directory where the extracted data will be stored - `lockfile` lock useful to check whether there is a previous instance still running -- A systemd timer for this script can be installed by running the `../install-systemd-wrappers.sh`. Or you can add a cron job like below. +- A systemd timer for this script can be installed by running the `bin/radar-systemd-wrappers`. Or you can add a cron job like below. To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u root` and add your task at the end of the file. The syntax is ```shell diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune similarity index 92% rename from dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune index 5dc1c228f..cef2c26d9 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-prune.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune @@ -1,6 +1,8 @@ #!/bin/bash -. ./util.sh +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +. bin/util.sh . ./.env check_command_exists docker diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract index fd72c5926..519bd4006 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract @@ -1,16 +1,19 @@ #!/bin/bash +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) + if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then printf "Usage:\n$0 []\nThe destination directory defaults to ./output\n" exit 1 fi -. ./util.sh +. "$DIR/bin/util.sh" +. "$DIR/.env" # HDFS filename to get HDFS_FILE=$1 # Absolute directory to write output to -OUTPUT_DIR=${2:-output} +OUTPUT_DIR=${2:-$DIR/output} OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" # Internal docker directory to write output to HDFS_OUTPUT_DIR=/home/output @@ -18,4 +21,4 @@ HDFS_OUTPUT_DIR=/home/output HDFS_COMMAND="hdfs dfs -get $HDFS_FILE $HDFS_OUTPUT_DIR" mkdir -p $OUTPUT_DIR -sudo-linux docker run -i --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" -e CLUSTER_NAME=radar-cns -e CORE_CONF_fs_defaultFS=hdfs://hdfs-namenode:8020 uhopper/hadoop:2.7.2 $HDFS_COMMAND +sudo-linux docker run -i --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" -e CORE_CONF_fs_defaultFS=hdfs://hdfs-namenode:8020 radarbase/hdfs:${HDFS_BASE_VERSION} $HDFS_COMMAND diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init index 37bc97262..4c65010d9 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init @@ -1,9 +1,8 @@ #!/bin/bash -DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) -STACK="${DIR}/bin/radar-docker" +cd "$(dirname "${BASH_SOURCE[0]}")/.." -. "$DIR/.env" +. ./.env echo "Are you sure you want to initialize HDFS?" @@ -14,13 +13,13 @@ select yn in "Yes" "No"; do esac done -"$STACK" up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 -"$STACK" quit hdfs-namenode-1 hdfs-namenode-2 -"$STACK" run --rm hdfs-namenode-1 namenode-1 -initializeSharedEdits -"$STACK" up -d hdfs-namenode-1 +bin/radar-stack up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 +bin/radar-stack quit hdfs-namenode-1 hdfs-namenode-2 +bin/radar-stack run --rm hdfs-namenode-1 namenode-1 -initializeSharedEdits +bin/radar-stack up -d hdfs-namenode-1 echo "Waiting for HDFS namenode to come online..." -while ! "$STACK" exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do +while ! bin/radar-stack exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do sleep 1 done -"$STACK" run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force -"$STACK" hdfs \ No newline at end of file +bin/radar-stack run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force +bin/radar-stack hdfs \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index acff2a866..11926c645 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -5,7 +5,9 @@ if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then exit 1 fi -. ./util.sh +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) + +. "$DIR/bin/util.sh" # HDFS restructure version JAR_VERSION=0.3.2 @@ -20,7 +22,7 @@ fi # HDFS filename to get HDFS_FILE=$1 # Absolute directory to write output to -OUTPUT_DIR=${2:-output} +OUTPUT_DIR=${2:-$DIR/output} OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" # Internal docker directory to write output to HDFS_OUTPUT_DIR=/output diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process index 8041f2446..c9118ffdf 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process @@ -1,5 +1,5 @@ #!/bin/bash -cd "$( dirname "${BASH_SOURCE[0]}" )" +cd "$( dirname "${BASH_SOURCE[0]}" )/.." # log file logfile=hdfs_output.log @@ -13,7 +13,7 @@ lockfile=.LOCKFILE # involved HDFS directory hdfs_directory=/topicAndroidNew -. ./util.sh +. ./bin/lock-util.sh # extract file from hdfs to backup storage if no other instance is running if [ ! -f $lockfile ]; then diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade index a6a20dde4..10521f634 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade @@ -4,13 +4,13 @@ if [ $# = 0 ]; then echo "Usage: $0 VERSION" exit 1 fi - + VERSION=$1 -STACK_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) -STACK="${STACK_DIR}/bin/radar-docker" +cd "$(dirname "${BASH_SOURCE[0]}")/.." -. "${STACK_DIR}/.env" -. "${STACK_DIR}/util.sh" +stack=bin/radar-stack +. ./.env +. bin/util.sh echo "Are you sure you want to to upgrade HDFS from version ${HDFS_BASE_VERSION} to ${VERSION}?" @@ -21,25 +21,25 @@ select yn in "Yes" "No"; do esac done -"$STACK" quit hdfs-namenode-1 hdfs-namenode-2 radar-hdfs-connector +$stack quit hdfs-namenode-1 hdfs-namenode-2 radar-hdfs-connector sed_i "s/HDFS_BASE_VERSION=.*/HDFS_BASE_VERSION=${VERSION}/" "${STACK_DIR}/.env" -"$STACK" build -"$STACK" up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 +$stack build +$stack up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 sleep 10 -NAME_1=$("$STACK" run --rm -d --name hdfs-namenode-1 hdfs-namenode-1 namenode-1 -upgrade) +NAME_1=$($stack run --rm -d --name hdfs-namenode-1 hdfs-namenode-1 namenode-1 -upgrade) echo "Waiting for the upgrade to finalize..." while ! docker exec $NAME_1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do sleep 1 done -"$STACK" run --no-deps --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -"$STACK" up --no-deps -d hdfs-namenode-2 +$stack run --no-deps --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby +$stack up --no-deps -d hdfs-namenode-2 sleep 10 docker exec $NAME_1 hdfs dfsadmin -finalizeUpgrade docker rm -vf $NAME_1 -"$STACK" hdfs -"$STACK" up -d radar-hdfs-connector \ No newline at end of file +$stack hdfs +$stack up -d radar-hdfs-connector \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/lock-util.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/lock-util.sh new file mode 100644 index 000000000..aff2ccef1 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/lock-util.sh @@ -0,0 +1,41 @@ +PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin + +# maximum file size in byte to rotate log +minimumsize=10000000 + +# current time +timestamp=$(date '+%Y-%m-%d %H:%M:%S'); + +# Write message in the log file +log_info() { + echo "$timestamp - $@" >> $logfile 2>&1 +} + +# Remove old lock +checkLock() { + uptime=$( $logfile + fi +} + +# Entry point +touch $logfile +log_info "### $timestamp ###" +log_info "Checking lock ..." +checkLock diff --git a/dcompose-stack/radar-cp-hadoop-stack/renew_ssl_certificate.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew similarity index 75% rename from dcompose-stack/radar-cp-hadoop-stack/renew_ssl_certificate.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew index fa18ead9a..ef37c4c47 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/renew_ssl_certificate.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew @@ -1,6 +1,8 @@ #!/bin/bash -. ./util.sh +cd "$( dirname "${BASH_SOURCE[0]}" )/.." + +. ./bin/util.sh . ./.env if [ -z ${SERVER_NAME} ]; then diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index 8ef995eee..103188a11 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -1,30 +1,39 @@ #!/bin/bash -stack=(docker-compose -f docker-compose.yml -f docker-compose.hdfs.yml) +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +stack="docker-compose -f docker-compose.yml -f docker-compose.hdfs.yml" if [ $# != 0 ]; then CMD=$1 shift case "${CMD}" in + "install") + . lib/perform-install.sh + ;; "rebuild") - ${stack[@]} build "$@" && \ - ${stack[@]} stop "$@" && \ - ${stack[@]} rm -vf "$@" && \ - ${stack[@]} up -d "$@" && \ + $stack build "$@" && \ + $stack stop "$@" && \ + $stack rm -vf "$@" && \ + $stack up -d "$@" && \ exit 0 || exit 1 ;; "quit") - ${stack[@]} stop "$@" && \ - ${stack[@]} rm -vf "$@" && \ + $stack stop "$@" && \ + $stack rm -vf "$@" && \ exit 0 || exit 1 ;; "hdfs") - exec ${stack[@]} up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 + exec $stack up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 + ;; + "stack") + . lib/perform-install.sh + exec $stack up ;; *) - exec ${stack[@]} "$CMD" "$@" + exec $stack "$CMD" "$@" ;; esac else - exec ${stack[@]} + exec $stack fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/check-health.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-health similarity index 87% rename from dcompose-stack/radar-cp-hadoop-stack/check-health.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/radar-health index fb37b632b..9b3b7d227 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/check-health.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-health @@ -1,9 +1,11 @@ #!/bin/bash # Check whether services are healthy. If not, restart them and notify the maintainer. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -. "$DIR/util.sh" -. .env +cd "$( dirname "${BASH_SOURCE[0]}" )/.." + +stack=bin/radar-docker +. bin/util.sh +. ./.env function hipchat_notify() { # Send notification via HipChat, if configured. @@ -32,7 +34,7 @@ unhealthy=() # see last line of loop while read service; do # check if a container was started for the service - container=$(sudo-linux docker-compose ps -q $service) + container=$(sudo-linux $stack ps -q $service) if [ -z "${container}" ]; then # no container means no running service continue @@ -41,9 +43,9 @@ while read service; do if [ "$health" = "unhealthy" ]; then echo "Service $service is unhealthy. Restarting." unhealthy+=("${service}") - sudo-linux docker-compose restart ${service} + sudo-linux $stack restart ${service} fi -done <<< "$(sudo-linux docker-compose config --services)" +done <<< "$(sudo-linux $stack config --services)" display_host="${SERVER_NAME} ($(hostname -f), $(curl -s http://ipecho.net/plain))" @@ -58,11 +60,11 @@ else # Send notification to MAINTAINER # start up the mail container if not already started - sudo-linux docker-compose up -d smtp + sudo-linux $stack up -d smtp # ensure that all topics are available - sudo-linux docker-compose run --rm kafka-init + sudo-linux $stack run --rm kafka-init # save the container, so that we can use exec to send an email later - container=$(sudo-linux docker-compose ps -q smtp) + container=$(sudo-linux $stack ps -q smtp) SAVEIFS=$IFS IFS=, display_services="[${unhealthy[*]}]" diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer similarity index 88% rename from dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer index 515dfa9a5..51b0a6f17 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-kacc.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer @@ -1,7 +1,10 @@ #!/bin/bash -# kafka-avro-console-consumer inside dockerized radar platform +cd "$( dirname "${BASH_SOURCE[0]}" )/.." + +docker-compose up -d kafka-1 schema-registry-1 +# kafka-avro-console-consumer inside dockerized radar platform if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" || $1 == "--"* ]]; then printf "Usage: $0 [passthrough options]\n" printf "\nkafka-avro-console-consumer passthrough options:\n" @@ -14,14 +17,9 @@ fi args=("$@") # get list of available topics -LS_CMD="docker-compose exec kafka-1 kafka-topics --zookeeper zookeeper-1:2181 --list" -topics=($($LS_CMD)) +topics=($(docker-compose exec kafka-1 kafka-topics --zookeeper zookeeper-1:2181 --list)) #printf "%s\n" "${topics[@]}" -# consumer command to run -KACC_CMD="kafka-avro-console-consumer --broker-list kafka-1:9092,kafka-2:9092,kafka-3:9092 --property schema.registry.url=http://schema-registry-1:8081 --property print.key=true --topic ${args[0]} ${args[@]:1}" -DOCKER_CMD="docker-compose exec schema-registry-1" - # check if is valid topic array_contains () { local array="$1[@]" @@ -37,12 +35,17 @@ array_contains () { done return $in } + if ! array_contains topics ${args[0]}; then echo -e "Topic ${args[0]} not available. Topics on server are:\n" printf "%s\n" "${topics[@]}" exit 1 fi +# consumer command to run +KACC_CMD="kafka-avro-console-consumer --broker-list kafka-1:9092,kafka-2:9092,kafka-3:9092 --property schema.registry.url=http://schema-registry-1:8081 --property print.key=true --topic ${args[0]} ${args[@]:1}" +DOCKER_CMD="docker-compose exec schema-registry-1" + # run consumer echo $DOCKER_CMD $KACC_CMD exec $DOCKER_CMD $KACC_CMD diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers similarity index 69% rename from dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh rename to dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers index 4dee0f8a6..f8cd02ded 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers @@ -1,6 +1,8 @@ #!/bin/bash -. ./util.sh +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +. bin/util.sh check_command_exists systemctl @@ -12,19 +14,17 @@ copy_template_if_absent /etc/systemd/system/radar-check-health.timer lib/systemd copy_template_if_absent /etc/systemd/system/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template copy_template_if_absent /etc/systemd/system/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-docker.service -inline_variable 'ExecStart=' "$DIR/lib/systemd/start-radar-stack.sh" /etc/systemd/system/radar-docker.service +inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-docker.service +inline_variable 'ExecStart=' "$PWD/bin/radar-docker stack" /etc/systemd/system/radar-docker.service -inline_variable 'WorkingDirectory=' "$DIR/hdfs" /etc/systemd/system/radar-output.service -inline_variable 'ExecStart=' "$DIR/bin/hdfs-restructure-process" /etc/systemd/system/radar-output.service +inline_variable 'WorkingDirectory=' "$PWD/hdfs" /etc/systemd/system/radar-output.service +inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure-process" /etc/systemd/system/radar-output.service -inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-check-health.service -inline_variable 'ExecStart=' "$DIR/check-health.sh" /etc/systemd/system/radar-check-health.service +inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-check-health.service +inline_variable 'ExecStart=' "$PWD/bin/radar-health" /etc/systemd/system/radar-check-health.service inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-renew-certificate.service -inline_variable 'ExecStart=' "$DIR/renew_ssl_certificate.sh" /etc/systemd/system/radar-renew-certificate.service +inline_variable 'ExecStart=' "$PWD/bin/radar-cert-renew" /etc/systemd/system/radar-renew-certificate.service sudo systemctl daemon-reload diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/util.sh b/dcompose-stack/radar-cp-hadoop-stack/bin/util.sh old mode 100644 new mode 100755 index e62a5447d..4ff9ddd7e --- a/dcompose-stack/radar-cp-hadoop-stack/bin/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/util.sh @@ -1,43 +1,186 @@ #!/bin/bash -PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin +# this will trap any errors or commands with non-zero exit status +# by calling function catch_errors() +trap catch_errors ERR; -# maximum file size in byte to rotate log -minimumsize=10000000 +function catch_errors() { + exit_code=$? + echo "### FAILURE ###"; + exit $exit_code; +} + +# Check whether given command exists and call it with the --version flag. +check_command_exists() { + if sudo-linux /bin/bash -c "command -v "$1" > /dev/null 2>&1"; then + echo "$1 version: $(sudo-linux "$1" --version)" + else + echo "RADAR Platform cannot start without $1. Please, install $1 and then try again" + exit 1 + fi +} + +# Check if the parent directory of given variable is set. Usage: +# check_parent_exists MY_PATH_VAR $MY_PATH_VAR +check_parent_exists() { + if [ -z "$2" ]; then + echo "Directory variable $1 is not set in .env" + fi + PARENT=$(dirname $2) + if [ ! -d "${PARENT}" ]; then + echo "RADAR-base stores volumes at ${PARENT}. If this folder does not exist, please create the entire path and then try again" + exit 1 + fi + if [ -d "$2" ]; then + sudo-linux chmod 700 "$2" + else + sudo-linux mkdir -p -m 0700 "$2" + fi +} + +# sudo if on Linux, not on OS X +# useful for docker, which doesn't need sudo on OS X +sudo-linux() { + if [ $(uname) == "Darwin" ]; then + "$@" + else + sudo "$@" + fi +} -# current time -timestamp=$(date '+%Y-%m-%d %H:%M:%S'); +# OS X/linux portable sed -i +sed_i() { + if [ $(uname) == "Darwin" ]; then + sed -i '' "$@" + else + sudo sed -i -- "$@" + fi +} -# Write message in the log file -log_info() { - echo "$timestamp - $@" >> $logfile 2>&1 +# Inline variable into a file, keeping indentation. +# Usage: +# inline_variable VARIABLE_SET VALUE FILE +# where VARIABLE_SET is a regex of the pattern currently used in given file to set a variable to a value. +# Example: +# inline_variable 'a=' 123 test.txt +# will replace a line ' a=232 ' with ' a=123' +inline_variable() { + sed_i 's|^\([[:space:]]*'"$1"'\).*$|\1'"$2"'|' "$3" } -# Remove old lock -checkLock() { - uptime=$( $logfile +self_signed_certificate() { + SERVER_NAME=$1 + echo "==> Generating self-signed certificate" + sudo-linux docker run -i --rm -v certs:/etc/openssl -v certs-data:/var/lib/openssl -v "${PWD}/lib/self-sign-certificate.sh:/self-sign-certificate.sh" alpine:3.5 \ + /self-sign-certificate.sh "/etc/openssl/live/${SERVER_NAME}" +} + +letsencrypt_certonly() { + SERVER_NAME=$1 + SSL_PATH="/etc/openssl/live/${SERVER_NAME}" + echo "==> Requesting Let's Encrypt SSL certificate for ${SERVER_NAME}" + + # start from a clean slate + sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "find /etc/openssl -name '${SERVER_NAME}*' -prune -exec rm -rf '{}' +" + + CERTBOT_DOCKER_OPTS=(-i --rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) + CERTBOT_OPTS=(--webroot --webroot-path=/data/letsencrypt --agree-tos -m "${MAINTAINER_EMAIL}" -d "${SERVER_NAME}" --non-interactive) + sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" + + # mark the directory as letsencrypt dir + sudo-linux docker run -i --rm -v certs:/etc/openssl alpine:3.5 /bin/touch "${SSL_PATH}/.letsencrypt" +} + +letsencrypt_renew() { + SERVER_NAME=$1 + echo "==> Renewing Let's Encrypt SSL certificate for ${SERVER_NAME}" + CERTBOT_DOCKER_OPTS=(-i --rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) + CERTBOT_OPTS=(-n --webroot --webroot-path=/data/letsencrypt -d "${SERVER_NAME}" --non-interactive) + sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" +} + +init_certificate() { + SERVER_NAME=$1 + SSL_PATH="/etc/openssl/live/${SERVER_NAME}" + if sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "[ ! -e '${SSL_PATH}/chain.pem' ]"; then + self_signed_certificate "${SERVER_NAME}" + fi +} + +request_certificate() { + SERVER_NAME=$1 + SELF_SIGNED=$2 + SSL_PATH="/etc/openssl/live/${SERVER_NAME}" + + init_certificate "${SERVER_NAME}" + CURRENT_CERT=$(sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "[ -e '${SSL_PATH}/.letsencrypt' ] && echo letsencrypt || echo self-signed") + + if [ "${CURRENT_CERT}" = "letsencrypt" ]; then + if [ "$3" != "force" ]; then + echo "Let's Encrypt SSL certificate already exists, not renewing" + return + fi + + if [ "${SELF_SIGNED}" = "yes" ]; then + echo "Converting Let's Encrypt SSL certificate to a self-signed SSL" + self_signed_certificate "${SERVER_NAME}" + else + letsencrypt_renew "${SERVER_NAME}" + fi + else + if [ "${SELF_SIGNED}" = "yes" ]; then + if [ "$3" = "force" ]; then + echo "WARN: Self-signed SSL certificate already existed, recreating" + self_signed_certificate "${SERVER_NAME}" + else + echo "Self-signed SSL certificate exists, not recreating" + return + fi + else + letsencrypt_certonly "${SERVER_NAME}" + fi fi + echo "Reloading webserver configuration" + sudo-linux docker-compose kill -s HUP webserver 1>/dev/null 2>&1 } -# Entry point -touch $logfile -log_info "### $timestamp ###" -log_info "Checking lock ..." -checkLock +echo "OS version: $(uname -a)" +check_command_exists docker +check_command_exists docker-compose diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml index bf924e8cc..43ed93663 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml @@ -11,7 +11,7 @@ services: #---------------------------------------------------------------------------# hdfs-datanode-1: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -32,7 +32,7 @@ services: SERVICE_9866_NAME: datanode SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -46,7 +46,7 @@ services: hdfs-datanode-2: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -67,7 +67,7 @@ services: SERVICE_9866_NAME: datanode SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -78,10 +78,10 @@ services: interval: 1m timeout: 10s retries: 3 - + hdfs-datanode-3: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -102,7 +102,7 @@ services: SERVICE_9866_NAME: datanode SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -116,7 +116,7 @@ services: hdfs-namenode-1: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -137,7 +137,7 @@ services: environment: SERVICE_8020_NAME: namenode SERVICE_9870_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -148,10 +148,10 @@ services: interval: 1m timeout: 10s retries: 3 - + hdfs-namenode-2: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -173,7 +173,7 @@ services: environment: SERVICE_8020_NAME: namenode SERVICE_9870_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -188,7 +188,7 @@ services: hdfs-journalnode-1: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -202,7 +202,7 @@ services: environment: SERVICE_8485_NAME: journalnode SERVICE_8480_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: @@ -210,10 +210,10 @@ services: interval: 1m timeout: 10s retries: 3 - + hdfs-journalnode-2: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -227,7 +227,7 @@ services: environment: SERVICE_8485_NAME: journalnode SERVICE_8480_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: @@ -238,7 +238,7 @@ services: hdfs-journalnode-3: build: - context: ./hdfs/images/hdfs + context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} @@ -252,7 +252,7 @@ services: environment: SERVICE_8485_NAME: journalnode SERVICE_8480_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 23c371c9b..bcffbecdb 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -254,8 +254,8 @@ services: # Kafka Init # #---------------------------------------------------------------------------# kafka-init: - build: kafka-radarinit - image: radarcns/kafka-init + build: images/radar-kafka-init + image: radarbase/kafka-init networks: - kafka - zookeeper @@ -635,8 +635,8 @@ services: # Catalog server from radar-schemas # #---------------------------------------------------------------------------# catalog-server: - build: images/kafka-radarinit - image: radarcns/kafka-init + build: images/radar-kafka-init + image: radarbase/kafka-init networks: - management command: radar-schemas-tools serve /schema/merged diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh index d66a7185e..60a7d079f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh @@ -7,23 +7,23 @@ wait_until() { local port=${2?} local retry=${3:-100} local sleep_secs=${4:-2} - + local address_up=0 - + while [ ${retry} -gt 0 ] ; do echo "Waiting until ${hostname}:${port} is up ... with retry count: ${retry}" if nc -z ${hostname} ${port}; then address_up=1 break - fi + fi retry=$((retry-1)) sleep ${sleep_secs} - done - + done + if [ $address_up -eq 0 ]; then echo "GIVE UP waiting until ${hostname}:${port} is up! " exit 1 - fi + fi } # apply template @@ -53,23 +53,23 @@ case $CMD in if [ "${HADOOP_NAMENODE_HA}" != "" ]; then su-exec hdfs hdfs zkfc -formatZK -force fi - fi + fi # wait_until ${HADOOP_QJOURNAL_ADDRESS%%:*} 8485 if [ "${HADOOP_NAMENODE_HA}" != "" ]; then - su-exec hdfs hdfs zkfc & - fi + su-exec hdfs hdfs zkfc & + fi exec su-exec hdfs hdfs namenode "$@" ;; "namenode-2") if [ ! -e "${HADOOP_TMP_DIR}/dfs/name/current/VERSION" ]; then - wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 + wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 su-exec hdfs hdfs namenode -bootstrapStandby fi su-exec hdfs hdfs zkfc & exec su-exec hdfs hdfs namenode "$@" ;; "datanode") - wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 + wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 exec su-exec hdfs hdfs datanode "$@" ;; "resourcemanager-1") @@ -80,38 +80,38 @@ case $CMD in exec su-exec yarn yarn nodemanager "$@" ;; "historyserver-1") - wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 - + wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 + set +e -x - + su-exec hdfs hdfs dfs -ls /tmp > /dev/null 2>&1 if [ $? -ne 0 ]; then su-exec hdfs hdfs dfs -mkdir -p /tmp su-exec hdfs hdfs dfs -chmod 1777 /tmp fi - + su-exec hdfs hdfs dfs -ls /user > /dev/null 2>&1 if [ $? -ne 0 ]; then su-exec hdfs hdfs dfs -mkdir -p /user/hdfs su-exec hdfs hdfs dfs -chmod 755 /user fi - + su-exec hdfs hdfs dfs -ls ${YARN_REMOTE_APP_LOG_DIR} > /dev/null 2>&1 if [ $? -ne 0 ]; then su-exec yarn hdfs dfs -mkdir -p ${YARN_REMOTE_APP_LOG_DIR} su-exec yarn hdfs dfs -chmod -R 1777 ${YARN_REMOTE_APP_LOG_DIR} su-exec yarn hdfs dfs -chown -R yarn:hadoop ${YARN_REMOTE_APP_LOG_DIR} fi - + su-exec hdfs hdfs dfs -ls ${YARN_APP_MAPRED_STAGING_DIR} > /dev/null 2>&1 if [ $? -ne 0 ]; then su-exec mapred hdfs dfs -mkdir -p ${YARN_APP_MAPRED_STAGING_DIR} su-exec mapred hdfs dfs -chmod -R 1777 ${YARN_APP_MAPRED_STAGING_DIR} su-exec mapred hdfs dfs -chown -R mapred:hadoop ${YARN_APP_MAPRED_STAGING_DIR} fi - - set -e +x - + + set -e +x + exec su-exec mapred mapred historyserver "$@" ;; *) diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache index 7c1804fc2..a6428defd 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache @@ -22,7 +22,7 @@ dfs.permissions.superusergroup hadoop - + dfs.nameservices {{HADOOP_CLUSTER_NAME}} @@ -37,8 +37,8 @@ dfs.ha.namenodes.{{HADOOP_CLUSTER_NAME}} {{HADOOP_NAMENODE_HA}} - - + + dfs.namenode.rpc-address.{{HADOOP_CLUSTER_NAME}}.nn1 {{HADOOP_NAMENODE1_HOSTNAME}}:8020 @@ -57,7 +57,7 @@ dfs.namenode.http-address.{{HADOOP_CLUSTER_NAME}}.nn2 {{HADOOP_NAMENODE2_HOSTNAME}}:9870 - + dfs.namenode.shared.edits.dir @@ -68,7 +68,7 @@ dfs.client.failover.proxy.provider.{{HADOOP_CLUSTER_NAME}} org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - + dfs.ha.fencing.methods shell(/usr/local/bin/fence.sh) @@ -91,7 +91,7 @@ dfs.namenode.rpc-address.{{HADOOP_CLUSTER_NAME}} {{HADOOP_NAMENODE1_HOSTNAME}}:8020 - + dfs.namenode.http-address.{{HADOOP_CLUSTER_NAME}} {{HADOOP_NAMENODE1_HOSTNAME}}:9870 diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/Dockerfile rename to dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/init.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/init.sh rename to dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/init.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_aggregated.sh b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/list_aggregated.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_aggregated.sh rename to dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/list_aggregated.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_raw.sh b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/list_raw.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/list_raw.sh rename to dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/list_raw.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/topic_init.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/kafka-radarinit/topic_init.sh rename to dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/topic_init.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh new file mode 100755 index 000000000..3aa7743f0 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +. bin/util.sh + +sudo-linux chmod og-rw ./.env +sudo-linux chmod og-rwx ./etc +if [ -e ./output ]; then + sudo-linux chmod og-rwx ./output +else + sudo-linux mkdir -m 0700 ./output +fi + +# Initialize and check all config files +check_config_present .env etc/env.template +check_config_present etc/radar-backend/radar.yml +check_config_present etc/managementportal/config/oauth_client_details.csv +check_config_present etc/redcap-integration/radar.yml +copy_template_if_absent etc/mongodb-connector/sink-mongo.properties +copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties +copy_template_if_absent etc/rest-api/radar.yml +copy_template_if_absent etc/webserver/nginx.conf + +. ./.env + +# Check provided directories and configurations +check_parent_exists HDFS_DATA_DIR_1 ${HDFS_DATA_DIR_1} +check_parent_exists HDFS_DATA_DIR_2 ${HDFS_DATA_DIR_2} +check_parent_exists HDFS_NAME_DIR_1 ${HDFS_NAME_DIR_1} +check_parent_exists HDFS_NAME_DIR_2 ${HDFS_NAME_DIR_2} +check_parent_exists MONGODB_DIR ${MONGODB_DIR} +check_parent_exists MP_POSTGRES_DIR ${MP_POSTGRES_DIR} + +if [ -z ${SERVER_NAME} ]; then + echo "Set SERVER_NAME variable in .env" + exit 1 +fi + +# Create networks and volumes +if ! sudo-linux docker network ls --format '{{.Name}}' | grep -q "^hadoop$"; then + echo "==> Creating docker network - hadoop" + sudo-linux docker network create hadoop > /dev/null +else + echo "==> Creating docker network - hadoop ALREADY EXISTS" +fi + +echo "==> Checking docker external volumes" +if ! sudo-linux docker volume ls -q | grep -q "^certs$"; then + sudo-linux docker volume create --name=certs --label certs +fi +if ! sudo-linux docker volume ls -q | grep -q "^certs-data$"; then + sudo-linux docker volume create --name=certs-data --label certs +fi + +# Initializing Kafka +echo "==> Setting up topics" +sudo-linux bin/radar-docker run --rm kafka-init + +echo "==> Configuring MongoDB Connector" +# Update sink-mongo.properties +inline_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sink-mongo.properties +inline_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties +inline_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties + +# Set topics +if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then + COMBINED_AGG_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_aggregated.sh 2>/dev/null | tail -n 1) + if [ -n "${RADAR_AGG_TOPIC_LIST}" ]; then + COMBINED_AGG_TOPIC_LIST="${RADAR_AGG_TOPIC_LIST},${COMBINED_AGG_TOPIC_LIST}" + fi +fi +inline_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/mongodb-connector/sink-mongo.properties + +echo "==> Configuring HDFS Connector" +if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then + COMBINED_RAW_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_raw.sh 2>/dev/null | tail -n 1) + if [ -n "${RADAR_RAW_TOPIC_LIST}" ]; then + COMBINED_RAW_TOPIC_LIST="${RADAR_RAW_TOPIC_LIST},${COMBINED_RAW_TOPIC_LIST}" + fi +fi +inline_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties + +echo "==> Configuring Management Portal" + + +keystorefile=etc/managementportal/config/keystore.jks +if [ -f "$keystorefile" ]; then + echo "--> Keystore for signing JWTs already exists. Not creating a new one." +else + echo "--> Generating keystore to hold RSA keypair for JWT signing" + if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then + sudo-linux keytool -genkeypair -dname "${MANAGEMENTPORTAL_KEY_DNAME}" -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase + else + sudo-linux keytool -genkeypair -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase + fi + sudo-linux chmod 400 "${keystorefile}" +fi + +echo "==> Configuring REST-API" + +# Set MongoDb credential +inline_variable 'username:[[:space:]]' "$HOTSTORAGE_USERNAME" etc/rest-api/radar.yml +inline_variable 'password:[[:space:]]' "$HOTSTORAGE_PASSWORD" etc/rest-api/radar.yml +inline_variable 'database_name:[[:space:]]' "$HOTSTORAGE_NAME" etc/rest-api/radar.yml + +echo "==> Configuring REDCap-Integration" + +echo "==> Configuring nginx" +inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx.conf +sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf +init_certificate "${SERVER_NAME}" + +echo "==> Starting RADAR-base Platform" +sudo-linux bin/radar-docker up -d "$@" + +request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" +echo "### SUCCESS ###" diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/start-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/start-radar-stack.sh deleted file mode 100755 index 2063ac666..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/start-radar-stack.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$DIR/../.." -./install-radar-stack.sh -exec docker-compose up \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh deleted file mode 100755 index 3b0e40e82..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/reboot-radar-stack.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. ./util.sh - -echo "==> Restarting RADAR-base Platform" -sudo-linux docker-compose restart diff --git a/dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh deleted file mode 100755 index b44163f9d..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/start-radar-stack.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. ./util.sh - -echo "==> Starting radar-base platform" -sudo-linux docker-compose start diff --git a/dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh deleted file mode 100755 index 8006f3fa0..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/stop-radar-stack.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. ./util.sh - -echo "==> Stopping RADAR-base Stack" -sudo-linux docker-compose stop diff --git a/dcompose-stack/radar-cp-hadoop-stack/util.sh b/dcompose-stack/radar-cp-hadoop-stack/util.sh deleted file mode 100755 index 0a534d696..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/util.sh +++ /dev/null @@ -1,184 +0,0 @@ -# this will trap any errors or commands with non-zero exit status -# by calling function catch_errors() -trap catch_errors ERR; - -function catch_errors() { - exit_code=$? - echo "### FAILURE ###"; - exit $exit_code; -} - -# Check whether given command exists and call it with the --version flag. -check_command_exists() { - if sudo-linux /bin/bash -c "command -v "$1" > /dev/null 2>&1"; then - echo "$1 version: $(sudo-linux "$1" --version)" - else - echo "RADAR Platform cannot start without $1. Please, install $1 and then try again" - exit 1 - fi -} - -# Check if the parent directory of given variable is set. Usage: -# check_parent_exists MY_PATH_VAR $MY_PATH_VAR -check_parent_exists() { - if [ -z "$2" ]; then - echo "Directory variable $1 is not set in .env" - fi - PARENT=$(dirname $2) - if [ ! -d "${PARENT}" ]; then - echo "RADAR-base stores volumes at ${PARENT}. If this folder does not exist, please create the entire path and then try again" - exit 1 - fi - if [ -d "$2" ]; then - sudo-linux chmod 700 "$2" - else - sudo-linux mkdir -p -m 0700 "$2" - fi -} - -# sudo if on Linux, not on OS X -# useful for docker, which doesn't need sudo on OS X -sudo-linux() { - if [ $(uname) == "Darwin" ]; then - "$@" - else - sudo "$@" - fi -} - -# OS X/linux portable sed -i -sed_i() { - if [ $(uname) == "Darwin" ]; then - sed -i '' "$@" - else - sudo sed -i -- "$@" - fi -} - -# Inline variable into a file, keeping indentation. -# Usage: -# inline_variable VARIABLE_SET VALUE FILE -# where VARIABLE_SET is a regex of the pattern currently used in given file to set a variable to a value. -# Example: -# inline_variable 'a=' 123 test.txt -# will replace a line ' a=232 ' with ' a=123' -inline_variable() { - sed_i 's|^\([[:space:]]*'"$1"'\).*$|\1'"$2"'|' "$3" -} - -# Copies the template (defined by the given config file with suffix -# ".template") to intended configuration file, if the file does not -# yet exist. -copy_template_if_absent() { - template=${2:-${1}.template} - if [ ! -f "$1" ]; then - if [ -e "$1" ]; then - echo "Configuration file ${1} is not a regular file." - exit 1 - else - sudo-linux cp -p "${template}" "$1" - fi - elif [ "$1" -ot "${template}" ]; then - echo "Configuration file ${1} is older than its template" - echo "${template}. Please edit ${1}" - echo "to ensure it matches the template, remove it or run touch on it." - exit 1 - fi -} - -check_config_present() { - template=${2:-${1}.template} - if [ ! -f "$1" ]; then - if [ -e "$1" ]; then - echo "Configuration file ${1} is not a regular file." - else - echo "Configuration file ${1} is not present." - echo "Please copy it from ${template} and modify it as needed." - fi - exit 1 - elif [ "$1" -ot "${template}" ]; then - echo "Configuration file ${1} is older than its template ${template}." - echo "Please edit ${1} to ensure it matches the template or run touch on it." - exit 1 - fi -} - -self_signed_certificate() { - SERVER_NAME=$1 - echo "==> Generating self-signed certificate" - sudo-linux docker run -i --rm -v certs:/etc/openssl -v certs-data:/var/lib/openssl -v "${PWD}/lib/self-sign-certificate.sh:/self-sign-certificate.sh" alpine:3.5 \ - /self-sign-certificate.sh "/etc/openssl/live/${SERVER_NAME}" -} - -letsencrypt_certonly() { - SERVER_NAME=$1 - SSL_PATH="/etc/openssl/live/${SERVER_NAME}" - echo "==> Requesting Let's Encrypt SSL certificate for ${SERVER_NAME}" - - # start from a clean slate - sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "find /etc/openssl -name '${SERVER_NAME}*' -prune -exec rm -rf '{}' +" - - CERTBOT_DOCKER_OPTS=(-i --rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) - CERTBOT_OPTS=(--webroot --webroot-path=/data/letsencrypt --agree-tos -m "${MAINTAINER_EMAIL}" -d "${SERVER_NAME}" --non-interactive) - sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" - - # mark the directory as letsencrypt dir - sudo-linux docker run -i --rm -v certs:/etc/openssl alpine:3.5 /bin/touch "${SSL_PATH}/.letsencrypt" -} - -letsencrypt_renew() { - SERVER_NAME=$1 - echo "==> Renewing Let's Encrypt SSL certificate for ${SERVER_NAME}" - CERTBOT_DOCKER_OPTS=(-i --rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) - CERTBOT_OPTS=(-n --webroot --webroot-path=/data/letsencrypt -d "${SERVER_NAME}" --non-interactive) - sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" -} - -init_certificate() { - SERVER_NAME=$1 - SSL_PATH="/etc/openssl/live/${SERVER_NAME}" - if sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "[ ! -e '${SSL_PATH}/chain.pem' ]"; then - self_signed_certificate "${SERVER_NAME}" - fi -} - -request_certificate() { - SERVER_NAME=$1 - SELF_SIGNED=$2 - SSL_PATH="/etc/openssl/live/${SERVER_NAME}" - - init_certificate "${SERVER_NAME}" - CURRENT_CERT=$(sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "[ -e '${SSL_PATH}/.letsencrypt' ] && echo letsencrypt || echo self-signed") - - if [ "${CURRENT_CERT}" = "letsencrypt" ]; then - if [ "$3" != "force" ]; then - echo "Let's Encrypt SSL certificate already exists, not renewing" - return - fi - - if [ "${SELF_SIGNED}" = "yes" ]; then - echo "Converting Let's Encrypt SSL certificate to a self-signed SSL" - self_signed_certificate "${SERVER_NAME}" - else - letsencrypt_renew "${SERVER_NAME}" - fi - else - if [ "${SELF_SIGNED}" = "yes" ]; then - if [ "$3" = "force" ]; then - echo "WARN: Self-signed SSL certificate already existed, recreating" - self_signed_certificate "${SERVER_NAME}" - else - echo "Self-signed SSL certificate exists, not recreating" - return - fi - else - letsencrypt_certonly "${SERVER_NAME}" - fi - fi - echo "Reloading webserver configuration" - sudo-linux docker-compose kill -s HUP webserver 1>/dev/null 2>&1 -} - -echo "OS version: $(uname -a)" -check_command_exists docker -check_command_exists docker-compose From 1daf93ddc47384213fd618c36c3a2678024eecd6 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:17:25 +0200 Subject: [PATCH 030/194] Remove duplicate script --- .../install-radar-stack.sh | 116 ------------------ 1 file changed, 116 deletions(-) delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh deleted file mode 100755 index b64450ecc..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -. ./util.sh - -sudo-linux chmod og-rw ./.env -sudo-linux chmod og-rwx ./etc -if [ -e ./output ]; then - sudo-linux chmod og-rwx ./output -else - sudo-linux mkdir -m 0700 ./output -fi - -# Initialize and check all config files -check_config_present .env etc/env.template -check_config_present etc/radar-backend/radar.yml -check_config_present etc/managementportal/config/oauth_client_details.csv -check_config_present etc/redcap-integration/radar.yml -copy_template_if_absent etc/mongodb-connector/sink-mongo.properties -copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties -copy_template_if_absent etc/rest-api/radar.yml -copy_template_if_absent etc/webserver/nginx.conf - -. ./.env - -# Check provided directories and configurations -check_parent_exists HDFS_DATA_DIR_1 ${HDFS_DATA_DIR_1} -check_parent_exists HDFS_DATA_DIR_2 ${HDFS_DATA_DIR_2} -check_parent_exists HDFS_NAME_DIR_1 ${HDFS_NAME_DIR_1} -check_parent_exists HDFS_NAME_DIR_2 ${HDFS_NAME_DIR_2} -check_parent_exists MONGODB_DIR ${MONGODB_DIR} -check_parent_exists MP_POSTGRES_DIR ${MP_POSTGRES_DIR} - -if [ -z ${SERVER_NAME} ]; then - echo "Set SERVER_NAME variable in .env" - exit 1 -fi - -# Create networks and volumes -if ! sudo-linux docker network ls --format '{{.Name}}' | grep -q "^hadoop$"; then - echo "==> Creating docker network - hadoop" - sudo-linux docker network create hadoop > /dev/null -else - echo "==> Creating docker network - hadoop ALREADY EXISTS" -fi - -echo "==> Checking docker external volumes" -if ! sudo-linux docker volume ls -q | grep -q "^certs$"; then - sudo-linux docker volume create --name=certs --label certs -fi -if ! sudo-linux docker volume ls -q | grep -q "^certs-data$"; then - sudo-linux docker volume create --name=certs-data --label certs -fi - -# Initializing Kafka -echo "==> Setting up topics" -sudo-linux docker-compose run --rm kafka-init - -echo "==> Configuring MongoDB Connector" -# Update sink-mongo.properties -inline_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sink-mongo.properties -inline_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties -inline_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties - -# Set topics -if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then - COMBINED_AGG_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_aggregated.sh 2>/dev/null | tail -n 1) - if [ -n "${RADAR_AGG_TOPIC_LIST}" ]; then - COMBINED_AGG_TOPIC_LIST="${RADAR_AGG_TOPIC_LIST},${COMBINED_AGG_TOPIC_LIST}" - fi -fi -inline_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/mongodb-connector/sink-mongo.properties - -echo "==> Configuring HDFS Connector" -if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then - COMBINED_RAW_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_raw.sh 2>/dev/null | tail -n 1) - if [ -n "${RADAR_RAW_TOPIC_LIST}" ]; then - COMBINED_RAW_TOPIC_LIST="${RADAR_RAW_TOPIC_LIST},${COMBINED_RAW_TOPIC_LIST}" - fi -fi -inline_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties - -echo "==> Configuring Management Portal" - - -keystorefile=etc/managementportal/config/keystore.jks -if [ -f "$keystorefile" ]; then - echo "--> Keystore for signing JWTs already exists. Not creating a new one." -else - echo "--> Generating keystore to hold RSA keypair for JWT signing" - if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then - sudo-linux keytool -genkeypair -dname "${MANAGEMENTPORTAL_KEY_DNAME}" -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase - else - sudo-linux keytool -genkeypair -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase - fi - sudo-linux chmod 400 "${keystorefile}" -fi - -echo "==> Configuring REST-API" - -# Set MongoDb credential -inline_variable 'username:[[:space:]]' "$HOTSTORAGE_USERNAME" etc/rest-api/radar.yml -inline_variable 'password:[[:space:]]' "$HOTSTORAGE_PASSWORD" etc/rest-api/radar.yml -inline_variable 'database_name:[[:space:]]' "$HOTSTORAGE_NAME" etc/rest-api/radar.yml - -echo "==> Configuring REDCap-Integration" - -echo "==> Configuring nginx" -inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx.conf -sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf -init_certificate "${SERVER_NAME}" - -echo "==> Starting RADAR-base Platform" -sudo-linux docker-compose up -d "$@" - -request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" -echo "### SUCCESS ###" From 03125d747ea490cd974deda29cff6ae7ff90e3b9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:21:10 +0200 Subject: [PATCH 031/194] Readme comments --- dcompose-stack/radar-cp-hadoop-stack/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index d4790a4e1..cc59d0193 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -48,7 +48,7 @@ Run ```shell bin/radar-docker install ``` -to start all the RADAR services. Use the `bin/radar-docker start|down|restart` to start, stop or reboot it. Note: whenever `.env` or `docker-compose.yml` are modified, this script needs to be called again. To start a reduced set of containers, call `bin/radar-docker install` with the intended containers as arguments. +to start all the RADAR services. Use the `bin/radar-docker start|down|restart` to start, stop or reboot it. In general, `bin/radar-docker` is a convenience script to `docker-compose`, so all commands that work on docker-compose also work on `bin/radar-docker`. Note: whenever `.env` or `docker-compose.yml` are modified, the `install` command needs to be called again. To start a reduced set of containers, call `bin/radar-docker install` with the intended containers as arguments. To enable a `systemd` service to control the platform, run ```shell @@ -86,6 +86,8 @@ bin/docker-prune sudo systemctl start radar-docker ``` +To rebuild an image and restart them, run `bin/radar-docker rebuild IMAGE`. To stop and remove an container, run `bin/radar-docker quit CONTAINER`. To start the HDFS cluster, run `bin/radar-docker hdfs`. + ### Monitoring a topic To see current data coming out of a Kafka topic, run From a66629812a26d8a2f7d666f9bf619f0a2a43139e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:22:30 +0200 Subject: [PATCH 032/194] Updated docker-compose version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1b61a01b0..131d8b6e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: required services: - docker env: - DOCKER_COMPOSE_VERSION: 1.11.2 + DOCKER_COMPOSE_VERSION: 1.21.1 before_install: - docker --version From 0dc66db053db0fbfa7f8a4a1ad72dd6a607e6bc3 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:29:25 +0200 Subject: [PATCH 033/194] Healthcheck in radar-docker --- .travis.yml | 1 - dcompose-stack/radar-cp-hadoop-stack/README.md | 2 +- dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker | 3 +++ .../radar-cp-hadoop-stack/bin/radar-systemd-wrappers | 3 +-- .../{bin/radar-health => lib/check-health.sh} | 0 5 files changed, 5 insertions(+), 4 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/{bin/radar-health => lib/check-health.sh} (100%) diff --git a/.travis.yml b/.travis.yml index 131d8b6e6..513ac5186 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,6 @@ script: - cp etc/mongodb-connector/sink-mongo.properties.template etc/mongodb-connector/sink-mongo.properties - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down - - sudo docker network rm hadoop # With Docker Swarm support # NOT SUPPORTED: docker swarm and docker beta features are not available in Travis diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index cc59d0193..a6b29d2c6 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -86,7 +86,7 @@ bin/docker-prune sudo systemctl start radar-docker ``` -To rebuild an image and restart them, run `bin/radar-docker rebuild IMAGE`. To stop and remove an container, run `bin/radar-docker quit CONTAINER`. To start the HDFS cluster, run `bin/radar-docker hdfs`. +To rebuild an image and restart them, run `bin/radar-docker rebuild IMAGE`. To stop and remove an container, run `bin/radar-docker quit CONTAINER`. To start the HDFS cluster, run `bin/radar-docker hdfs`. For a health check, run `bin/radar-docker health`. ### Monitoring a topic diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index 103188a11..a9e26292e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -23,6 +23,9 @@ if [ $# != 0 ]; then $stack rm -vf "$@" && \ exit 0 || exit 1 ;; + "health") + . lib/radar-health.sh + ;; "hdfs") exec $stack up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 ;; diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers index f8cd02ded..c2a44745e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers @@ -21,12 +21,11 @@ inline_variable 'WorkingDirectory=' "$PWD/hdfs" /etc/systemd/system/radar-output inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure-process" /etc/systemd/system/radar-output.service inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-check-health.service -inline_variable 'ExecStart=' "$PWD/bin/radar-health" /etc/systemd/system/radar-check-health.service +inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" /etc/systemd/system/radar-check-health.service inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-renew-certificate.service inline_variable 'ExecStart=' "$PWD/bin/radar-cert-renew" /etc/systemd/system/radar-renew-certificate.service - sudo systemctl daemon-reload sudo systemctl enable radar-docker sudo systemctl enable radar-output.timer diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-health b/dcompose-stack/radar-cp-hadoop-stack/lib/check-health.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/bin/radar-health rename to dcompose-stack/radar-cp-hadoop-stack/lib/check-health.sh From e22b1d074658477aa76e44fdcebc0a46d6a76521 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:32:45 +0200 Subject: [PATCH 034/194] Moved util and lock-util to lib --- dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune | 2 +- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract | 2 +- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure | 2 +- .../radar-cp-hadoop-stack/bin/hdfs-restructure-process | 2 +- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade | 2 +- .../radar-cp-hadoop-stack/hash-backup/initialize-hb.sh | 2 +- dcompose-stack/radar-cp-hadoop-stack/{bin => lib}/lock-util.sh | 0 dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 2 +- dcompose-stack/radar-cp-hadoop-stack/{bin => lib}/util.sh | 0 9 files changed, 7 insertions(+), 7 deletions(-) mode change 100644 => 100755 dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh rename dcompose-stack/radar-cp-hadoop-stack/{bin => lib}/lock-util.sh (100%) rename dcompose-stack/radar-cp-hadoop-stack/{bin => lib}/util.sh (100%) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune index cef2c26d9..942b842bc 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune @@ -2,7 +2,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." -. bin/util.sh +. lib/util.sh . ./.env check_command_exists docker diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract index 519bd4006..5ac15d336 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract @@ -7,7 +7,7 @@ if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then exit 1 fi -. "$DIR/bin/util.sh" +. "$DIR/lib/util.sh" . "$DIR/.env" # HDFS filename to get diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 11926c645..e66189b08 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -7,7 +7,7 @@ fi DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) -. "$DIR/bin/util.sh" +. "$DIR/lib/util.sh" # HDFS restructure version JAR_VERSION=0.3.2 diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process index c9118ffdf..76e99171c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process @@ -13,7 +13,7 @@ lockfile=.LOCKFILE # involved HDFS directory hdfs_directory=/topicAndroidNew -. ./bin/lock-util.sh +. ./lib/lock-util.sh # extract file from hdfs to backup storage if no other instance is running if [ ! -f $lockfile ]; then diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade index 10521f634..14a57f67b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade @@ -10,7 +10,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." stack=bin/radar-stack . ./.env -. bin/util.sh +. lib/util.sh echo "Are you sure you want to to upgrade HDFS from version ${HDFS_BASE_VERSION} to ${VERSION}?" diff --git a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh old mode 100644 new mode 100755 index 72dfa2e8d..89da7e23a --- a/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/hash-backup/initialize-hb.sh @@ -1,7 +1,7 @@ #!/bin/bash cd "$( dirname "${BASH_SOURCE[0]}" )" . "./backup.conf" -. "../util.sh" > /dev/null +. "../lib/util.sh" > /dev/null IFS=',' read -r -a inputs <<< "$INPUTS" diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/lock-util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/lock-util.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/bin/lock-util.sh rename to dcompose-stack/radar-cp-hadoop-stack/lib/lock-util.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 3aa7743f0..7fc1553e3 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -2,7 +2,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." -. bin/util.sh +. lib/util.sh sudo-linux chmod og-rw ./.env sudo-linux chmod og-rwx ./etc diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/bin/util.sh rename to dcompose-stack/radar-cp-hadoop-stack/lib/util.sh From fbaac0cd1ca276566b3a735fd50db87ea3105621 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:39:07 +0200 Subject: [PATCH 035/194] Small reference fixes --- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure | 3 +++ dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew | 2 +- .../radar-cp-hadoop-stack/bin/radar-systemd-wrappers | 2 +- dcompose-stack/radar-cp-hadoop-stack/lib/check-health.sh | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 163a60662..bf0f8185f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -9,6 +9,9 @@ DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) . "$DIR/lib/util.sh" +# Start HDFS if not started already +sudo-linux "$DIR/bin/radar-docker" hdfs + # HDFS restructure version DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.0 diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew index ef37c4c47..72fac5939 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew @@ -2,7 +2,7 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/.." -. ./bin/util.sh +. ./lib/util.sh . ./.env if [ -z ${SERVER_NAME} ]; then diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers index c2a44745e..7436c4720 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers @@ -2,7 +2,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." -. bin/util.sh +. lib/util.sh check_command_exists systemctl diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/check-health.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/check-health.sh index 9b3b7d227..6a07c0958 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/check-health.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/check-health.sh @@ -4,7 +4,7 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/.." stack=bin/radar-docker -. bin/util.sh +. lib/util.sh . ./.env function hipchat_notify() { From d74f9c2e61556ab2c3c8407b8a35744fe695e010 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 16 Jul 2018 15:48:24 +0200 Subject: [PATCH 036/194] Small hdfs init fixes --- .../radar-cp-hadoop-stack/bin/hdfs-ha-init | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init index 4c65010d9..b40acb793 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init @@ -13,13 +13,13 @@ select yn in "Yes" "No"; do esac done -bin/radar-stack up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 -bin/radar-stack quit hdfs-namenode-1 hdfs-namenode-2 -bin/radar-stack run --rm hdfs-namenode-1 namenode-1 -initializeSharedEdits -bin/radar-stack up -d hdfs-namenode-1 +bin/radar-docker up -d --remove-orphans zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 +bin/radar-docker quit hdfs-namenode-1 hdfs-namenode-2 +bin/radar-docker run --rm hdfs-namenode-1 namenode-1 -initializeSharedEdits +bin/radar-docker up -d hdfs-namenode-1 echo "Waiting for HDFS namenode to come online..." -while ! bin/radar-stack exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do +while ! bin/radar-docker exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do sleep 1 done -bin/radar-stack run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force -bin/radar-stack hdfs \ No newline at end of file +bin/radar-docker run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force +bin/radar-docker hdfs From f41418b1076461f2a7f4e11ba201918172304723 Mon Sep 17 00:00:00 2001 From: nivethika Date: Mon, 16 Jul 2018 17:22:45 +0200 Subject: [PATCH 037/194] use util.sh and sed_i --- dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh b/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh index 37ebb5a5a..051d3db60 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/migrate-postgres-data.sh @@ -2,6 +2,7 @@ set -e +. ./util.sh . ./.env @@ -30,7 +31,7 @@ mv "${MP_POSTGRES_DIR}/data/" "${MP_POSTGRES_DIR}/data-backup-$(date +%FT%TZ)/" mv "${POSTGRES_NEW_DIR}" "${MP_POSTGRES_DIR}/data/" # change postgres version -sed -i "s| image: postgres:.*| image: postgres:${POSTGRES_NEW_VERSION}|" docker-compose.yml +sed_i 's| image: postgres:.*| image: postgres:${POSTGRES_NEW_VERSION}|' docker-compose.yml echo "Starting postgres..." From b1300cd23da92855b4f64395b7cf326a01dc3fd6 Mon Sep 17 00:00:00 2001 From: nivethika Date: Mon, 16 Jul 2018 17:27:38 +0200 Subject: [PATCH 038/194] update connector's version --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 168cc6407..0e8539187 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -428,7 +428,7 @@ services: # RADAR mongo connector # #---------------------------------------------------------------------------# radar-mongodb-connector: - image: radarbase/kafka-connect-mongodb-sink:0.2.1 + image: radarbase/kafka-connect-mongodb-sink:0.2.2 restart: on-failure volumes: - ./etc/mongodb-connector/sink-mongo.properties:/etc/kafka-connect/sink.properties @@ -480,7 +480,7 @@ services: # RADAR HDFS connector # #---------------------------------------------------------------------------# radar-hdfs-connector: - image: radarbase/radar-hdfs-connector-auto:0.1 + image: radarbase/radar-connect-hdfs-sink:0.2.0 restart: on-failure volumes: - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties From bd69ce1fa8aa1278595cdbdff20ff6c3a5d34173 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 09:33:53 +0200 Subject: [PATCH 039/194] Apply PR comments --- .../radar-cp-hadoop-stack/bin/radar-docker | 93 ++++++++++++------- .../docker-compose.hdfs.yml | 5 + .../radar-cp-hadoop-stack/docker-compose.yml | 2 +- 3 files changed, 65 insertions(+), 35 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index a9e26292e..054ba2515 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -3,40 +3,65 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." stack="docker-compose -f docker-compose.yml -f docker-compose.hdfs.yml" -if [ $# != 0 ]; then - CMD=$1 - shift +function print_usage() { + echo 'Control the RADAR-base platform directly' + echo "Usage: $0 [COMMAND] ..." + echo + echo 'Commands:' + echo ' install [SERVICE...] install the RADAR-base platform, starting it in the background' + echo ' if services are provided in the arguments, only those services' + echo ' are actually started.' + echo ' hdfs start the HDFS cluster stack' + echo ' quit SERVICE... stop and remove given services' + echo ' rebuild SERVICE... rebuild and restart given services' + echo ' stack [SERVICE...] install the RADAR-base platform and start it in the foreground' + echo ' if services are provided in the arguments, only those services' + echo ' are actually started.' + echo + echo 'docker-compose commands and options are also accepted:' + echo '======================================================' + echo +} - case "${CMD}" in - "install") - . lib/perform-install.sh - ;; - "rebuild") - $stack build "$@" && \ - $stack stop "$@" && \ - $stack rm -vf "$@" && \ - $stack up -d "$@" && \ - exit 0 || exit 1 - ;; - "quit") - $stack stop "$@" && \ - $stack rm -vf "$@" && \ - exit 0 || exit 1 - ;; - "health") - . lib/radar-health.sh - ;; - "hdfs") - exec $stack up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 - ;; - "stack") - . lib/perform-install.sh - exec $stack up - ;; - *) - exec $stack "$CMD" "$@" - ;; - esac -else +if [ $# = 0 ]; then + print_usage exec $stack fi + + +CMD=$1 +shift + +# advanced pattern matching +shopt -s extglob + +case "${CMD}" in +install) + . lib/perform-install.sh + ;; +rebuild) + exec $stack up -d --force-recreate --build -V "$@" + ;; +quit) + $stack stop "$@" && \ + $stack rm -vf "$@" && \ + exit 0 || exit 1 + ;; +health) + . lib/radar-health.sh + ;; +hdfs) + exec $stack up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 + ;; +stack) + . lib/perform-install.sh + exec $stack up "$@" + ;; +@(--help|-h)) + print_usage + exec $stack "$CMD" "$@" + ;; +*) + exec $stack "$CMD" "$@" + ;; +esac diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml index 43ed93663..fc83f130e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml @@ -15,6 +15,7 @@ services: args: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} + hostname: hdfs-datanode-1 command: datanode networks: - hadoop @@ -51,6 +52,7 @@ services: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} command: datanode + hostname: hdfs-datanode-2 networks: - hadoop - zookeeper @@ -86,6 +88,7 @@ services: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} command: datanode + hostname: hdfs-datanode-3 networks: - hadoop - zookeeper @@ -121,6 +124,7 @@ services: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} command: namenode-1 + hostname: hdfs-namenode-1 networks: - hadoop - zookeeper @@ -156,6 +160,7 @@ services: BASE_VERSION: ${HDFS_BASE_VERSION} image: radarcns/hdfs:${HDFS_BASE_VERSION} command: namenode-2 + hostname: hdfs-namenode-2 networks: - hadoop - zookeeper diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index bcffbecdb..4b494d071 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -420,7 +420,7 @@ services: timeout: 5s retries: 3 - #------------------------------------------------------------------context: ./images/hdfs---# + #---------------------------------------------------------------------------# # RADAR backend streams # #---------------------------------------------------------------------------# radar-backend-stream: From d9a22746ba2d19eede97d98758df423e5e557d3b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 09:34:10 +0200 Subject: [PATCH 040/194] Comment HDFS commands --- .../radar-cp-hadoop-stack/bin/hdfs-ha-init | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init index b40acb793..e38b9a35c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init @@ -13,13 +13,19 @@ select yn in "Yes" "No"; do esac done +echo "==> Setting up name node 1" bin/radar-docker up -d --remove-orphans zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 bin/radar-docker quit hdfs-namenode-1 hdfs-namenode-2 -bin/radar-docker run --rm hdfs-namenode-1 namenode-1 -initializeSharedEdits +bin/radar-docker run --name hdfs-namenode-1 --rm hdfs-namenode-1 sh -c "su-exec hdfs hdfs namenode -initializeSharedEdits && su-exec hdfs hdfs zkfc -formatZK -force" bin/radar-docker up -d hdfs-namenode-1 -echo "Waiting for HDFS namenode to come online..." -while ! bin/radar-docker exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do +printf "==> Waiting for name node 1 to come online" +while ! bin/radar-docker exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/ 2>/dev/null; do sleep 1 + printf '.' done +echo +echo "==> Setting up name node 2" bin/radar-docker run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force + +echo "==> Bringing up HDFS cluster" bin/radar-docker hdfs From 22391f08577a8e3c85f6c8abfaa6fefecf0dac8c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 09:44:40 +0200 Subject: [PATCH 041/194] Printing progress of HDFS scripts --- .../radar-cp-hadoop-stack/bin/hdfs-ha-init | 5 +++-- .../radar-cp-hadoop-stack/bin/hdfs-upgrade | 22 +++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init index e38b9a35c..aa7e3268b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init @@ -4,7 +4,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." . ./.env -echo "Are you sure you want to initialize HDFS?" +echo "Are you sure you want to initialize HDFS High Availability?" select yn in "Yes" "No"; do case $yn in @@ -19,10 +19,11 @@ bin/radar-docker quit hdfs-namenode-1 hdfs-namenode-2 bin/radar-docker run --name hdfs-namenode-1 --rm hdfs-namenode-1 sh -c "su-exec hdfs hdfs namenode -initializeSharedEdits && su-exec hdfs hdfs zkfc -formatZK -force" bin/radar-docker up -d hdfs-namenode-1 printf "==> Waiting for name node 1 to come online" -while ! bin/radar-docker exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/ 2>/dev/null; do +while ! bin/radar-docker exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/ 2>/dev/null >/dev/null; do sleep 1 printf '.' done +sleep 5 echo echo "==> Setting up name node 2" bin/radar-docker run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade index 14a57f67b..4b967e119 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade @@ -21,25 +21,39 @@ select yn in "Yes" "No"; do esac done +echo "==> Stopping HDFS name nodes" $stack quit hdfs-namenode-1 hdfs-namenode-2 radar-hdfs-connector -sed_i "s/HDFS_BASE_VERSION=.*/HDFS_BASE_VERSION=${VERSION}/" "${STACK_DIR}/.env" +echo "==> Updating HDFS version to ${VERSION}" +if grep -q HDFS_BASE_VERSION ./.env; then + sed_i "s/HDFS_BASE_VERSION=.*/HDFS_BASE_VERSION=${VERSION}/" ./.env +else + echo "HDFS_BASE_VERSION=${VERSION}" >> ./.env +fi $stack build + +echo "==> Starting updated HDFS HA Quorum" $stack up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 sleep 10 +echo "==> Upgrading name node 1" NAME_1=$($stack run --rm -d --name hdfs-namenode-1 hdfs-namenode-1 namenode-1 -upgrade) echo "Waiting for the upgrade to finalize..." while ! docker exec $NAME_1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do sleep 1 done +sleep 5 +echo "==> Upgrading name node 2" $stack run --no-deps --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby +echo "==> Starting name node 2" $stack up --no-deps -d hdfs-namenode-2 sleep 10 -docker exec $NAME_1 hdfs dfsadmin -finalizeUpgrade - +echo "==> Finalizing upgrade" +docker exec $NAME_1 su-exec hdfs hdfs dfsadmin -finalizeUpgrade docker rm -vf $NAME_1 + +echo "==> Bringing up HDFS cluster" $stack hdfs -$stack up -d radar-hdfs-connector \ No newline at end of file +$stack up -d radar-hdfs-connector From 524bb99a6af605683770b057bb92784dd36c76dd Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 09:51:21 +0200 Subject: [PATCH 042/194] Fix typo --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 4b494d071..13d92f3f7 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -402,7 +402,7 @@ services: CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kcontext: ./images/hdfsonnect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" CONNECT_REST_ADVERTISED_HOST_NAME: "radar-mongodb-connector" CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 From 82f50f92138387c9f4ec4a44a95d91ed333ebcea Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 09:54:50 +0200 Subject: [PATCH 043/194] Remove unneeded zookeeper dependencies --- .../docker-compose.hdfs.yml | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml index fc83f130e..7b38b6ff8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml @@ -19,11 +19,7 @@ services: command: datanode networks: - hadoop - - zookeeper depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - hdfs-namenode-1 - hdfs-namenode-2 volumes: @@ -33,7 +29,6 @@ services: SERVICE_9866_NAME: datanode SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -55,11 +50,7 @@ services: hostname: hdfs-datanode-2 networks: - hadoop - - zookeeper depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - hdfs-namenode-1 - hdfs-namenode-2 volumes: @@ -69,7 +60,6 @@ services: SERVICE_9866_NAME: datanode SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -91,11 +81,7 @@ services: hostname: hdfs-datanode-3 networks: - hadoop - - zookeeper depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - hdfs-namenode-1 - hdfs-namenode-2 volumes: @@ -105,7 +91,6 @@ services: SERVICE_9866_NAME: datanode SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 @@ -200,14 +185,12 @@ services: command: journalnode networks: - hadoop - - zookeeper volumes: - "${HDFS_JOURNAL_DIR_1}:/hadoop/dfs/journal" restart: always environment: SERVICE_8485_NAME: journalnode SERVICE_8480_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: @@ -225,14 +208,12 @@ services: command: journalnode networks: - hadoop - - zookeeper volumes: - "${HDFS_JOURNAL_DIR_2}:/hadoop/dfs/journal" restart: always environment: SERVICE_8485_NAME: journalnode SERVICE_8480_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: @@ -250,14 +231,12 @@ services: command: journalnode networks: - hadoop - - zookeeper volumes: - "${HDFS_JOURNAL_DIR_3}:/hadoop/dfs/journal" restart: always environment: SERVICE_8485_NAME: journalnode SERVICE_8480_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: From d14d67118e989dfa6fe1945883e8b13d2e8237dd Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 11:18:08 +0200 Subject: [PATCH 044/194] export env var from env.template --- .travis.yml | 1 + dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b61a01b0..f72da17f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,7 @@ script: - cp etc/webserver/nginx.conf.template etc/webserver/nginx.conf - cp etc/hdfs-connector/sink-hdfs.properties.template etc/hdfs-connector/sink-hdfs.properties - cp etc/mongodb-connector/sink-mongo.properties.template etc/mongodb-connector/sink-mongo.properties + - . etc/env.template - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down - sudo docker network rm hadoop diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index a9246b534..03fd8c191 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -5,18 +5,18 @@ SELF_SIGNED_CERT=yes MAINTAINER_EMAIL=me@example.com RADAR_RAW_TOPIC_LIST= RADAR_AGG_TOPIC_LIST= -HOTSTORAGE_USERNAME= +HOTSTORAGE_USERNAME=mongodb-user HOTSTORAGE_PASSWORD=XXXXXXXX -HOTSTORAGE_NAME= +HOTSTORAGE_NAME=mongodb-database HDFS_DATA_DIR_1=/usr/local/var/lib/docker/hdfs-data-1 HDFS_DATA_DIR_2=/usr/local/var/lib/docker/hdfs-data-2 HDFS_NAME_DIR_1=/usr/local/var/lib/docker/hdfs-name-1 HDFS_NAME_DIR_2=/usr/local/var/lib/docker/hdfs-name-2 MONGODB_DIR=/usr/local/var/lib/docker/mongodb MP_POSTGRES_DIR=/usr/local/var/lib/docker/postgres -POSTGRES_USER= -POSTGRES_PASSWORD= -FROM_EMAIL= +POSTGRES_USER=postgresdb-user +POSTGRES_PASSWORD=postgresdb-pwd +FROM_EMAIL=from-email MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT=false HEALTHCHECK_HIPCHAT_NOTIFY=no HEALTHCHECK_HIPCHAT_ROOM_ID="" From 1ba2d00a25d8975a052c84b12d1d44e07189a375 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 11:31:47 +0200 Subject: [PATCH 045/194] echo postgres version --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f72da17f6..e9d798117 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,7 @@ script: - cp etc/hdfs-connector/sink-hdfs.properties.template etc/hdfs-connector/sink-hdfs.properties - cp etc/mongodb-connector/sink-mongo.properties.template etc/mongodb-connector/sink-mongo.properties - . etc/env.template + - echo POSTGRES_NEW_VERSION="$POSTGRES_NEW_VERSION" - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down - sudo docker network rm hadoop From d9236b024b4ed2fc14c2901ef21424b0eb46d011 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 11:42:47 +0200 Subject: [PATCH 046/194] upgrade docker-compose version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e9d798117..5a3c66acc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: required services: - docker env: - DOCKER_COMPOSE_VERSION: 1.11.2 + DOCKER_COMPOSE_VERSION: 1.21.1 before_install: - docker --version From d070775295a23df384bc6e52c5f9f09104a441a4 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 13:20:38 +0200 Subject: [PATCH 047/194] Remove old README comment --- dcompose-stack/radar-cp-hadoop-stack/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index a6b29d2c6..1ae4eb753 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -162,11 +162,7 @@ Add a cron job to run the `radar-health` script periodically like - */5 * * * * /home/ubuntu/RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/bin/radar-health ``` -You can check the logs of CRON by typing `grep CRON /var/log/syslog` -Also you will need to change the directory. So just add the following to the top of the `radar-health` script - -```sh -cd "$( dirname "${BASH_SOURCE[0]}" )/.." -``` +You can check the logs of CRON by typing `grep CRON /var/log/syslog`. ### HDFS From 17b19efa53e29822c587ce78e43deebb54e621c8 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 13:50:45 +0200 Subject: [PATCH 048/194] mock config setup for travis and use install-radar-stack --- .travis.yml | 21 ++++------------ .../radar-cp-hadoop-stack/travis/setup-env.sh | 19 +++++++++++++++ .../travis/travis-env.template | 24 +++++++++++++++++++ .../travis/travis-smtp.template | 4 ++++ 4 files changed, 51 insertions(+), 17 deletions(-) create mode 100755 dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh create mode 100644 dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template create mode 100644 dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template diff --git a/.travis.yml b/.travis.yml index 5a3c66acc..32f40e268 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,23 +25,10 @@ script: #- sudo $HOME/bin/docker-compose down # With email and HDFS support - - cd ../radar-cp-hadoop-stack - - sudo docker network create hadoop - - export SERVER_NAME=localhost - - export HDFS_DATA_DIR_1=$PWD/hdfs-data1 - - export HDFS_DATA_DIR_2=$PWD/hdfs-data2 - - export HDFS_NAME_DIR_1=$PWD/hdfs-name1 - - export HDFS_NAME_DIR_2=$PWD/hdfs-name2 - - echo $"SMARTHOST_ADDRESS=mail.example.com\nSMARTHOST_PORT=587\nSMARTHOST_USER=user@example.com\nSMARTHOST_PASSWORD=XXXXXXXX" > etc/smtp.env - - sudo docker volume create certs - - sudo docker volume create certs-data - - cp etc/radar-backend/radar.yml.template etc/radar-backend/radar.yml - - cp etc/webserver/nginx.conf.template etc/webserver/nginx.conf - - cp etc/hdfs-connector/sink-hdfs.properties.template etc/hdfs-connector/sink-hdfs.properties - - cp etc/mongodb-connector/sink-mongo.properties.template etc/mongodb-connector/sink-mongo.properties - - . etc/env.template - - echo POSTGRES_NEW_VERSION="$POSTGRES_NEW_VERSION" - - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] + - cd ../radar-cp-hadoop-stack/travis + - ./setup-env.sh + - cd ../ + - ./install-radar-stack.sh && sleep 60 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down - sudo docker network rm hadoop diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh new file mode 100755 index 000000000..ad0323ec0 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# create hadoop netword +sudo docker network create hadoop + +# create volumes +sudo docker volume create certs +sudo docker volume create certs-data + +# setup mock configs +cp ./travis-env.template ../.env +cp ./travis-smtp.template ../etc/smtp.env +cp ../etc/radar-backend/radar.yml.template ../etc/radar-backend/radar.yml +cp ../etc/webserver/nginx.conf.template ../etc/webserver/nginx.conf +cp ../etc/hdfs-connector/sink-hdfs.properties.template ../etc/hdfs-connector/sink-hdfs.properties +cp ../etc/mongodb-connector/sink-mongo.properties.template ../etc/mongodb-connector/sink-mongo.properties +cp ../etc/managementportal/config/oauth_client_details.csv.template ../etc/managementportal/config/oauth_client_details.csv + +echo "Setup mock configurations are finished..." \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template new file mode 100644 index 000000000..4b927a802 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -0,0 +1,24 @@ +SERVER_NAME=localhost +MANAGEMENTPORTAL_KEY_DNAME=CN=localhost,OU=MyName,O=MyOrg,L=MyCity,S=MyState,C=MyCountryCode +MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET=test +SELF_SIGNED_CERT=yes +MAINTAINER_EMAIL=me@example.com +RADAR_RAW_TOPIC_LIST= +RADAR_AGG_TOPIC_LIST= +HOTSTORAGE_USERNAME=mongodb-user +HOTSTORAGE_PASSWORD=mongo-pwd +HOTSTORAGE_NAME=mongodb-database +HDFS_DATA_DIR_1=$PWD/hdfs-data-1 +HDFS_DATA_DIR_2=$PWD/hdfs-data-2 +HDFS_NAME_DIR_1=$PWD/hdfs-name-1 +HDFS_NAME_DIR_2=$PWD/hdfs-name-2 +MONGODB_DIR=$PWD/mongodb +MP_POSTGRES_DIR=$PWD/postgres +POSTGRES_USER=postgresdb-user +POSTGRES_PASSWORD=postgresdb-pwd +FROM_EMAIL=from-email +MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT=false +HEALTHCHECK_HIPCHAT_NOTIFY=no +HEALTHCHECK_HIPCHAT_ROOM_ID="" +HEALTHCHECK_HIPCHAT_TOKEN="" +POSTGRES_NEW_VERSION=10.4-alpine diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template new file mode 100644 index 000000000..58a8cdb8b --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template @@ -0,0 +1,4 @@ +SMARTHOST_ADDRESS=mail.example.com +SMARTHOST_PORT=587 +SMARTHOST_USER=user@example.com +SMARTHOST_PASSWORD=XXXXXXXX \ No newline at end of file From 9ccb8d4594ef43a8ac9441253ec774e9bbec4cbe Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 13:53:54 +0200 Subject: [PATCH 049/194] Update systemd wrappers --- .../radar-cp-hadoop-stack/README.md | 12 +++++------ .../radar-cp-hadoop-stack/bin/radar-docker | 21 +++++++++++++------ .../install-systemd-wrappers.sh} | 6 ++---- 3 files changed, 23 insertions(+), 16 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/{bin/radar-systemd-wrappers => lib/install-systemd-wrappers.sh} (89%) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 1ae4eb753..420a6b23b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -52,7 +52,7 @@ to start all the RADAR services. Use the `bin/radar-docker start|down|restart` t To enable a `systemd` service to control the platform, run ```shell -bin/radar-systemd-wrappers +bin/radar-docker install-systemd ``` After that command, the RADAR platform should be controlled via `systemctl`. ```shell @@ -142,24 +142,24 @@ Portainer provides simple interactive UI-based docker management. If running loc The [kafka-manager](https://github.com/yahoo/kafka-manager) is an interactive web based tool for managing Apache Kafka. Kafka manager has beed integrated in the stack. It is accessible at `http:///kafkamanager/` ### Check Health -Each of the containers in the stack monitor their own health and show the output as healthy or unhealthy. A script called `bin/radar-health` is used to check this output and send an email to the maintainer if a container is unhealthy. +Each of the containers in the stack monitor their own health and show the output as healthy or unhealthy. A script called `bin/radar-docker health` is used to check this output and send an email to the maintainer if a container is unhealthy. First check that the `MAINTAINER_EMAIL` in the .env file is correct. Then make sure that the SMTP server is configured properly and running. -If systemd integration is enabled, the `radar-health` script will check health of containers every five minutes. It can then be run directly by running if systemd wrappers have been installed +If systemd integration is enabled, the `bin/radar-docker health` script will check health of containers every five minutes. It can then be run directly by running if systemd wrappers have been installed ``` sudo systemctl start radar-check-health.service ``` Otherwise, the following manual commands can be invoked. -Add a cron job to run the `radar-health` script periodically like - +Add a cron job to run the `bin/radar-docker health` script periodically like - 1. Edit the crontab file for the current user by typing `$ crontab -e` 2. Add your job and time interval. For example, add the following for checking health every 5 mins - ``` -*/5 * * * * /home/ubuntu/RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/bin/radar-health +*/5 * * * * /home/ubuntu/RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker health ``` You can check the logs of CRON by typing `grep CRON /var/log/syslog`. @@ -183,7 +183,7 @@ This folder contains useful scripts to manage the extraction of data from HDFS i - `storage_directory` is the directory where the extracted data will be stored - `lockfile` lock useful to check whether there is a previous instance still running -- A systemd timer for this script can be installed by running the `bin/radar-systemd-wrappers`. Or you can add a cron job like below. +- A systemd timer for this script can be installed by running the `bin/radar-docker install-systemd`. Or you can add a cron job like below. To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u root` and add your task at the end of the file. The syntax is ```shell diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index 054ba2515..f994e0c33 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -8,15 +8,18 @@ function print_usage() { echo "Usage: $0 [COMMAND] ..." echo echo 'Commands:' - echo ' install [SERVICE...] install the RADAR-base platform, starting it in the background' + echo ' cert-renew Request or renew SSL certificate' + echo ' foreground [SERVICE...] Install the RADAR-base platform and start it in the foreground.' echo ' if services are provided in the arguments, only those services' echo ' are actually started.' - echo ' hdfs start the HDFS cluster stack' - echo ' quit SERVICE... stop and remove given services' - echo ' rebuild SERVICE... rebuild and restart given services' - echo ' stack [SERVICE...] install the RADAR-base platform and start it in the foreground' + echo ' install [SERVICE...] Install the RADAR-base platform, starting it in the background.' echo ' if services are provided in the arguments, only those services' echo ' are actually started.' + echo ' install-systemd Install systemd wrappers to manage the RADAR platform' + echo ' hdfs Start the HDFS cluster stack.' + echo ' health Check the health of the cluster and restart failing services.' + echo ' quit SERVICE... Stop and remove given services.' + echo ' rebuild SERVICE... Rebuild and restart given services.' echo echo 'docker-compose commands and options are also accepted:' echo '======================================================' @@ -36,9 +39,15 @@ shift shopt -s extglob case "${CMD}" in +cert-renew) + request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" force + ;; install) . lib/perform-install.sh ;; +install-systemd) + . lib/install-systemd-wrappers.sh + ;; rebuild) exec $stack up -d --force-recreate --build -V "$@" ;; @@ -53,7 +62,7 @@ health) hdfs) exec $stack up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 ;; -stack) +foreground) . lib/perform-install.sh exec $stack up "$@" ;; diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh similarity index 89% rename from dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers rename to dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index 7436c4720..55530c5bc 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-systemd-wrappers +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -1,5 +1,3 @@ -#!/bin/bash - cd "$(dirname "${BASH_SOURCE[0]}")/.." . lib/util.sh @@ -15,7 +13,7 @@ copy_template_if_absent /etc/systemd/system/radar-renew-certificate.service lib/ copy_template_if_absent /etc/systemd/system/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-docker.service -inline_variable 'ExecStart=' "$PWD/bin/radar-docker stack" /etc/systemd/system/radar-docker.service +inline_variable 'ExecStart=' "$PWD/bin/radar-docker foreground" /etc/systemd/system/radar-docker.service inline_variable 'WorkingDirectory=' "$PWD/hdfs" /etc/systemd/system/radar-output.service inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure-process" /etc/systemd/system/radar-output.service @@ -24,7 +22,7 @@ inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-check-healt inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" /etc/systemd/system/radar-check-health.service inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-renew-certificate.service -inline_variable 'ExecStart=' "$PWD/bin/radar-cert-renew" /etc/systemd/system/radar-renew-certificate.service +inline_variable 'ExecStart=' "$PWD/bin/radar-docker cert-renew" /etc/systemd/system/radar-renew-certificate.service sudo systemctl daemon-reload sudo systemctl enable radar-docker From 208f6d3ddc55a1ffa915a9918b89490f47ea4d58 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 13:54:13 +0200 Subject: [PATCH 050/194] Fix hdfs images and health checks --- .../docker-compose.hdfs.yml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml index 7b38b6ff8..1c4bd768e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml @@ -14,7 +14,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} hostname: hdfs-datanode-1 command: datanode networks: @@ -35,7 +35,7 @@ services: HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 HADOOP_DFS_REPLICATION: 2 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 @@ -45,7 +45,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} command: datanode hostname: hdfs-datanode-2 networks: @@ -66,7 +66,7 @@ services: HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 HADOOP_DFS_REPLICATION: 2 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 @@ -76,7 +76,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} command: datanode hostname: hdfs-datanode-3 networks: @@ -97,7 +97,7 @@ services: HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 HADOOP_DFS_REPLICATION: 2 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 @@ -107,7 +107,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} command: namenode-1 hostname: hdfs-namenode-1 networks: @@ -133,7 +133,7 @@ services: HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 HADOOP_QJOURNAL_ADDRESS: hdfs-journalnode-1:8485;hdfs-journalnode-2:8485;hdfs-journalnode-3:8485 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 @@ -143,7 +143,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} command: namenode-2 hostname: hdfs-namenode-2 networks: @@ -171,7 +171,7 @@ services: HADOOP_DFS_REPLICATION: 2 HADOOP_QJOURNAL_ADDRESS: hdfs-journalnode-1:8485;hdfs-journalnode-2:8485;hdfs-journalnode-3:8485 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 @@ -181,7 +181,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} command: journalnode networks: - hadoop @@ -194,7 +194,7 @@ services: HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 @@ -204,7 +204,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} command: journalnode networks: - hadoop @@ -217,7 +217,7 @@ services: HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 @@ -227,7 +227,7 @@ services: context: ./images/hdfs args: BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarcns/hdfs:${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} command: journalnode networks: - hadoop @@ -240,7 +240,7 @@ services: HADOOP_HEAPSIZE: 1000 HADOOP_NAMENODE_HA: nn1,nn2 healthcheck: - test: ["CMD", "hdfs", "dfs", "-ls", "/"] + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m timeout: 10s retries: 3 From 7df913a2a9ef9936d728f687f61d97522e628050 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 13:54:34 +0200 Subject: [PATCH 051/194] Configure high availability for extract/restructure --- .../radar-cp-hadoop-stack/bin/hdfs-extract | 12 +++++++++++- .../radar-cp-hadoop-stack/bin/hdfs-restructure | 12 ++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract index 5ac15d336..7108db264 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract @@ -19,6 +19,16 @@ OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" HDFS_OUTPUT_DIR=/home/output # HDFS command to run HDFS_COMMAND="hdfs dfs -get $HDFS_FILE $HDFS_OUTPUT_DIR" +DOCKER_OPTS=( + -i --rm + --network hadoop + -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" + -e HADOOP_NAMENODE_HA=nn1,nn2 + -e HADOOP_NAMENODE1_HOSTNAME=hdfs-namenode-1 + -e HADOOP_NAMENODE2_HOSTNAME=hdfs-namenode-2) mkdir -p $OUTPUT_DIR -sudo-linux docker run -i --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" -e CORE_CONF_fs_defaultFS=hdfs://hdfs-namenode:8020 radarbase/hdfs:${HDFS_BASE_VERSION} $HDFS_COMMAND + +echo "==> Extracting HDFS path $HDFS_FILE to $OUTPUT_DIR" +echo docker run "${DOCKER_OPTS[@]}" radarbase/hdfs:${HDFS_BASE_VERSION} $HDFS_COMMAND +sudo-linux docker run "${DOCKER_OPTS[@]}" radarbase/hdfs:${HDFS_BASE_VERSION} $HDFS_COMMAND diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index bf0f8185f..2a75c0ea8 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -13,7 +13,7 @@ DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) sudo-linux "$DIR/bin/radar-docker" hdfs # HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.0 +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.1-SNAPSHOT # HDFS filename to get HDFS_FILE=$1 @@ -23,7 +23,15 @@ OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" # Internal docker directory to write output to HDFS_OUTPUT_DIR=/output # HDFS command to run -HDFS_COMMAND=(--compression gzip --deduplicate -u hdfs://hdfs-namenode:8020 -o "$HDFS_OUTPUT_DIR" "$HDFS_FILE" ) +HDFS_COMMAND=( + --compression gzip + --deduplicate + -n hadoop + --namenode-1 hdfs-namenode-1 + --namenode-2 hdfs-namenode-2 + --namenode-ha nn1,nn2 + -o "$HDFS_OUTPUT_DIR" + "$HDFS_FILE" ) mkdir -p $OUTPUT_DIR sudo-linux docker run -t --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" $DOCKER_IMAGE "${HDFS_COMMAND[@]}" From f296249e5b0cd87b843b5aaf849729d2319d481c Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 13:56:26 +0200 Subject: [PATCH 052/194] add mock redcap-integration config --- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index ad0323ec0..1895d2ee3 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -15,5 +15,6 @@ cp ../etc/webserver/nginx.conf.template ../etc/webserver/nginx.conf cp ../etc/hdfs-connector/sink-hdfs.properties.template ../etc/hdfs-connector/sink-hdfs.properties cp ../etc/mongodb-connector/sink-mongo.properties.template ../etc/mongodb-connector/sink-mongo.properties cp ../etc/managementportal/config/oauth_client_details.csv.template ../etc/managementportal/config/oauth_client_details.csv +cp ../etc/redcap-integration/radar.yml.template ../etc/redcap-integration/radar.yml echo "Setup mock configurations are finished..." \ No newline at end of file From ed94d5fa4e34a517e3f5ccd3849bfc262fdf1ed9 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 14:01:38 +0200 Subject: [PATCH 053/194] change volume dir values --- .../radar-cp-hadoop-stack/travis/travis-env.template | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 4b927a802..6e64e9c0a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -8,12 +8,12 @@ RADAR_AGG_TOPIC_LIST= HOTSTORAGE_USERNAME=mongodb-user HOTSTORAGE_PASSWORD=mongo-pwd HOTSTORAGE_NAME=mongodb-database -HDFS_DATA_DIR_1=$PWD/hdfs-data-1 -HDFS_DATA_DIR_2=$PWD/hdfs-data-2 -HDFS_NAME_DIR_1=$PWD/hdfs-name-1 -HDFS_NAME_DIR_2=$PWD/hdfs-name-2 -MONGODB_DIR=$PWD/mongodb -MP_POSTGRES_DIR=$PWD/postgres +HDFS_DATA_DIR_1=/usr/local/var/lib/docker/hdfs-data-1 +HDFS_DATA_DIR_2=/usr/local/var/lib/docker/hdfs-data-2 +HDFS_NAME_DIR_1=/usr/local/var/lib/docker/hdfs-name-1 +HDFS_NAME_DIR_2=/usr/local/var/lib/docker/hdfs-name-2 +MONGODB_DIR=/usr/local/var/lib/docker/mongodb +MP_POSTGRES_DIR=/usr/local/var/lib/docker/postgres POSTGRES_USER=postgresdb-user POSTGRES_PASSWORD=postgresdb-pwd FROM_EMAIL=from-email From f518516ae09942a38f27c9bcdfcd3f36773cfea8 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 14:10:44 +0200 Subject: [PATCH 054/194] make parent directory for docker volumes --- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index 1895d2ee3..07819eed1 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -7,6 +7,9 @@ sudo docker network create hadoop sudo docker volume create certs sudo docker volume create certs-data +# create folder for docker volumes +mkdir -p /usr/local/var/lib/docker/ + # setup mock configs cp ./travis-env.template ../.env cp ./travis-smtp.template ../etc/smtp.env From 5994a125f8df16042aade9e6e4521eaa4394818c Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 14:12:17 +0200 Subject: [PATCH 055/194] remove created directories after check --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 32f40e268..05f98bf87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,7 @@ script: - cd ../ - ./install-radar-stack.sh && sleep 60 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down + - sudo rm -r /usr/local/var/lib/docker/ - sudo docker network rm hadoop # With Docker Swarm support From afd319e80f57f124ee92ac4aad1351abdf3417de Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 14:29:53 +0200 Subject: [PATCH 056/194] add sudo --- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index 07819eed1..00deac6d9 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -8,7 +8,7 @@ sudo docker volume create certs sudo docker volume create certs-data # create folder for docker volumes -mkdir -p /usr/local/var/lib/docker/ +sudo mkdir -p /usr/local/var/lib/docker/ # setup mock configs cp ./travis-env.template ../.env From afc3c0c4d8c1ad6a9eb2abbf0e35e11e31c7ba67 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 14:58:08 +0200 Subject: [PATCH 057/194] add radar-stop and docker prune --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 05f98bf87..9a86d2913 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,8 @@ script: - ./setup-env.sh - cd ../ - ./install-radar-stack.sh && sleep 60 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - - sudo $HOME/bin/docker-compose down + - ./stop-radar-stack.sh + - echo Yes | ./docker-prune.sh - sudo rm -r /usr/local/var/lib/docker/ - sudo docker network rm hadoop From 240498dfcc1186056ba7119e81b810b27e7025ef Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 15:12:01 +0200 Subject: [PATCH 058/194] yes command for prune --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9a86d2913..981ee9105 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ script: - cd ../ - ./install-radar-stack.sh && sleep 60 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - ./stop-radar-stack.sh - - echo Yes | ./docker-prune.sh + - yes Y | ./docker-prune.sh - sudo rm -r /usr/local/var/lib/docker/ - sudo docker network rm hadoop From 622ab61e705fa084cf02624165000122ddf6cf73 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 15:29:30 +0200 Subject: [PATCH 059/194] yes command for prune --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 981ee9105..6e7ab8a68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ script: - cd ../ - ./install-radar-stack.sh && sleep 60 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - ./stop-radar-stack.sh - - yes Y | ./docker-prune.sh + - yes | ./docker-prune.sh - sudo rm -r /usr/local/var/lib/docker/ - sudo docker network rm hadoop From 46b33957558b1abc537fa9f78c8e8752dc953821 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 15:31:55 +0200 Subject: [PATCH 060/194] remove code for clean-up --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e7ab8a68..c659644b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,10 +29,6 @@ script: - ./setup-env.sh - cd ../ - ./install-radar-stack.sh && sleep 60 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - - ./stop-radar-stack.sh - - yes | ./docker-prune.sh - - sudo rm -r /usr/local/var/lib/docker/ - - sudo docker network rm hadoop # With Docker Swarm support # NOT SUPPORTED: docker swarm and docker beta features are not available in Travis From 81d13bb4eb2667a8ba727cb050f76899309438fb Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 15:33:06 +0200 Subject: [PATCH 061/194] Revert to non-HA HDFS setup HDFS with HA turns out to be difficult to configure and has quite some overhead --- .../radar-cp-hadoop-stack/README.md | 3 - .../radar-cp-hadoop-stack/bin/hdfs-extract | 4 +- .../radar-cp-hadoop-stack/bin/hdfs-ha-init | 32 ---- .../bin/hdfs-restructure | 8 +- .../radar-cp-hadoop-stack/bin/hdfs-upgrade | 23 +-- .../radar-cp-hadoop-stack/bin/radar-docker | 2 +- .../docker-compose.hdfs.yml | 138 +----------------- .../hdfs-connector/sink-hdfs-high.properties | 4 +- .../hdfs-connector/sink-hdfs-low.properties | 4 +- .../hdfs-connector/sink-hdfs-med.properties | 4 +- .../images/hdfs/Dockerfile | 4 +- .../images/hdfs/entrypoint.sh | 35 ++++- .../images/hdfs/hdfs-site.xml.mustache | 13 +- .../lib/perform-install.sh | 2 +- 14 files changed, 61 insertions(+), 215 deletions(-) delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 420a6b23b..11ed8f76c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -168,9 +168,6 @@ You can check the logs of CRON by typing `grep CRON /var/log/syslog`. This folder contains useful scripts to manage the extraction of data from HDFS in the RADAR-base Platform. -- `bin/hdfs-ha-init` - - This initializes the HDFS structure to use High Availability mode with two name nodes. This needs to be called after converting from - the older uhopper images. - `bin/hdfs-upgrade VERSION` - Perform an upgrade from an older version of the [Smizy HDFS base image](https://hub.docker.com/r/smizy/hadoop-base/) to a newer one. E.g. from `2.7.6-alpine`, which is compatible with the `uhopper` image, to `3.0.3-alpine`. - `bin/hdfs-restructure` diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract index 7108db264..ee5923e57 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-extract @@ -23,9 +23,7 @@ DOCKER_OPTS=( -i --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" - -e HADOOP_NAMENODE_HA=nn1,nn2 - -e HADOOP_NAMENODE1_HOSTNAME=hdfs-namenode-1 - -e HADOOP_NAMENODE2_HOSTNAME=hdfs-namenode-2) + -e HADOOP_NAMENODE1_HOSTNAME=hdfs-namenode-1) mkdir -p $OUTPUT_DIR diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init deleted file mode 100755 index aa7e3268b..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-ha-init +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -cd "$(dirname "${BASH_SOURCE[0]}")/.." - -. ./.env - -echo "Are you sure you want to initialize HDFS High Availability?" - -select yn in "Yes" "No"; do - case $yn in - Yes ) break;; - No ) exit;; - esac -done - -echo "==> Setting up name node 1" -bin/radar-docker up -d --remove-orphans zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 -bin/radar-docker quit hdfs-namenode-1 hdfs-namenode-2 -bin/radar-docker run --name hdfs-namenode-1 --rm hdfs-namenode-1 sh -c "su-exec hdfs hdfs namenode -initializeSharedEdits && su-exec hdfs hdfs zkfc -formatZK -force" -bin/radar-docker up -d hdfs-namenode-1 -printf "==> Waiting for name node 1 to come online" -while ! bin/radar-docker exec hdfs-namenode-1 hdfs dfs -test -e hdfs://hdfs-namenode-1/ 2>/dev/null >/dev/null; do - sleep 1 - printf '.' -done -sleep 5 -echo -echo "==> Setting up name node 2" -bin/radar-docker run --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -force - -echo "==> Bringing up HDFS cluster" -bin/radar-docker hdfs diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 2a75c0ea8..ed2bb16ab 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -13,7 +13,8 @@ DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) sudo-linux "$DIR/bin/radar-docker" hdfs # HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.1-SNAPSHOT +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:dev +docker pull $DOCKER_IMAGE # HDFS filename to get HDFS_FILE=$1 @@ -26,10 +27,7 @@ HDFS_OUTPUT_DIR=/output HDFS_COMMAND=( --compression gzip --deduplicate - -n hadoop - --namenode-1 hdfs-namenode-1 - --namenode-2 hdfs-namenode-2 - --namenode-ha nn1,nn2 + -n hdfs-namenode-1 -o "$HDFS_OUTPUT_DIR" "$HDFS_FILE" ) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade index 4b967e119..343e63115 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade @@ -8,7 +8,7 @@ fi VERSION=$1 cd "$(dirname "${BASH_SOURCE[0]}")/.." -stack=bin/radar-stack +stack=bin/radar-docker . ./.env . lib/util.sh @@ -22,7 +22,7 @@ select yn in "Yes" "No"; do done echo "==> Stopping HDFS name nodes" -$stack quit hdfs-namenode-1 hdfs-namenode-2 radar-hdfs-connector +$stack quit hdfs-namenode-1 radar-hdfs-connector echo "==> Updating HDFS version to ${VERSION}" if grep -q HDFS_BASE_VERSION ./.env; then @@ -33,26 +33,13 @@ fi $stack build -echo "==> Starting updated HDFS HA Quorum" -$stack up -d zookeeper-1 zookeeper-2 zookeeper-3 hdfs-journalnode-1 hdfs-journalnode-2 hdfs-journalnode-3 -sleep 10 echo "==> Upgrading name node 1" -NAME_1=$($stack run --rm -d --name hdfs-namenode-1 hdfs-namenode-1 namenode-1 -upgrade) - -echo "Waiting for the upgrade to finalize..." -while ! docker exec $NAME_1 hdfs dfs -test -e hdfs://hdfs-namenode-1/; do - sleep 1 -done +$stack run --rm --name hdfs-namenode-1 hdfs-namenode-1 namenode-1 -upgradeOnly +$stack up -d hdfs-namenode-1 sleep 5 -echo "==> Upgrading name node 2" -$stack run --no-deps --rm --name hdfs-namenode-2 hdfs-namenode-2 namenode-2 -bootstrapStandby -echo "==> Starting name node 2" -$stack up --no-deps -d hdfs-namenode-2 -sleep 10 echo "==> Finalizing upgrade" -docker exec $NAME_1 su-exec hdfs hdfs dfsadmin -finalizeUpgrade -docker rm -vf $NAME_1 +$stack exec hdfs-namenode-1 su-exec hdfs hdfs dfsadmin -finalizeUpgrade echo "==> Bringing up HDFS cluster" $stack hdfs diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index f994e0c33..b704fe11a 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -60,7 +60,7 @@ health) . lib/radar-health.sh ;; hdfs) - exec $stack up -d hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 + exec $stack up -d --remove-orphans hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 ;; foreground) . lib/perform-install.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml index 1c4bd768e..07fad11b8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml @@ -21,7 +21,6 @@ services: - hadoop depends_on: - hdfs-namenode-1 - - hdfs-namenode-2 volumes: - "${HDFS_DATA_DIR_1}:/hadoop/dfs/data" restart: always @@ -30,9 +29,7 @@ services: SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 HADOOP_DFS_REPLICATION: 2 healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] @@ -52,7 +49,6 @@ services: - hadoop depends_on: - hdfs-namenode-1 - - hdfs-namenode-2 volumes: - "${HDFS_DATA_DIR_2}:/hadoop/dfs/data" restart: always @@ -61,9 +57,7 @@ services: SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 HADOOP_DFS_REPLICATION: 2 healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] @@ -83,7 +77,6 @@ services: - hadoop depends_on: - hdfs-namenode-1 - - hdfs-namenode-2 volumes: - "${HDFS_DATA_DIR_3}:/hadoop/dfs/data" restart: always @@ -92,9 +85,7 @@ services: SERVICE_9867_IGNORE: "true" SERVICE_9864_IGNORE: "true" HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 HADOOP_DFS_REPLICATION: 2 healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] @@ -112,133 +103,16 @@ services: hostname: hdfs-namenode-1 networks: - hadoop - - zookeeper - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - - hdfs-journalnode-1 - - hdfs-journalnode-2 - - hdfs-journalnode-3 volumes: - - "${HDFS_NAME_DIR_1}:/hadoop/dfs/name" + - "${HDFS_NAME_DIR_1}:/hadoop/dfs/name/1" + - "${HDFS_NAME_DIR_2}:/hadoop/dfs/name/2" restart: always environment: SERVICE_8020_NAME: namenode SERVICE_9870_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 - HADOOP_QJOURNAL_ADDRESS: hdfs-journalnode-1:8485;hdfs-journalnode-2:8485;hdfs-journalnode-3:8485 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-namenode-2: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - command: namenode-2 - hostname: hdfs-namenode-2 - networks: - - hadoop - - zookeeper - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - - hdfs-journalnode-1 - - hdfs-journalnode-2 - - hdfs-journalnode-3 - - hdfs-namenode-1 - volumes: - - "${HDFS_NAME_DIR_2}:/hadoop/dfs/name" - restart: always - environment: - SERVICE_8020_NAME: namenode - SERVICE_9870_IGNORE: "true" - HADOOP_ZOOKEEPER_QUORUM: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 - HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_NAMENODE2_HOSTNAME: hdfs-namenode-2 - HADOOP_DFS_REPLICATION: 2 - HADOOP_QJOURNAL_ADDRESS: hdfs-journalnode-1:8485;hdfs-journalnode-2:8485;hdfs-journalnode-3:8485 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-journalnode-1: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - command: journalnode - networks: - - hadoop - volumes: - - "${HDFS_JOURNAL_DIR_1}:/hadoop/dfs/journal" - restart: always - environment: - SERVICE_8485_NAME: journalnode - SERVICE_8480_IGNORE: "true" - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-journalnode-2: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - command: journalnode - networks: - - hadoop - volumes: - - "${HDFS_JOURNAL_DIR_2}:/hadoop/dfs/journal" - restart: always - environment: - SERVICE_8485_NAME: journalnode - SERVICE_8480_IGNORE: "true" - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-journalnode-3: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - command: journalnode - networks: - - hadoop - volumes: - - "${HDFS_JOURNAL_DIR_3}:/hadoop/dfs/journal" - restart: always - environment: - SERVICE_8485_NAME: journalnode - SERVICE_8480_IGNORE: "true" - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE_HA: nn1,nn2 + HADOOP_DFS_NAME_DIR: file:///hadoop/dfs/name/1,file:///hadoop/dfs/name/2 healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m @@ -252,13 +126,15 @@ services: image: radarbase/radar-hdfs-connector-auto:0.2.0 restart: on-failure volumes: - - ../etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties + - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties networks: - zookeeper - kafka - hadoop depends_on: - zookeeper-1 + - zookeeper-2 + - zookeeper-3 - kafka-1 - kafka-2 - kafka-3 @@ -268,7 +144,6 @@ services: - hdfs-datanode-2 - hdfs-datanode-3 - hdfs-namenode-1 - - hdfs-namenode-2 environment: CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 CONNECT_REST_PORT: 8083 @@ -288,7 +163,6 @@ services: CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" KAFKA_BROKERS: 3 - CONNECT_LOG4J_ROOT_LOGLEVEL: WARN CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" healthcheck: test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-high.properties b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-high.properties index 707ff0736..e940be5e6 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-high.properties +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-high.properties @@ -3,6 +3,6 @@ connector.class=io.confluent.connect.hdfs.HdfsSinkConnector tasks.max=4 topics=android_empatica_e4_blood_volume_pulse,android_empatica_e4_acceleration flush.size=120000 -hdfs.url=hdfs://hdfs-namenode:8020 +hdfs.url=hdfs://hdfs-namenode-1:8020 format.class=org.radarcns.sink.hdfs.AvroFormatRadar -topics.dir=topicAndroidNew \ No newline at end of file +topics.dir=/topicAndroidNew diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-low.properties b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-low.properties index ec81d423b..d10f6ac5c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-low.properties +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-low.properties @@ -3,6 +3,6 @@ connector.class=io.confluent.connect.hdfs.HdfsSinkConnector tasks.max=4 topics=android_empatica_e4_battery_level,android_empatica_e4_inter_beat_interval,android_empatica_e4_sensor_status flush.size=3000 -hdfs.url=hdfs://hdfs-namenode:8020 +hdfs.url=hdfs://hdfs-namenode-1:8020 format.class=org.radarcns.sink.hdfs.AvroFormatRadar -topics.dir=topicAndroidNew \ No newline at end of file +topics.dir=/topicAndroidNew diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-med.properties b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-med.properties index 0799fbfda..5d3389914 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-med.properties +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs-med.properties @@ -3,6 +3,6 @@ connector.class=io.confluent.connect.hdfs.HdfsSinkConnector tasks.max=4 topics=android_empatica_e4_electrodermal_activity,android_empatica_e4_temperature flush.size=12000 -hdfs.url=hdfs://hdfs-namenode:8020 +hdfs.url=hdfs://hdfs-namenode-1:8020 format.class=org.radarcns.sink.hdfs.AvroFormatRadar -topics.dir=topicAndroidEmpatica \ No newline at end of file +topics.dir=/topicAndroidNew diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile index f4a33fdc9..dc5382490 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/Dockerfile @@ -1,5 +1,7 @@ ARG BASE_VERSION=3.0.3-alpine FROM smizy/hadoop-base:${BASE_VERSION} -COPY ./hdfs-site.xml.mustache ${HDFS_CONF_DIR}/ +ENV HADOOP_DFS_NAME_DIR file://hadoop/dfs/name + +COPY ./hdfs-site.xml.mustache ${HADOOP_CONF_DIR}/ COPY ./entrypoint.sh /usr/local/bin/ diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh index 60a7d079f..21fed099f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh @@ -26,6 +26,27 @@ wait_until() { fi } +format_hdfs() { + NAME_DIR=$1 + shift + IFS=',' read -r -a namedirs <<< $(echo "$NAME_DIR" | sed -e 's#file://##g') + + for namedir in "${namedirs[@]}"; do + mkdir -p "$namedir" + if [ ! -d "$namedir" ]; then + echo "Namenode name directory not found: $namedir" + exit 2 + fi + + if [ ! -e "$namedir/current/VERSION" ]; then + echo "Formatting namenode name directory: $namedir is not yet formatted" + su-exec hdfs hdfs namenode $@ + return 0 + fi + done + return 1 +} + # apply template for template in $(ls ${HADOOP_CONF_DIR}/*.mustache) do @@ -48,11 +69,8 @@ case $CMD in exec su-exec hdfs hdfs journalnode "$@" ;; "namenode-1") - if [ ! -e "${HADOOP_TMP_DIR}/dfs/name/current/VERSION" ]; then - su-exec hdfs hdfs namenode -format -force - if [ "${HADOOP_NAMENODE_HA}" != "" ]; then - su-exec hdfs hdfs zkfc -formatZK -force - fi + if format_hdfs "$HADOOP_DFS_NAME_DIR" -format -force && [ "${HADOOP_NAMENODE_HA}" != "" ]; then + su-exec hdfs hdfs zkfc -formatZK -force fi # wait_until ${HADOOP_QJOURNAL_ADDRESS%%:*} 8485 if [ "${HADOOP_NAMENODE_HA}" != "" ]; then @@ -61,10 +79,11 @@ case $CMD in exec su-exec hdfs hdfs namenode "$@" ;; "namenode-2") - if [ ! -e "${HADOOP_TMP_DIR}/dfs/name/current/VERSION" ]; then - wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 - su-exec hdfs hdfs namenode -bootstrapStandby + wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 + if format_hdfs "$HADOOP_DFS_NAME_DIR" -bootstrapStandby && [ "${HADOOP_NAMENODE_HA}" != "" ]; then + su-exec hdfs hdfs zkfc -formatZK -force fi + su-exec hdfs hdfs zkfc & exec su-exec hdfs hdfs namenode "$@" ;; diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache index a6428defd..7726e7536 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache @@ -17,6 +17,10 @@ + + dfs.namenode.name.dir + {{HADOOP_DFS_NAME_DIR}} + dfs.permissions.superusergroup @@ -83,7 +87,6 @@ dfs.ha.automatic-failover.enabled true - {{/HADOOP_NAMENODE_HA}} {{^HADOOP_NAMENODE_HA}} @@ -99,12 +102,12 @@ {{/HADOOP_NAMENODE_HA}} - dfs.client.block.write.replace-datanode-on-failure.policy - NEVER + dfs.client.block.write.replace-datanode-on-failure.policy + NEVER - dfs.client.block.write.replace-datanode-on-failure.enable - false + dfs.client.block.write.replace-datanode-on-failure.enable + false diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 7fc1553e3..89a3e7ee5 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -112,7 +112,7 @@ sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' et init_certificate "${SERVER_NAME}" echo "==> Starting RADAR-base Platform" -sudo-linux bin/radar-docker up -d "$@" +sudo-linux bin/radar-docker up -d --remove-orphans "$@" request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" echo "### SUCCESS ###" From 656b735e7ad9b3ed777537ad2c269e871418dddd Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 15:35:53 +0200 Subject: [PATCH 062/194] Revert to single docker-compose file again --- .../radar-cp-hadoop-stack/bin/radar-docker | 2 +- .../docker-compose.hdfs.yml | 171 ------------------ .../radar-cp-hadoop-stack/docker-compose.yml | 166 +++++++++++++++++ 3 files changed, 167 insertions(+), 172 deletions(-) delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index b704fe11a..ef1f799e9 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -1,7 +1,7 @@ #!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")/.." -stack="docker-compose -f docker-compose.yml -f docker-compose.hdfs.yml" +stack="docker-compose" function print_usage() { echo 'Control the RADAR-base platform directly' diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml deleted file mode 100644 index 07fad11b8..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.hdfs.yml +++ /dev/null @@ -1,171 +0,0 @@ ---- -version: '2.4' - -networks: - hadoop: - external: true - -services: - #---------------------------------------------------------------------------# - # RADAR Cold Storage # - #---------------------------------------------------------------------------# - hdfs-datanode-1: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - hostname: hdfs-datanode-1 - command: datanode - networks: - - hadoop - depends_on: - - hdfs-namenode-1 - volumes: - - "${HDFS_DATA_DIR_1}:/hadoop/dfs/data" - restart: always - environment: - SERVICE_9866_NAME: datanode - SERVICE_9867_IGNORE: "true" - SERVICE_9864_IGNORE: "true" - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_DFS_REPLICATION: 2 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-datanode-2: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - command: datanode - hostname: hdfs-datanode-2 - networks: - - hadoop - depends_on: - - hdfs-namenode-1 - volumes: - - "${HDFS_DATA_DIR_2}:/hadoop/dfs/data" - restart: always - environment: - SERVICE_9866_NAME: datanode - SERVICE_9867_IGNORE: "true" - SERVICE_9864_IGNORE: "true" - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_DFS_REPLICATION: 2 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-datanode-3: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - command: datanode - hostname: hdfs-datanode-3 - networks: - - hadoop - depends_on: - - hdfs-namenode-1 - volumes: - - "${HDFS_DATA_DIR_3}:/hadoop/dfs/data" - restart: always - environment: - SERVICE_9866_NAME: datanode - SERVICE_9867_IGNORE: "true" - SERVICE_9864_IGNORE: "true" - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_DFS_REPLICATION: 2 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - hdfs-namenode-1: - build: - context: ./images/hdfs - args: - BASE_VERSION: ${HDFS_BASE_VERSION} - image: radarbase/hdfs:${HDFS_BASE_VERSION} - command: namenode-1 - hostname: hdfs-namenode-1 - networks: - - hadoop - volumes: - - "${HDFS_NAME_DIR_1}:/hadoop/dfs/name/1" - - "${HDFS_NAME_DIR_2}:/hadoop/dfs/name/2" - restart: always - environment: - SERVICE_8020_NAME: namenode - SERVICE_9870_IGNORE: "true" - HADOOP_HEAPSIZE: 1000 - HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 - HADOOP_DFS_NAME_DIR: file:///hadoop/dfs/name/1,file:///hadoop/dfs/name/2 - healthcheck: - test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] - interval: 1m - timeout: 10s - retries: 3 - - #---------------------------------------------------------------------------# - # RADAR HDFS connector # - #---------------------------------------------------------------------------# - radar-hdfs-connector: - image: radarbase/radar-hdfs-connector-auto:0.2.0 - restart: on-failure - volumes: - - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties - networks: - - zookeeper - - kafka - - hadoop - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - - kafka-1 - - kafka-2 - - kafka-3 - - schema-registry-1 - - kafka-init - - hdfs-datanode-1 - - hdfs-datanode-2 - - hdfs-datanode-3 - - hdfs-namenode-1 - environment: - CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 - CONNECT_REST_PORT: 8083 - CONNECT_GROUP_ID: "default" - CONNECT_CONFIG_STORAGE_TOPIC: "default.config" - CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" - CONNECT_STATUS_STORAGE_TOPIC: "default.status" - CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" - CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" - CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" - CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" - CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 - CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" - KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" - KAFKA_BROKERS: 3 - CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" - healthcheck: - test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] - interval: 1m - timeout: 5s - retries: 3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 13d92f3f7..a7ca69d88 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -23,6 +23,8 @@ networks: management: driver: bridge internal: true + hadoop: + external: true volumes: kafka-1-data: {} @@ -369,6 +371,170 @@ services: env_file: - ./etc/smtp.env + #---------------------------------------------------------------------------# + # RADAR Cold Storage # + #---------------------------------------------------------------------------# + hdfs-datanode-1: + build: + context: ./images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} + hostname: hdfs-datanode-1 + command: datanode + networks: + - hadoop + depends_on: + - hdfs-namenode-1 + volumes: + - "${HDFS_DATA_DIR_1}:/hadoop/dfs/data" + restart: always + environment: + SERVICE_9866_NAME: datanode + SERVICE_9867_IGNORE: "true" + SERVICE_9864_IGNORE: "true" + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_DFS_REPLICATION: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-datanode-2: + build: + context: ./images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} + command: datanode + hostname: hdfs-datanode-2 + networks: + - hadoop + depends_on: + - hdfs-namenode-1 + volumes: + - "${HDFS_DATA_DIR_2}:/hadoop/dfs/data" + restart: always + environment: + SERVICE_9866_NAME: datanode + SERVICE_9867_IGNORE: "true" + SERVICE_9864_IGNORE: "true" + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_DFS_REPLICATION: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-datanode-3: + build: + context: ./images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} + command: datanode + hostname: hdfs-datanode-3 + networks: + - hadoop + depends_on: + - hdfs-namenode-1 + volumes: + - "${HDFS_DATA_DIR_3}:/hadoop/dfs/data" + restart: always + environment: + SERVICE_9866_NAME: datanode + SERVICE_9867_IGNORE: "true" + SERVICE_9864_IGNORE: "true" + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_DFS_REPLICATION: 2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] + interval: 1m + timeout: 10s + retries: 3 + + hdfs-namenode-1: + build: + context: ./images/hdfs + args: + BASE_VERSION: ${HDFS_BASE_VERSION} + image: radarbase/hdfs:${HDFS_BASE_VERSION} + command: namenode-1 + hostname: hdfs-namenode-1 + networks: + - hadoop + volumes: + - "${HDFS_NAME_DIR_1}:/hadoop/dfs/name/1" + - "${HDFS_NAME_DIR_2}:/hadoop/dfs/name/2" + restart: always + environment: + SERVICE_8020_NAME: namenode + SERVICE_9870_IGNORE: "true" + HADOOP_HEAPSIZE: 1000 + HADOOP_NAMENODE1_HOSTNAME: hdfs-namenode-1 + HADOOP_DFS_NAME_DIR: file:///hadoop/dfs/name/1,file:///hadoop/dfs/name/2 + healthcheck: + test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] + interval: 1m + timeout: 10s + retries: 3 + + #---------------------------------------------------------------------------# + # RADAR HDFS connector # + #---------------------------------------------------------------------------# + radar-hdfs-connector: + image: radarbase/radar-hdfs-connector-auto:0.2.0 + restart: on-failure + volumes: + - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties + networks: + - zookeeper + - kafka + - hadoop + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + - kafka-init + - hdfs-datanode-1 + - hdfs-datanode-2 + - hdfs-datanode-3 + - hdfs-namenode-1 + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "default" + CONNECT_CONFIG_STORAGE_TOPIC: "default.config" + CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" + CONNECT_STATUS_STORAGE_TOPIC: "default.status" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" + healthcheck: + test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] + interval: 1m + timeout: 5s + retries: 3 + #---------------------------------------------------------------------------# # RADAR mongo connector # #---------------------------------------------------------------------------# From e060473246f9c517491586f69ff8235b8ac15606 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 15:42:07 +0200 Subject: [PATCH 063/194] Revert layout of dev --- .../radar-cp-hadoop-stack/docker-compose.yml | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index a7ca69d88..0c732f446 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -357,20 +357,6 @@ services: timeout: 5s retries: 3 - #---------------------------------------------------------------------------# - # Email server # - #---------------------------------------------------------------------------# - smtp: - image: namshi/smtp:latest - networks: - - mail - - default - volumes: - - /var/spool/exim - restart: always - env_file: - - ./etc/smtp.env - #---------------------------------------------------------------------------# # RADAR Cold Storage # #---------------------------------------------------------------------------# @@ -485,30 +471,40 @@ services: retries: 3 #---------------------------------------------------------------------------# - # RADAR HDFS connector # + # Email server # #---------------------------------------------------------------------------# - radar-hdfs-connector: - image: radarbase/radar-hdfs-connector-auto:0.2.0 + smtp: + image: namshi/smtp:latest + networks: + - mail + - default + volumes: + - /var/spool/exim + restart: always + env_file: + - ./etc/smtp.env + + #---------------------------------------------------------------------------# + # RADAR mongo connector # + #---------------------------------------------------------------------------# + radar-mongodb-connector: + image: radarbase/kafka-connect-mongodb-sink:0.2.2 restart: on-failure volumes: - - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties + - ./etc/mongodb-connector/sink-mongo.properties:/etc/kafka-connect/sink.properties networks: - zookeeper - kafka - - hadoop + - hotstorage depends_on: - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - kafka-1 - kafka-2 - kafka-3 - schema-registry-1 + - rest-proxy-1 - kafka-init - - hdfs-datanode-1 - - hdfs-datanode-2 - - hdfs-datanode-3 - - hdfs-namenode-1 + - hotstorage environment: CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 CONNECT_REST_PORT: 8083 @@ -523,39 +519,47 @@ services: CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" - CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-mongodb-connector" CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 - CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" - KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + CONNECT_CONSUMER_MAX_POLL_RECORDS: 500 + CONNECT_CONSUMER_MAX_POLL_INTERVAL_MS: 300000 + CONNECT_CONSUMER_SESSION_TIMEOUT_MS: 10000 + CONNECT_CONSUMER_HEARTBEAT_INTERVAL_MS: 3000 + CONNECT_PLUGIN_PATH: /usr/share/java/kafka-connect/plugins KAFKA_BROKERS: 3 + CONNECT_LOG4J_ROOT_LOGLEVEL: WARN CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" healthcheck: - test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] + test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-connector-mongodb-sink/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] interval: 1m timeout: 5s retries: 3 #---------------------------------------------------------------------------# - # RADAR mongo connector # + # RADAR HDFS connector # #---------------------------------------------------------------------------# - radar-mongodb-connector: - image: radarbase/kafka-connect-mongodb-sink:0.2.2 + radar-hdfs-connector: + image: radarbase/radar-hdfs-connector-auto:0.2.0 restart: on-failure volumes: - - ./etc/mongodb-connector/sink-mongo.properties:/etc/kafka-connect/sink.properties + - ./etc/hdfs-connector/sink-hdfs.properties:/etc/kafka-connect/sink-hdfs.properties networks: - zookeeper - kafka - - hotstorage + - hadoop depends_on: - zookeeper-1 + - zookeeper-2 + - zookeeper-3 - kafka-1 - kafka-2 - kafka-3 - schema-registry-1 - - rest-proxy-1 - kafka-init - - hotstorage + - hdfs-datanode-1 + - hdfs-datanode-2 + - hdfs-datanode-3 + - hdfs-namenode-1 environment: CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 CONNECT_REST_PORT: 8083 @@ -570,18 +574,14 @@ services: CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/tmp/connect2.offset" - CONNECT_REST_ADVERTISED_HOST_NAME: "radar-mongodb-connector" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-hdfs-connector" CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 - CONNECT_CONSUMER_MAX_POLL_RECORDS: 500 - CONNECT_CONSUMER_MAX_POLL_INTERVAL_MS: 300000 - CONNECT_CONSUMER_SESSION_TIMEOUT_MS: 10000 - CONNECT_CONSUMER_HEARTBEAT_INTERVAL_MS: 3000 - CONNECT_PLUGIN_PATH: /usr/share/java/kafka-connect/plugins + CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" KAFKA_BROKERS: 3 - CONNECT_LOG4J_ROOT_LOGLEVEL: WARN CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" healthcheck: - test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-connector-mongodb-sink/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] + test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-hdfs-sink-android-15000/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] interval: 1m timeout: 5s retries: 3 From 6b1ddbb6d922d7eb104b1564371b73ecdc903aa3 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 15:50:40 +0200 Subject: [PATCH 064/194] change description --- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index 00deac6d9..275704303 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -1,5 +1,5 @@ #!/bin/bash - +echo "Starting to configure mock configurations for test" # create hadoop netword sudo docker network create hadoop @@ -20,4 +20,4 @@ cp ../etc/mongodb-connector/sink-mongo.properties.template ../etc/mongodb-connec cp ../etc/managementportal/config/oauth_client_details.csv.template ../etc/managementportal/config/oauth_client_details.csv cp ../etc/redcap-integration/radar.yml.template ../etc/redcap-integration/radar.yml -echo "Setup mock configurations are finished..." \ No newline at end of file +echo "Setting up mock configurations finished..." \ No newline at end of file From 5c495804ab52c91473b0020fd28a7fe6cb7311ed Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 15:54:43 +0200 Subject: [PATCH 065/194] Initialize hdfs directory --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 513ac5186..901f5ba25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,7 @@ script: - cp etc/webserver/nginx.conf.template etc/webserver/nginx.conf - cp etc/hdfs-connector/sink-hdfs.properties.template etc/hdfs-connector/sink-hdfs.properties - cp etc/mongodb-connector/sink-mongo.properties.template etc/mongodb-connector/sink-mongo.properties + - echo "HDFS_BASE_VERSION=3.0.3-alpine" >> .env - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down From 6acf2e9bbc6ad80987ab3688a21e1b300b548e86 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 15:59:30 +0200 Subject: [PATCH 066/194] Use latest useful image --- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index ed2bb16ab..da617ac80 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -13,8 +13,7 @@ DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) sudo-linux "$DIR/bin/radar-docker" hdfs # HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:dev -docker pull $DOCKER_IMAGE +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.0 # HDFS filename to get HDFS_FILE=$1 @@ -27,7 +26,7 @@ HDFS_OUTPUT_DIR=/output HDFS_COMMAND=( --compression gzip --deduplicate - -n hdfs-namenode-1 + -u hdfs-namenode-1 -o "$HDFS_OUTPUT_DIR" "$HDFS_FILE" ) From caaa6ea10f39138f3dc9155c8707e40aff1e6e1e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 16:03:05 +0200 Subject: [PATCH 067/194] Reverted to released restructure script --- .../radar-cp-hadoop-stack/bin/hdfs-restructure | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index da617ac80..3c1fa5f32 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -5,28 +5,31 @@ if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then exit 1 fi -DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd) +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -. "$DIR/lib/util.sh" +# Absolute directory to write output to +OUTPUT_DIR=${2:-$DIR/output} +OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" + +cd $DIR + +. lib/util.sh # Start HDFS if not started already -sudo-linux "$DIR/bin/radar-docker" hdfs +sudo-linux bin/radar-docker hdfs # HDFS restructure version DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.0 # HDFS filename to get HDFS_FILE=$1 -# Absolute directory to write output to -OUTPUT_DIR=${2:-$DIR/output} -OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" # Internal docker directory to write output to HDFS_OUTPUT_DIR=/output # HDFS command to run HDFS_COMMAND=( --compression gzip --deduplicate - -u hdfs-namenode-1 + -u hdfs://hdfs-namenode-1:8020 -o "$HDFS_OUTPUT_DIR" "$HDFS_FILE" ) From fd86c2ad3a6055c0a99bfcfe3c6713755f03fa38 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 16:05:19 +0200 Subject: [PATCH 068/194] In Travis fix hdfs_data_3 reference --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 901f5ba25..534c2bc05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,8 +30,10 @@ script: - export SERVER_NAME=localhost - export HDFS_DATA_DIR_1=$PWD/hdfs-data1 - export HDFS_DATA_DIR_2=$PWD/hdfs-data2 + - export HDFS_DATA_DIR_3=$PWD/hdfs-data3 - export HDFS_NAME_DIR_1=$PWD/hdfs-name1 - export HDFS_NAME_DIR_2=$PWD/hdfs-name2 + - export HDFS_BASE_VERSION=3.0.3-alpine - echo $"SMARTHOST_ADDRESS=mail.example.com\nSMARTHOST_PORT=587\nSMARTHOST_USER=user@example.com\nSMARTHOST_PASSWORD=XXXXXXXX" > etc/smtp.env - sudo docker volume create certs - sudo docker volume create certs-data @@ -39,7 +41,6 @@ script: - cp etc/webserver/nginx.conf.template etc/webserver/nginx.conf - cp etc/hdfs-connector/sink-hdfs.properties.template etc/hdfs-connector/sink-hdfs.properties - cp etc/mongodb-connector/sink-mongo.properties.template etc/mongodb-connector/sink-mongo.properties - - echo "HDFS_BASE_VERSION=3.0.3-alpine" >> .env - sudo $HOME/bin/docker-compose up -d --build && sleep 15 && [ -z "$($HOME/bin/docker-compose ps | tail -n +3 | grep " Exit ")" ] - sudo $HOME/bin/docker-compose down From 65dbe69709d9d3f8b217716573ec38ba404449d0 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 16:13:18 +0200 Subject: [PATCH 069/194] Update README --- dcompose-stack/radar-cp-hadoop-stack/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 32bc096e4..53b9e0cac 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -86,6 +86,10 @@ sudo systemctl stop radar-docker sudo systemctl start radar-docker ``` +## Postgres Data Migration +If a major Postgres version upgrade is planned, existing data need to be migrated to the new version. To do so, +1. Change the value for `POSTGRES_NEW_VERSION` at `.env` +2. Run `./migrate-postgres-dats.sh` ## Data extraction If systemd integration is enabled, HDFS data will be extracted to the `./output` directory every hour. It can then be run directly by running From e4db3732f38c0f83b68dbc5113e9b3466f5e5fd7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 16:15:10 +0200 Subject: [PATCH 070/194] Increase offsets timeout (fixes #142) --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index ff2d7534e..62504a4d6 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -136,6 +136,7 @@ services: KAFKA_INTER_BROKER_PROTOCOL_VERSION: "1.1" KAFKA_LOG_MESSAGE_FORMAT_VERSION: "1.1" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + KAFKA_OFFSETS_RETENTION_MINUTES: 10080 healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/1 || exit 1"] interval: 1m30s @@ -166,6 +167,7 @@ services: KAFKA_INTER_BROKER_PROTOCOL_VERSION: "1.1" KAFKA_LOG_MESSAGE_FORMAT_VERSION: "1.1" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + KAFKA_OFFSETS_RETENTION_MINUTES: 10080 healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/2 || exit 1"] interval: 1m30s @@ -196,6 +198,7 @@ services: KAFKA_INTER_BROKER_PROTOCOL_VERSION: "1.1" KAFKA_LOG_MESSAGE_FORMAT_VERSION: "1.1" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + KAFKA_OFFSETS_RETENTION_MINUTES: 10080 healthcheck: test: ["CMD-SHELL", "echo dump | nc zookeeper-1 2181 | grep -q /brokers/ids/3 || exit 1"] interval: 1m30s From 3c31ab927b3604c338397c445db3212ee7884409 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 17 Jul 2018 16:29:07 +0200 Subject: [PATCH 071/194] remove docker network and volume creation from script --- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index 275704303..b4bd42a71 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -1,11 +1,5 @@ #!/bin/bash echo "Starting to configure mock configurations for test" -# create hadoop netword -sudo docker network create hadoop - -# create volumes -sudo docker volume create certs -sudo docker volume create certs-data # create folder for docker volumes sudo mkdir -p /usr/local/var/lib/docker/ From 3f6e5e01a829f0c6584f538bd0b5c9e0e7a48e3f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 16:32:04 +0200 Subject: [PATCH 072/194] Increase kafka schemas retention time --- dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh index b64450ecc..c90265770 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/install-radar-stack.sh @@ -54,6 +54,9 @@ fi # Initializing Kafka echo "==> Setting up topics" sudo-linux docker-compose run --rm kafka-init +KAFKA_SCHEMA_RETENTION_MS=${KAFKA_SCHEMA_RETENTION_MS:-5400000000} +KAFKA_SCHEMA_RETENTION_CMD='kafka-configs --zookeeper "${KAFKA_ZOOKEEPER_CONNECT}" --entity-type topics --entity-name _schemas --alter --add-config min.compaction.lag.ms='${KAFKA_SCHEMA_RETENTION_MS}',cleanup.policy=compact' +sudo-linux docker-compose exec kafka-1 bash -c "$KAFKA_SCHEMA_RETENTION_CMD" echo "==> Configuring MongoDB Connector" # Update sink-mongo.properties From b58e6f27fd21cc208cd51d6c39b025a24c59570e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 16:32:24 +0200 Subject: [PATCH 073/194] Upgrade scripts to alpine 3.7 --- dcompose-stack/radar-cp-hadoop-stack/util.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/util.sh b/dcompose-stack/radar-cp-hadoop-stack/util.sh index 0a534d696..d5d4e0a5c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/util.sh @@ -106,7 +106,7 @@ check_config_present() { self_signed_certificate() { SERVER_NAME=$1 echo "==> Generating self-signed certificate" - sudo-linux docker run -i --rm -v certs:/etc/openssl -v certs-data:/var/lib/openssl -v "${PWD}/lib/self-sign-certificate.sh:/self-sign-certificate.sh" alpine:3.5 \ + sudo-linux docker run -i --rm -v certs:/etc/openssl -v certs-data:/var/lib/openssl -v "${PWD}/lib/self-sign-certificate.sh:/self-sign-certificate.sh" alpine:3.7 \ /self-sign-certificate.sh "/etc/openssl/live/${SERVER_NAME}" } @@ -116,14 +116,14 @@ letsencrypt_certonly() { echo "==> Requesting Let's Encrypt SSL certificate for ${SERVER_NAME}" # start from a clean slate - sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "find /etc/openssl -name '${SERVER_NAME}*' -prune -exec rm -rf '{}' +" + sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.7 /bin/sh -c "find /etc/openssl -name '${SERVER_NAME}*' -prune -exec rm -rf '{}' +" CERTBOT_DOCKER_OPTS=(-i --rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) CERTBOT_OPTS=(--webroot --webroot-path=/data/letsencrypt --agree-tos -m "${MAINTAINER_EMAIL}" -d "${SERVER_NAME}" --non-interactive) sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" # mark the directory as letsencrypt dir - sudo-linux docker run -i --rm -v certs:/etc/openssl alpine:3.5 /bin/touch "${SSL_PATH}/.letsencrypt" + sudo-linux docker run -i --rm -v certs:/etc/openssl alpine:3.7 /bin/touch "${SSL_PATH}/.letsencrypt" } letsencrypt_renew() { @@ -137,7 +137,7 @@ letsencrypt_renew() { init_certificate() { SERVER_NAME=$1 SSL_PATH="/etc/openssl/live/${SERVER_NAME}" - if sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "[ ! -e '${SSL_PATH}/chain.pem' ]"; then + if sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.7 /bin/sh -c "[ ! -e '${SSL_PATH}/chain.pem' ]"; then self_signed_certificate "${SERVER_NAME}" fi } @@ -148,7 +148,7 @@ request_certificate() { SSL_PATH="/etc/openssl/live/${SERVER_NAME}" init_certificate "${SERVER_NAME}" - CURRENT_CERT=$(sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.5 /bin/sh -c "[ -e '${SSL_PATH}/.letsencrypt' ] && echo letsencrypt || echo self-signed") + CURRENT_CERT=$(sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.7 /bin/sh -c "[ -e '${SSL_PATH}/.letsencrypt' ] && echo letsencrypt || echo self-signed") if [ "${CURRENT_CERT}" = "letsencrypt" ]; then if [ "$3" != "force" ]; then From fe8603e1c2d2f0070d84bddf7c489512ae2ed770 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 17 Jul 2018 17:27:49 +0200 Subject: [PATCH 074/194] Update radar-docker --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index ef1f799e9..5d71ea222 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -5,7 +5,7 @@ stack="docker-compose" function print_usage() { echo 'Control the RADAR-base platform directly' - echo "Usage: $0 [COMMAND] ..." + echo "Usage: $0 [COMMAND|-h|--help] ..." echo echo 'Commands:' echo ' cert-renew Request or renew SSL certificate' @@ -21,6 +21,9 @@ function print_usage() { echo ' quit SERVICE... Stop and remove given services.' echo ' rebuild SERVICE... Rebuild and restart given services.' echo + echo 'Options:' + echo ' -h, --help Print this usage information.' + echo echo 'docker-compose commands and options are also accepted:' echo '======================================================' echo From e31bcb5eaef9f9337c51554ccd175760880cd47b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 18 Jul 2018 16:18:48 +0200 Subject: [PATCH 075/194] Added more sane defaults --- .../etc/radar-backend/radar.yml.template | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index 29bb78511..0832513b4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -19,10 +19,11 @@ broker: #Kafka internal parameters stream_properties: - auto_commit_interval_ms: 1000 max.request.size: 3500042 #Set message.max.bytes for kafka brokers higher than or equal to this value retries: 15 - session_timeout_ms: 20000 + session.timeout.ms: 20000 + commit.interval.ms: 300000 + request.timeout.ms: 10860000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority stream_priority: From a1925a67413210c6a66e2ce64cd12e9667eafa3e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 19 Jul 2018 12:02:06 +0200 Subject: [PATCH 076/194] Revert to older schemas to fix ManagementPortal See RADAR-Base/ManagementPortal#291 --- .../radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile index ce955c5e3..f5c7d331e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile @@ -25,7 +25,7 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.3.4 +ENV RADAR_SCHEMAS_VERSION=0.3.3 RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 From c653d32ce1dfe57e89ecd4b618be21eacf3dbe2e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 19 Jul 2018 12:04:06 +0200 Subject: [PATCH 077/194] Tweak command scripts --- dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune | 1 + dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune index 942b842bc..966e4b7ed 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune @@ -13,6 +13,7 @@ select yn in "Yes" "No"; do Yes ) sudo-linux docker system prune --filter "label!=certs" "$@"; sudo-linux rm -rf "$HDFS_DATA_DIR_1"; sudo-linux rm -rf "$HDFS_DATA_DIR_2"; + sudo-linux rm -rf "$HDFS_DATA_DIR_3"; sudo-linux rm -rf "$HDFS_NAME_DIR_1"; sudo-linux rm -rf "$HDFS_NAME_DIR_2"; sudo-linux rm -rf "$MONGODB_DIR"; diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index 383b3fb4c..5b7147fc9 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -52,7 +52,7 @@ install-systemd) . lib/install-systemd-wrappers.sh ;; rebuild) - exec $stack up -d --force-recreate --build -V "$@" + exec $stack up -d --force-recreate --build --no-deps -V "$@" ;; quit) $stack stop "$@" && \ From b3afc440c662d19970aa9a5f186f7d89c334e328 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 23 Jul 2018 14:40:47 +0200 Subject: [PATCH 078/194] Use hdfs restructure snapshot version --- .../radar-cp-hadoop-stack/bin/hdfs-restructure | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 3c1fa5f32..05486f143 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -19,7 +19,7 @@ cd $DIR sudo-linux bin/radar-docker hdfs # HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.4.0 +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.0-SNAPSHOT # HDFS filename to get HDFS_FILE=$1 @@ -29,9 +29,12 @@ HDFS_OUTPUT_DIR=/output HDFS_COMMAND=( --compression gzip --deduplicate - -u hdfs://hdfs-namenode-1:8020 + -n hdfs-namenode-1 -o "$HDFS_OUTPUT_DIR" + --num-threads 3 + --timer + --tmp-dir "$HDFS_OUTPUT_DIR/+tmp" "$HDFS_FILE" ) mkdir -p $OUTPUT_DIR -sudo-linux docker run -t --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" $DOCKER_IMAGE "${HDFS_COMMAND[@]}" +sudo-linux docker run -it --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" $DOCKER_IMAGE "${HDFS_COMMAND[@]}" From 1f02ef51407c5d8a1663ccc350e10ae9b65c7e9f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 26 Jul 2018 17:03:47 +0200 Subject: [PATCH 079/194] Use radar-hdfs-restructure version 0.5.0 --- .../bin/hdfs-restructure | 27 +++++++++++++------ .../bin/hdfs-restructure-process | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 05486f143..5d49df123 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -1,5 +1,22 @@ #!/bin/bash +# HDFS restructure version +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.0 +# HDFS restructure script flags +HDFS_RESTRUCTURE_OPTS=( + --compression gzip + --deduplicate + --num-threads 3 + --timer + ) +# HDFS restructure JVM flags +RADAR_HDFS_RESTRUCTURE_OPTS="$JAVA_OPTS -Xmx2g" + +# Without DOCKER_OPTS, run in interactive mode. +# From systemd or cron, override DOCKER_OPTS to remove +# interactive mode, e.g., $ DOCKER_OPTS="" bin/hdfs-restructure /mydir +DOCKER_OPTS=${DOCKER_OPTS:--i} + if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then printf "Usage:\n$0 []\nThe destination directory defaults to ./output\n" exit 1 @@ -18,23 +35,17 @@ cd $DIR # Start HDFS if not started already sudo-linux bin/radar-docker hdfs -# HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.0-SNAPSHOT - # HDFS filename to get HDFS_FILE=$1 # Internal docker directory to write output to HDFS_OUTPUT_DIR=/output # HDFS command to run HDFS_COMMAND=( - --compression gzip - --deduplicate + "${HDFS_RESTRUCTURE_OPTS[@]}" -n hdfs-namenode-1 -o "$HDFS_OUTPUT_DIR" - --num-threads 3 - --timer --tmp-dir "$HDFS_OUTPUT_DIR/+tmp" "$HDFS_FILE" ) mkdir -p $OUTPUT_DIR -sudo-linux docker run -it --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" $DOCKER_IMAGE "${HDFS_COMMAND[@]}" +sudo-linux docker run ${DOCKER_OPTS} -t --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" -e RADAR_HDFS_RESTRUCTURE_OPTS $DOCKER_IMAGE "${HDFS_COMMAND[@]}" diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process index 76e99171c..b6637186c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process @@ -19,7 +19,7 @@ hdfs_directory=/topicAndroidNew if [ ! -f $lockfile ]; then log_info "Creating lock ..." touch $lockfile - (./hdfs-restructure $hdfs_directory $storage_directory >> $logfile 2>&1) + (DOCKER_OPTS="" ./hdfs-restructure $hdfs_directory $storage_directory >> $logfile 2>&1) log_info "Removing lock ..." rm $lockfile else From c68ba5d3641e71a7c82fd3f686c5156b6579003a Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 30 Jul 2018 15:24:51 +0200 Subject: [PATCH 080/194] Proper profiler with hdfs-restructure --- .../radar-cp-hadoop-stack/bin/hdfs-restructure | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 5d49df123..f909e689f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -7,16 +7,22 @@ HDFS_RESTRUCTURE_OPTS=( --compression gzip --deduplicate --num-threads 3 - --timer ) # HDFS restructure JVM flags -RADAR_HDFS_RESTRUCTURE_OPTS="$JAVA_OPTS -Xmx2g" - +export RADAR_HDFS_RESTRUCTURE_OPTS="$JAVA_OPTS -Xmx4g" # Without DOCKER_OPTS, run in interactive mode. # From systemd or cron, override DOCKER_OPTS to remove # interactive mode, e.g., $ DOCKER_OPTS="" bin/hdfs-restructure /mydir DOCKER_OPTS=${DOCKER_OPTS:--i} +# For profiling, run e.g. jvisualvm and connect to localhost:$PROFILE_PORT: +# PROFILE_PORT=9101 bin/hdfs-restructure /myTopic +# Note that profiling the application makes it slower. +if [ ! -z $PROFILE_PORT ]; then + export RADAR_HDFS_RESTRUCTURE_OPTS="$RADAR_HDFS_RESTRUCTURE_OPTS -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${PROFILE_PORT} -Dcom.sun.management.jmxremote.rmi.port=${PROFILE_PORT} -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" + DOCKER_OPTS="$DOCKER_OPTS -p ${PROFILE_PORT}:${PROFILE_PORT}" +fi + if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then printf "Usage:\n$0 []\nThe destination directory defaults to ./output\n" exit 1 From 45e14218731708fd59cbd892d1fd6b80fee34c7c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 31 Jul 2018 09:33:15 +0200 Subject: [PATCH 081/194] Cleaned up code --- .../bin/hdfs-restructure | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index f909e689f..323c9b39c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -1,7 +1,7 @@ #!/bin/bash # HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.0 +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.1 # HDFS restructure script flags HDFS_RESTRUCTURE_OPTS=( --compression gzip @@ -12,14 +12,23 @@ HDFS_RESTRUCTURE_OPTS=( export RADAR_HDFS_RESTRUCTURE_OPTS="$JAVA_OPTS -Xmx4g" # Without DOCKER_OPTS, run in interactive mode. # From systemd or cron, override DOCKER_OPTS to remove -# interactive mode, e.g., $ DOCKER_OPTS="" bin/hdfs-restructure /mydir +# interactive mode, e.g., +# DOCKER_OPTS="" bin/hdfs-restructure /mydir DOCKER_OPTS=${DOCKER_OPTS:--i} -# For profiling, run e.g. jvisualvm and connect to localhost:$PROFILE_PORT: -# PROFILE_PORT=9101 bin/hdfs-restructure /myTopic +# For profiling, run e.g. jvisualvm and connect to localhost:$PROFILE_PORT +# after running: +# PROFILE_PORT=9101 bin/hdfs-restructure /mydir # Note that profiling the application makes it slower. if [ ! -z $PROFILE_PORT ]; then - export RADAR_HDFS_RESTRUCTURE_OPTS="$RADAR_HDFS_RESTRUCTURE_OPTS -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${PROFILE_PORT} -Dcom.sun.management.jmxremote.rmi.port=${PROFILE_PORT} -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" + export RADAR_HDFS_RESTRUCTURE_OPTS="$RADAR_HDFS_RESTRUCTURE_OPTS + -Djava.rmi.server.hostname=${PROFILE_HOST:-localhost} + -Dcom.sun.management.jmxremote + -Dcom.sun.management.jmxremote.port=${PROFILE_PORT} + -Dcom.sun.management.jmxremote.rmi.port=${PROFILE_PORT} + -Dcom.sun.management.jmxremote.local.only=false + -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false" DOCKER_OPTS="$DOCKER_OPTS -p ${PROFILE_PORT}:${PROFILE_PORT}" fi From 74ab40c884607a6abab649cb40cdb831a7422354 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 31 Jul 2018 13:43:09 +0100 Subject: [PATCH 082/194] Add security around admin tools --- .../radar-cp-hadoop-stack/README.md | 31 ++++++++++++------- .../radar-cp-hadoop-stack/docker-compose.yml | 6 +++- .../radar-cp-hadoop-stack/etc/env.template | 3 ++ .../etc/webserver/nginx.conf.template | 14 +++++++++ 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index af79534fc..473590c24 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -5,30 +5,38 @@ This docker-compose stack contains the full operational RADAR platform. Once con ## Configuration 1. First copy `etc/env.template` file to `./.env` and check and modify all its variables. - - - 1.1. To have a valid HTTPS connection for a public host, set `SELF_SIGNED_CERT=no`. You need to provide a public valid DNS name as `SERVER_NAME` for SSL certificate to work. IP addresses will not work. - + + + 1.1. To have a valid HTTPS connection for a public host, set `SELF_SIGNED_CERT=no`. You need to provide a public valid DNS name as `SERVER_NAME` for SSL certificate to work. IP addresses will not work. + 1.2. Set `MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET` to a secret to be used by the Management Portal frontend. - + 1.3. If you want to enable auto import of source types from the catalog server set the variable `MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT` to `true`. + 1.4. To better secure portainer create a hash for your desired password using the command + ```shell + htpasswd -nb -B admin | cut -d ":" -f 2 + ``` + and just updating the `PORTAINER_PASSWORD_HASH` variable in .env file. The default hash is set using password `admin` + 2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). 3. Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. -5. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. +5. Next copy the `etc/webserver/nginx.conf.template` to `etc/webserver/nginx.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. You can remove this restriction if not required. For easy configuration two example subnet/IPs are included in the template. + +6. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. -6. (Optional) Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, +7. (Optional) Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, - 6.1 Create multipe property files that have different `flush.size` for given topics. + 7.1 Create multipe property files that have different `flush.size` for given topics. Examples [sink-hdfs-high.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties) , [sink-hdfs-low.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties) - 6.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` environment variable to `radar-hdfs-connector` service in `docker-compose` file. + 7.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` environment variable to `radar-hdfs-connector` service in `docker-compose` file. - 6.3 Add created property files to the `radar-hdfs-connector` service in `docker-compose` with name abides to prefix-value mentioned in `CONNECTOR_PROPERTY_FILE_PREFIX` + 7.3 Add created property files to the `radar-hdfs-connector` service in `docker-compose` with name abides to prefix-value mentioned in `CONNECTOR_PROPERTY_FILE_PREFIX` ```ini radar-hdfs-connector: @@ -159,7 +167,7 @@ Otherwise, the following manual commands can be invoked. Add a cron job to run the `bin/radar-docker health` script periodically like - 1. Edit the crontab file for the current user by typing `$ crontab -e` -2. Add your job and time interval. For example, add the following for checking health every 5 mins - +2. Add your job and time interval. For example, add the following for checking health every 5 mins - ``` */5 * * * * /home/ubuntu/RADAR-Docker/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker health @@ -198,4 +206,3 @@ To add a script to `CRON` as `root`, run on the command-line `sudo crontab -e -u ``` For example, `*/2 * * * * /absolute/path/to/script-name.sh` will execute `script-name.sh` every `2` minutes. - diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 70eeaecde..f83be9a06 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -663,7 +663,8 @@ services: restart: always portainer: - image: portainer/portainer:1.11.1 + image: portainer/portainer:1.19.1 + command: --admin-password '${PORTAINER_PASSWORD_HASH}' networks: - monitor volumes: @@ -772,6 +773,9 @@ services: - kafka-3 environment: ZK_HOSTS: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + KAFKA_MANAGER_AUTH_ENABLED: "true" + KAFKA_MANAGER_USERNAME: ${KAFKA_MANAGER_USERNAME} + KAFKA_MANAGER_PASSWORD: ${KAFKA_MANAGER_PASSWORD} restart: always healthcheck: test: ["CMD-SHELL", "[ $$(wget -q -O - localhost:9000/kafkamanager/api/health) = healthy ] || exit 1"] diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 1af54776f..ec258b1aa 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -27,3 +27,6 @@ HDFS_JOURNAL_DIR_1=/usr/local/var/lib/docker/hdfs-journal-1 HDFS_JOURNAL_DIR_2=/usr/local/var/lib/docker/hdfs-journal-2 HDFS_JOURNAL_DIR_3=/usr/local/var/lib/docker/hdfs-journal-3 POSTGRES_VERSION=10.4-alpine +KAFKA_MANAGER_USERNAME=kafkamanager-user +KAFKA_MANAGER_PASSWORD=kafkamanager-pwd +PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index cc9a28043..137a3929d 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -87,11 +87,20 @@ http { proxy_set_header Host $host; } location /portainer/ { + # restrict access to only certain IPs + allow 193.60.0.0/14; # KCL + allow 91.206.81.2; # The Hyve + deny all; + proxy_pass http://portainer:9000/; proxy_http_version 1.1; proxy_set_header Connection ""; } location /portainer/api/websocket/ { + # restrict access to only certain IPs + allow 193.60.0.0/14; # KCL + allow 91.206.81.2; # The Hyve + deny all; proxy_pass http://portainer:9000/api/websocket/; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -116,6 +125,11 @@ http { proxy_set_header Host $host; } location /kafkamanager/{ + # restrict access to only certain IPs + allow 193.60.0.0/14; # KCL + allow 91.206.81.2; # The Hyve + deny all; + proxy_pass http://kafka-manager:9000; proxy_set_header Host $host; } From a1eb5903c9d11f852c8841c8248de06379cef5b7 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 31 Jul 2018 13:46:00 +0100 Subject: [PATCH 083/194] update docs --- dcompose-stack/radar-cp-hadoop-stack/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 473590c24..a3aad4b57 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -17,7 +17,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con ```shell htpasswd -nb -B admin | cut -d ":" -f 2 ``` - and just updating the `PORTAINER_PASSWORD_HASH` variable in .env file. The default hash is set using password `admin` + then just update the `PORTAINER_PASSWORD_HASH` variable in .env file with the hash value generated above. The default hash is set in the template was generated using the password `admin` 2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). @@ -25,13 +25,13 @@ This docker-compose stack contains the full operational RADAR platform. Once con 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. -5. Next copy the `etc/webserver/nginx.conf.template` to `etc/webserver/nginx.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. You can remove this restriction if not required. For easy configuration two example subnet/IPs are included in the template. +5. Next copy the `etc/webserver/nginx.conf.template` to `etc/webserver/nginx.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two example subnet/IPs are included in the template. You can remove this restriction if not required. 6. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. 7. (Optional) Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, - 7.1 Create multipe property files that have different `flush.size` for given topics. + 7.1 Create multiple property files that have different `flush.size` for given topics. Examples [sink-hdfs-high.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties) , [sink-hdfs-low.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties) 7.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` environment variable to `radar-hdfs-connector` service in `docker-compose` file. From 1cc6ea9927b07c7f7b9b4eb4b5a70394fa79ec4f Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 31 Jul 2018 14:08:00 +0100 Subject: [PATCH 084/194] better ip filtering --- dcompose-stack/radar-cp-hadoop-stack/README.md | 2 +- .../radar-cp-hadoop-stack/docker-compose.yml | 1 + .../etc/webserver/ip-filter.whitelist | 4 ++++ .../etc/webserver/nginx.conf.template | 15 +++------------ 4 files changed, 9 insertions(+), 13 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index a3aad4b57..b139c9ca4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -25,7 +25,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. -5. Next copy the `etc/webserver/nginx.conf.template` to `etc/webserver/nginx.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two example subnet/IPs are included in the template. You can remove this restriction if not required. +5. Next edit the `etc/webserver/ip-filter.whitelist` file and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two example subnet/IPs are included. You can remove this restriction if not required. 6. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index f83be9a06..2310d81bb 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -699,6 +699,7 @@ services: - certs-data:/data/letsencrypt - "./etc/webserver/nginx.conf:/etc/nginx/nginx.conf:ro" - "./etc/webserver/cors.conf:/etc/nginx/cors.conf:ro" + - "./etc/webserver/ip-filter.whitelist:/etc/nginx/ip-filter.whitelist:ro" # healthcheck hard to do, however, it is possible to monitor this externally # with # docker logs --since 2m radarcphadoopstack_webserver_1 | grep "connect() failed" diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist new file mode 100644 index 000000000..39accb4a2 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist @@ -0,0 +1,4 @@ +# restrict access to only certain IPs +allow 193.60.0.0/14; # KCL +allow 91.206.81.2; # The Hyve +deny all; diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 137a3929d..df7aed930 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -87,20 +87,14 @@ http { proxy_set_header Host $host; } location /portainer/ { - # restrict access to only certain IPs - allow 193.60.0.0/14; # KCL - allow 91.206.81.2; # The Hyve - deny all; + include ip-filter.whitelist; proxy_pass http://portainer:9000/; proxy_http_version 1.1; proxy_set_header Connection ""; } location /portainer/api/websocket/ { - # restrict access to only certain IPs - allow 193.60.0.0/14; # KCL - allow 91.206.81.2; # The Hyve - deny all; + include ip-filter.whitelist; proxy_pass http://portainer:9000/api/websocket/; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -125,10 +119,7 @@ http { proxy_set_header Host $host; } location /kafkamanager/{ - # restrict access to only certain IPs - allow 193.60.0.0/14; # KCL - allow 91.206.81.2; # The Hyve - deny all; + include ip-filter.whitelist; proxy_pass http://kafka-manager:9000; proxy_set_header Host $host; From 362339d893f04c7761e890c1ae829dc3a35e80e0 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Tue, 31 Jul 2018 14:29:25 +0100 Subject: [PATCH 085/194] update travis files --- dcompose-stack/radar-cp-hadoop-stack/README.md | 2 +- .../etc/radar-backend/radar.yml.template | 5 +++-- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 2 +- .../radar-cp-hadoop-stack/travis/travis-env.template | 3 +++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index b139c9ca4..a9ddfbf40 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -17,7 +17,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con ```shell htpasswd -nb -B admin | cut -d ":" -f 2 ``` - then just update the `PORTAINER_PASSWORD_HASH` variable in .env file with the hash value generated above. The default hash is set in the template was generated using the password `admin` + then just update the `PORTAINER_PASSWORD_HASH` variable in .env file with the hash value generated above. The default hash set in the template was generated using the password `admin` 2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index 0832513b4..5a116cf4d 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -21,9 +21,10 @@ broker: stream_properties: max.request.size: 3500042 #Set message.max.bytes for kafka brokers higher than or equal to this value retries: 15 - session.timeout.ms: 20000 + session.timeout.ms: 500000 commit.interval.ms: 300000 request.timeout.ms: 10860000 + state.cleanup.delay.ms: 10860000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority stream_priority: @@ -75,7 +76,7 @@ disconnect_monitor: - android_empatica_e4_temperature timeout: 1800 # seconds after which a stream is set disconnected alert_repetitions: 2 # number of additional emails to send after the first - + #====================== Source statistics monitor ======================# statistics_monitors: - name: Empatica E4 diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index b4bd42a71..9c9ba065e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -14,4 +14,4 @@ cp ../etc/mongodb-connector/sink-mongo.properties.template ../etc/mongodb-connec cp ../etc/managementportal/config/oauth_client_details.csv.template ../etc/managementportal/config/oauth_client_details.csv cp ../etc/redcap-integration/radar.yml.template ../etc/redcap-integration/radar.yml -echo "Setting up mock configurations finished..." \ No newline at end of file +echo "Setting up mock configurations finished..." diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index e3386b8b6..123fa4837 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -27,3 +27,6 @@ HDFS_JOURNAL_DIR_1=/usr/local/var/lib/docker/hdfs-journal-1 HDFS_JOURNAL_DIR_2=/usr/local/var/lib/docker/hdfs-journal-2 HDFS_JOURNAL_DIR_3=/usr/local/var/lib/docker/hdfs-journal-3 POSTGRES_VERSION=10.4-alpine +KAFKA_MANAGER_USERNAME=kafkamanager-user +KAFKA_MANAGER_PASSWORD=kafkamanager-pwd +PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 From 66f3412e66d867898d90c8a87fb2635cfeb453a3 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 1 Aug 2018 12:03:49 +0100 Subject: [PATCH 086/194] update ip access control --- dcompose-stack/radar-cp-hadoop-stack/README.md | 2 +- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 8 ++++---- .../etc/webserver/ip-access-control.conf.template | 5 +++++ .../etc/webserver/ip-filter.whitelist | 4 ---- .../etc/webserver/nginx.conf.template | 6 +++--- .../radar-cp-hadoop-stack/lib/perform-install.sh | 1 + 7 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-access-control.conf.template delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index a9ddfbf40..d1c98fe3c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -25,7 +25,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. -5. Next edit the `etc/webserver/ip-filter.whitelist` file and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two example subnet/IPs are included. You can remove this restriction if not required. +5. (Optional) Next mv the `etc/webserver/ip-access-control.conf.template` to `etc/webserver/ip-access-control.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two examples are included in the comments. By default all IPs are allowed. 6. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 2310d81bb..f920b7deb 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -699,7 +699,7 @@ services: - certs-data:/data/letsencrypt - "./etc/webserver/nginx.conf:/etc/nginx/nginx.conf:ro" - "./etc/webserver/cors.conf:/etc/nginx/cors.conf:ro" - - "./etc/webserver/ip-filter.whitelist:/etc/nginx/ip-filter.whitelist:ro" + - "./etc/webserver/ip-access-control.conf:/etc/nginx/ip-access-control.conf:ro" # healthcheck hard to do, however, it is possible to monitor this externally # with # docker logs --since 2m radarcphadoopstack_webserver_1 | grep "connect() failed" diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index ec258b1aa..04ba3f3f9 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -6,12 +6,12 @@ MAINTAINER_EMAIL=me@example.com RADAR_RAW_TOPIC_LIST= RADAR_AGG_TOPIC_LIST= HOTSTORAGE_USERNAME=mongodb-user -HOTSTORAGE_PASSWORD=XXXXXXXX +HOTSTORAGE_PASSWORD= HOTSTORAGE_NAME=mongodb-database MONGODB_DIR=/usr/local/var/lib/docker/mongodb MP_POSTGRES_DIR=/usr/local/var/lib/docker/postgres POSTGRES_USER=postgresdb-user -POSTGRES_PASSWORD=postgresdb-pwd +POSTGRES_PASSWORD= FROM_EMAIL=from-email MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT=false HEALTHCHECK_HIPCHAT_NOTIFY=no @@ -28,5 +28,5 @@ HDFS_JOURNAL_DIR_2=/usr/local/var/lib/docker/hdfs-journal-2 HDFS_JOURNAL_DIR_3=/usr/local/var/lib/docker/hdfs-journal-3 POSTGRES_VERSION=10.4-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user -KAFKA_MANAGER_PASSWORD=kafkamanager-pwd -PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 +KAFKA_MANAGER_PASSWORD= +PORTAINER_PASSWORD_HASH= diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-access-control.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-access-control.conf.template new file mode 100644 index 000000000..5ac1bebe6 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-access-control.conf.template @@ -0,0 +1,5 @@ +allow all; +# restrict access to only certain IPs +# allow 1.2.3.0/14; +# allow 10.20.30.40; +# deny all; diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist deleted file mode 100644 index 39accb4a2..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/ip-filter.whitelist +++ /dev/null @@ -1,4 +0,0 @@ -# restrict access to only certain IPs -allow 193.60.0.0/14; # KCL -allow 91.206.81.2; # The Hyve -deny all; diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index df7aed930..9b59f2ee8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -87,14 +87,14 @@ http { proxy_set_header Host $host; } location /portainer/ { - include ip-filter.whitelist; + include ip-access-control.conf; proxy_pass http://portainer:9000/; proxy_http_version 1.1; proxy_set_header Connection ""; } location /portainer/api/websocket/ { - include ip-filter.whitelist; + include ip-access-control.conf; proxy_pass http://portainer:9000/api/websocket/; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -119,7 +119,7 @@ http { proxy_set_header Host $host; } location /kafkamanager/{ - include ip-filter.whitelist; + include ip-access-control.conf; proxy_pass http://kafka-manager:9000; proxy_set_header Host $host; diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index e9e02b54a..cd06c01b1 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -21,6 +21,7 @@ copy_template_if_absent etc/mongodb-connector/sink-mongo.properties copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml copy_template_if_absent etc/webserver/nginx.conf +copy_template_if_absent etc/webserver/ip-access-control.conf . ./.env From a83c4a555a52dbc8947a819eaae4a279594a0df9 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 1 Aug 2018 22:38:05 +0100 Subject: [PATCH 087/194] Query for passwords if not set --- .../lib/perform-install.sh | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index cd06c01b1..23c7411fd 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -38,6 +38,32 @@ if [ -z ${SERVER_NAME} ]; then exit 1 fi +# Checking provided passwords +if [[ -z ${HOTSTORAGE_PASSWORD} ]]; then + echo "No Hotstorage Password specified in the .env file. Please enter the password now: " + read hotstorage_pass + inline_variable 'HOTSTORAGE_PASSWORD=' "${hotstorage_pass}" .env +fi + +if [[ -z ${POSTGRES_PASSWORD} ]]; then + echo "No Postgres Password specified in the .env file. Please enter the password now: " + read postgres_pass + inline_variable 'POSTGRES_PASSWORD=' "${postgres_pass}" .env +fi + +if [[ -z ${KAFKA_MANAGER_PASSWORD} ]]; then + echo "No Kafka Manager Password specified in the .env file. Please enter the password now: " + read kafkamanager_pass + inline_variable 'KAFKA_MANAGER_PASSWORD=' "${kafkamanager_pass}" .env +fi + +if [[ -z ${PORTAINER_PASSWORD_HASH} ]]; then + echo "No Portainer Password specified in the .env file. Please enter the password now: " + read portainer_pass + portainer_pass_hash=`sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB admin ${portainer_pass} | cut -d ":" -f 2` + inline_variable 'PORTAINER_PASSWORD_HASH=' "${portainer_pass_hash}" .env +fi + # Create networks and volumes if ! sudo-linux docker network ls --format '{{.Name}}' | grep -q "^hadoop$"; then echo "==> Creating docker network - hadoop" From 641ac255a7c43c4cb71b66f75d1c6f79b70af63f Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 1 Aug 2018 22:54:39 +0100 Subject: [PATCH 088/194] remove cadvisor --- .../radar-cp-hadoop-stack/docker-compose.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index f920b7deb..a647325a3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -648,20 +648,6 @@ services: #---------------------------------------------------------------------------# # Docker Monitoring # #---------------------------------------------------------------------------# - cadvisor: - image: google/cadvisor:v0.24.1 - networks: - - monitor - - default - ports: - - "8080:8080" - volumes: - - "/:/rootfs:ro" - - "/var/run:/var/run:rw" - - "/sys:/sys:ro" - - "/var/lib/docker/:/var/lib/docker:ro" - restart: always - portainer: image: portainer/portainer:1.19.1 command: --admin-password '${PORTAINER_PASSWORD_HASH}' From 51b136127f9dc25a5ee58db765b909aebdee55b0 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Wed, 1 Aug 2018 23:02:35 +0100 Subject: [PATCH 089/194] update doc --- dcompose-stack/radar-cp-hadoop-stack/README.md | 2 +- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index d1c98fe3c..f8889c248 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -17,7 +17,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con ```shell htpasswd -nb -B admin | cut -d ":" -f 2 ``` - then just update the `PORTAINER_PASSWORD_HASH` variable in .env file with the hash value generated above. The default hash set in the template was generated using the password `admin` + then just update the `PORTAINER_PASSWORD_HASH` variable in .env file with the hash value generated above. 2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index a647325a3..8625ae028 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -668,7 +668,6 @@ services: - monitor - default depends_on: - - cadvisor - portainer - rest-api - schema-registry-1 From 42e8dbcf78e8614fe988cc4fa9389b5d571432f0 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 2 Aug 2018 09:49:23 +0100 Subject: [PATCH 090/194] remove stale location --- .../radar-cp-hadoop-stack/etc/webserver/nginx.conf.template | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 9b59f2ee8..02ff9953a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -109,10 +109,6 @@ http { proxy_pass http://radar-integration:8080/redcap/; proxy_set_header Host $host; } - location /frontend/config/ { - proxy_pass http://rest-api:8080/radar/frontend/; - proxy_set_header Host $host; - } location /managementportal/ { include cors.conf; proxy_pass http://managementportal-app:8080/managementportal/; From ad91783c4dac70988a60a3e272af2bff2f8ff836 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Thu, 2 Aug 2018 11:10:20 +0100 Subject: [PATCH 091/194] Changes based on PR comments --- dcompose-stack/radar-cp-hadoop-stack/README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index f8889c248..6a781d56d 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -13,11 +13,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con 1.3. If you want to enable auto import of source types from the catalog server set the variable `MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT` to `true`. - 1.4. To better secure portainer create a hash for your desired password using the command - ```shell - htpasswd -nb -B admin | cut -d ":" -f 2 - ``` - then just update the `PORTAINER_PASSWORD_HASH` variable in .env file with the hash value generated above. + 1.4. Leave the `PORTAINER_PASSWORD_HASH` variable in .env file empty and run the install script (`bin/radar-docker install`). This should query for a new password and set its hash in this variable. To update the password, just empty the variable again and run the install script. 2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). @@ -25,7 +21,7 @@ This docker-compose stack contains the full operational RADAR platform. Once con 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. -5. (Optional) Next mv the `etc/webserver/ip-access-control.conf.template` to `etc/webserver/ip-access-control.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two examples are included in the comments. By default all IPs are allowed. +5. (Optional) Next copy the `etc/webserver/ip-access-control.conf.template` to `etc/webserver/ip-access-control.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two examples are included in the comments. By default all IPs are allowed. 6. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. From c86751e25ee29b707a9946ca753d7b22e45781af Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 7 Aug 2018 13:20:51 +0200 Subject: [PATCH 092/194] Fix restructure for empty DOCKER_OPTS --- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 323c9b39c..a9be0e739 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -14,7 +14,7 @@ export RADAR_HDFS_RESTRUCTURE_OPTS="$JAVA_OPTS -Xmx4g" # From systemd or cron, override DOCKER_OPTS to remove # interactive mode, e.g., # DOCKER_OPTS="" bin/hdfs-restructure /mydir -DOCKER_OPTS=${DOCKER_OPTS:--i} +DOCKER_OPTS=${DOCKER_OPTS--i} # For profiling, run e.g. jvisualvm and connect to localhost:$PROFILE_PORT # after running: From 23b38d9a1038692ef1ce3a244c72ac63a3c7c284 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 7 Aug 2018 13:23:50 +0200 Subject: [PATCH 093/194] Fix linux command-line options and hadoop permissions --- .../bin/hdfs-restructure-process | 2 +- .../radar-cp-hadoop-stack/bin/hdfs-upgrade | 4 +-- .../images/hdfs/entrypoint.sh | 34 +++++++++---------- .../images/hdfs/hdfs-site.xml.mustache | 5 +++ .../lib/install-systemd-wrappers.sh | 24 +++++++------ .../lib/perform-install.sh | 4 ++- .../radar-cp-hadoop-stack/lib/util.sh | 9 ++--- 7 files changed, 46 insertions(+), 36 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process index 76e99171c..54cc81f68 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process @@ -19,7 +19,7 @@ hdfs_directory=/topicAndroidNew if [ ! -f $lockfile ]; then log_info "Creating lock ..." touch $lockfile - (./hdfs-restructure $hdfs_directory $storage_directory >> $logfile 2>&1) + (./bin/hdfs-restructure $hdfs_directory $storage_directory >> $logfile 2>&1) log_info "Removing lock ..." rm $lockfile else diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade index cf53269ea..105f10fcc 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade @@ -39,10 +39,10 @@ $stack build echo "==> Upgrading name node 1" $stack run --rm --name hdfs-namenode-1 hdfs-namenode-1 namenode-1 -upgradeOnly $stack up -d hdfs-namenode-1 -sleep 5 +sleep 30 echo "==> Finalizing upgrade" -$stack exec hdfs-namenode-1 su-exec hdfs hdfs dfsadmin -finalizeUpgrade +$stack exec hdfs-namenode-1 hdfs dfsadmin -finalizeUpgrade echo "==> Bringing up HDFS cluster" $stack hdfs diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh index 21fed099f..3816ca9fc 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/entrypoint.sh @@ -40,7 +40,7 @@ format_hdfs() { if [ ! -e "$namedir/current/VERSION" ]; then echo "Formatting namenode name directory: $namedir is not yet formatted" - su-exec hdfs hdfs namenode $@ + hdfs namenode $@ return 0 fi done @@ -66,30 +66,30 @@ shift case $CMD in "journalnode") - exec su-exec hdfs hdfs journalnode "$@" + exec hdfs journalnode "$@" ;; "namenode-1") if format_hdfs "$HADOOP_DFS_NAME_DIR" -format -force && [ "${HADOOP_NAMENODE_HA}" != "" ]; then - su-exec hdfs hdfs zkfc -formatZK -force + hdfs zkfc -formatZK -force fi # wait_until ${HADOOP_QJOURNAL_ADDRESS%%:*} 8485 if [ "${HADOOP_NAMENODE_HA}" != "" ]; then - su-exec hdfs hdfs zkfc & + hdfs zkfc & fi - exec su-exec hdfs hdfs namenode "$@" + exec hdfs namenode "$@" ;; "namenode-2") wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 if format_hdfs "$HADOOP_DFS_NAME_DIR" -bootstrapStandby && [ "${HADOOP_NAMENODE_HA}" != "" ]; then - su-exec hdfs hdfs zkfc -formatZK -force + hdfs zkfc -formatZK -force fi - su-exec hdfs hdfs zkfc & - exec su-exec hdfs hdfs namenode "$@" + hdfs zkfc & + exec hdfs namenode "$@" ;; "datanode") wait_until ${HADOOP_NAMENODE1_HOSTNAME} 8020 - exec su-exec hdfs hdfs datanode "$@" + exec hdfs datanode "$@" ;; "resourcemanager-1") exec su-exec yarn yarn resourcemanager "$@" @@ -103,26 +103,26 @@ case $CMD in set +e -x - su-exec hdfs hdfs dfs -ls /tmp > /dev/null 2>&1 + hdfs dfs -ls /tmp > /dev/null 2>&1 if [ $? -ne 0 ]; then - su-exec hdfs hdfs dfs -mkdir -p /tmp - su-exec hdfs hdfs dfs -chmod 1777 /tmp + hdfs dfs -mkdir -p /tmp + hdfs dfs -chmod 1777 /tmp fi - su-exec hdfs hdfs dfs -ls /user > /dev/null 2>&1 + hdfs dfs -ls /user > /dev/null 2>&1 if [ $? -ne 0 ]; then - su-exec hdfs hdfs dfs -mkdir -p /user/hdfs - su-exec hdfs hdfs dfs -chmod 755 /user + hdfs dfs -mkdir -p /user/hdfs + hdfs dfs -chmod 755 /user fi - su-exec hdfs hdfs dfs -ls ${YARN_REMOTE_APP_LOG_DIR} > /dev/null 2>&1 + hdfs dfs -ls ${YARN_REMOTE_APP_LOG_DIR} > /dev/null 2>&1 if [ $? -ne 0 ]; then su-exec yarn hdfs dfs -mkdir -p ${YARN_REMOTE_APP_LOG_DIR} su-exec yarn hdfs dfs -chmod -R 1777 ${YARN_REMOTE_APP_LOG_DIR} su-exec yarn hdfs dfs -chown -R yarn:hadoop ${YARN_REMOTE_APP_LOG_DIR} fi - su-exec hdfs hdfs dfs -ls ${YARN_APP_MAPRED_STAGING_DIR} > /dev/null 2>&1 + hdfs dfs -ls ${YARN_APP_MAPRED_STAGING_DIR} > /dev/null 2>&1 if [ $? -ne 0 ]; then su-exec mapred hdfs dfs -mkdir -p ${YARN_APP_MAPRED_STAGING_DIR} su-exec mapred hdfs dfs -chmod -R 1777 ${YARN_APP_MAPRED_STAGING_DIR} diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache index 7726e7536..22d55a1b1 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache +++ b/dcompose-stack/radar-cp-hadoop-stack/images/hdfs/hdfs-site.xml.mustache @@ -110,4 +110,9 @@ dfs.client.block.write.replace-datanode-on-failure.enable false + + + dfs.namenode.datanode.registration.ip-hostname-check + false + diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index 55530c5bc..9b8e113ca 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -1,9 +1,9 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." +echo $(pwd) . lib/util.sh -check_command_exists systemctl - +echo "==> Copying templates" copy_template_if_absent /etc/systemd/system/radar-docker.service lib/systemd/radar-docker.service.template copy_template_if_absent /etc/systemd/system/radar-output.service lib/systemd/radar-output.service.template copy_template_if_absent /etc/systemd/system/radar-output.timer lib/systemd/radar-output.timer.template @@ -12,6 +12,7 @@ copy_template_if_absent /etc/systemd/system/radar-check-health.timer lib/systemd copy_template_if_absent /etc/systemd/system/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template copy_template_if_absent /etc/systemd/system/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template +echo "==> Inlining variables" inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-docker.service inline_variable 'ExecStart=' "$PWD/bin/radar-docker foreground" /etc/systemd/system/radar-docker.service @@ -24,12 +25,13 @@ inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" /etc/systemd/system/ inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-renew-certificate.service inline_variable 'ExecStart=' "$PWD/bin/radar-docker cert-renew" /etc/systemd/system/radar-renew-certificate.service -sudo systemctl daemon-reload -sudo systemctl enable radar-docker -sudo systemctl enable radar-output.timer -sudo systemctl enable radar-check-health.timer -sudo systemctl enable radar-renew-certificate.timer -sudo systemctl start radar-docker -sudo systemctl start radar-output.timer -sudo systemctl start radar-check-health.timer -sudo systemctl start radar-renew-certificate.timer +echo "==> Reloading systemd" +systemctl daemon-reload +systemctl enable radar-docker +systemctl enable radar-output.timer +systemctl enable radar-check-health.timer +systemctl enable radar-renew-certificate.timer +systemctl start radar-docker +systemctl start radar-output.timer +systemctl start radar-check-health.timer +systemctl start radar-renew-certificate.timer diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 23c7411fd..1572a2ece 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -82,10 +82,12 @@ fi # Initializing Kafka echo "==> Setting up topics" +sudo-linux bin/radar-docker up -d zookeeper-1 zookeeper-2 zookeeper-3 kafka-1 kafka-2 kafka-3 schema-registry-1 +sleep 60 sudo-linux bin/radar-docker run --rm kafka-init KAFKA_SCHEMA_RETENTION_MS=${KAFKA_SCHEMA_RETENTION_MS:-5400000000} KAFKA_SCHEMA_RETENTION_CMD='kafka-configs --zookeeper "${KAFKA_ZOOKEEPER_CONNECT}" --entity-type topics --entity-name _schemas --alter --add-config min.compaction.lag.ms='${KAFKA_SCHEMA_RETENTION_MS}',cleanup.policy=compact' -sudo-linux bin/radar-docker exec kafka-1 bash -c "$KAFKA_SCHEMA_RETENTION_CMD" +sudo-linux bin/radar-docker exec -T kafka-1 bash -c "$KAFKA_SCHEMA_RETENTION_CMD" echo "==> Configuring MongoDB Connector" # Update sink-mongo.properties diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh index f4e7d6038..b4426d8ea 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh @@ -78,6 +78,7 @@ copy_template_if_absent() { echo "Configuration file ${1} is not a regular file." exit 1 else + echo "Copying configuration file ${1} from template ${template}" sudo-linux cp -p "${template}" "$1" fi elif [ "$1" -ot "${template}" ]; then @@ -108,7 +109,7 @@ check_config_present() { self_signed_certificate() { SERVER_NAME=$1 echo "==> Generating self-signed certificate" - sudo-linux docker run -i --rm -v certs:/etc/openssl -v certs-data:/var/lib/openssl -v "${PWD}/lib/self-sign-certificate.sh:/self-sign-certificate.sh" alpine:3.7 \ + sudo-linux docker run --rm -v certs:/etc/openssl -v certs-data:/var/lib/openssl -v "${PWD}/lib/self-sign-certificate.sh:/self-sign-certificate.sh" alpine:3.7 \ /self-sign-certificate.sh "/etc/openssl/live/${SERVER_NAME}" } @@ -120,18 +121,18 @@ letsencrypt_certonly() { # start from a clean slate sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.7 /bin/sh -c "find /etc/openssl -name '${SERVER_NAME}*' -prune -exec rm -rf '{}' +" - CERTBOT_DOCKER_OPTS=(-i --rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) + CERTBOT_DOCKER_OPTS=(--rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) CERTBOT_OPTS=(--webroot --webroot-path=/data/letsencrypt --agree-tos -m "${MAINTAINER_EMAIL}" -d "${SERVER_NAME}" --non-interactive) sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" # mark the directory as letsencrypt dir - sudo-linux docker run -i --rm -v certs:/etc/openssl alpine:3.7 /bin/touch "${SSL_PATH}/.letsencrypt" + sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.7 /bin/touch "${SSL_PATH}/.letsencrypt" } letsencrypt_renew() { SERVER_NAME=$1 echo "==> Renewing Let's Encrypt SSL certificate for ${SERVER_NAME}" - CERTBOT_DOCKER_OPTS=(-i --rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) + CERTBOT_DOCKER_OPTS=(--rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) CERTBOT_OPTS=(-n --webroot --webroot-path=/data/letsencrypt -d "${SERVER_NAME}" --non-interactive) sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" } From e8f79dc80d94488ace738e1f28d232cb26ddc353 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 8 Aug 2018 14:40:26 +0200 Subject: [PATCH 094/194] Fix hdfs template hostname --- .../etc/hdfs-connector/sink-hdfs.properties.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs.properties.template index 12052219e..e7251ff1a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs.properties.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-connector/sink-hdfs.properties.template @@ -4,6 +4,6 @@ tasks.max=4 topics=android_empatica_e4_electrodermal_activity,android_empatica_e4_blood_volume_pulse,android_empatica_e4_temperature flush.size=80000 rotate.interval.ms=900000 -hdfs.url=hdfs://hdfs-namenode:8020 +hdfs.url=hdfs://hdfs-namenode-1:8020 format.class=org.radarcns.sink.hdfs.AvroFormatRadar topics.dir=topicAndroidNew From 27ec5ba08ab36fc8473a646581424c2ebc00e4b8 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 08:57:50 +0200 Subject: [PATCH 095/194] Configure Kafka-manager basic auth from nginx --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 +--- dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore | 1 + .../radar-cp-hadoop-stack/lib/perform-install.sh | 9 +++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 8625ae028..96c872213 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -685,6 +685,7 @@ services: - "./etc/webserver/nginx.conf:/etc/nginx/nginx.conf:ro" - "./etc/webserver/cors.conf:/etc/nginx/cors.conf:ro" - "./etc/webserver/ip-access-control.conf:/etc/nginx/ip-access-control.conf:ro" + - "./etc/webserver/kafka-manager.htpasswd:/etc/nginx/kafka-manager.htpasswd:ro" # healthcheck hard to do, however, it is possible to monitor this externally # with # docker logs --since 2m radarcphadoopstack_webserver_1 | grep "connect() failed" @@ -759,9 +760,6 @@ services: - kafka-3 environment: ZK_HOSTS: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 - KAFKA_MANAGER_AUTH_ENABLED: "true" - KAFKA_MANAGER_USERNAME: ${KAFKA_MANAGER_USERNAME} - KAFKA_MANAGER_PASSWORD: ${KAFKA_MANAGER_PASSWORD} restart: always healthcheck: test: ["CMD-SHELL", "[ $$(wget -q -O - localhost:9000/kafkamanager/api/health) = healthy ] || exit 1"] diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index 189a8c8f2..66a66367f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -1,6 +1,7 @@ /mongodb-connector/sink-mongo.properties /hdfs-connector/sink-hdfs.properties /webserver/nginx.conf +/webserver/kafka-manager.htpasswd /radar-backend/radar.yml /smtp.env /rest-api/device-catalog.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 1572a2ece..10742c8e6 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -53,14 +53,14 @@ fi if [[ -z ${KAFKA_MANAGER_PASSWORD} ]]; then echo "No Kafka Manager Password specified in the .env file. Please enter the password now: " - read kafkamanager_pass - inline_variable 'KAFKA_MANAGER_PASSWORD=' "${kafkamanager_pass}" .env + read KAFKA_MANAGER_PASSWORD + inline_variable 'KAFKA_MANAGER_PASSWORD=' "${KAFKA_MANAGER_PASSWORD}" .env fi if [[ -z ${PORTAINER_PASSWORD_HASH} ]]; then echo "No Portainer Password specified in the .env file. Please enter the password now: " read portainer_pass - portainer_pass_hash=`sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB admin ${portainer_pass} | cut -d ":" -f 2` + portainer_pass_hash=$(sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB admin ${portainer_pass} | cut -d ":" -f 2) inline_variable 'PORTAINER_PASSWORD_HASH=' "${portainer_pass_hash}" .env fi @@ -136,7 +136,8 @@ inline_variable 'username:[[:space:]]' "$HOTSTORAGE_USERNAME" etc/rest-api/radar inline_variable 'password:[[:space:]]' "$HOTSTORAGE_PASSWORD" etc/rest-api/radar.yml inline_variable 'database_name:[[:space:]]' "$HOTSTORAGE_NAME" etc/rest-api/radar.yml -echo "==> Configuring REDCap-Integration" +echo "==> Configuring Kafka-manager" +sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB "${KAFKA_MANAGER_USERNAME}" "${KAFKA_MANAGER_PASSWORD}" > etc/webserver/kafka-manager.passwd echo "==> Configuring nginx" inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx.conf From 001528446cac01b0c917694c846348785e45948d Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 08:58:15 +0200 Subject: [PATCH 096/194] Fixed gateway health check --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 96c872213..212d4a6e8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -783,7 +783,7 @@ services: command: ["radar-gateway", "/etc/radar-gateway/gateway.yml"] healthcheck: # should give an unauthenticated response, rather than a 404 - test: ["CMD-SHELL", "wget --spider localhost:8080/radar-gateway/ 2>&1 | grep -q 401 || exit 1"] + test: ["CMD-SHELL", "wget --spider localhost/radar-gateway/topics 2>&1 | grep -q 401 || exit 1"] interval: 1m30s timeout: 5s retries: 3 From 9b50e753199251c037723b1cf6085f2a144b739c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 15:53:40 +0200 Subject: [PATCH 097/194] Also remove volumes on a prune --- dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune | 1 + 1 file changed, 1 insertion(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune index 966e4b7ed..9f6e84c7f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/docker-prune @@ -11,6 +11,7 @@ echo "This will delete all the data associated with RADAR in HDFS, MongoDB and P select yn in "Yes" "No"; do case $yn in Yes ) sudo-linux docker system prune --filter "label!=certs" "$@"; + sudo-linux docker volume prune --filter "label!=certs" "$@"; sudo-linux rm -rf "$HDFS_DATA_DIR_1"; sudo-linux rm -rf "$HDFS_DATA_DIR_2"; sudo-linux rm -rf "$HDFS_DATA_DIR_3"; From 6cdc897c3664147b5089e2b5244ff3340b8f6d71 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 15:54:24 +0200 Subject: [PATCH 098/194] Upgrade radar-schemas 0.3.6 --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- .../radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 212d4a6e8..0b7605f85 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -260,7 +260,7 @@ services: #---------------------------------------------------------------------------# kafka-init: build: images/radar-kafka-init - image: radarbase/kafka-init + image: radarbase/kafka-init:0.3.6 networks: - kafka - zookeeper @@ -793,7 +793,7 @@ services: #---------------------------------------------------------------------------# catalog-server: build: images/radar-kafka-init - image: radarbase/kafka-init + image: radarbase/kafka-init:0.3.6 networks: - management command: radar-schemas-tools serve /schema/merged diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile index f5c7d331e..8a8bb9d6b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile @@ -25,7 +25,9 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.3.3 + +ENV RADAR_SCHEMAS_VERSION=0.3.6 + RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 From 836150cc708cb900b0fbae9b389a262c4d37ccaa Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 15:58:32 +0200 Subject: [PATCH 099/194] Upgrade radar-gateway --- .../radar-cp-hadoop-stack/docker-compose.yml | 4 +- .../radar-cp-hadoop-stack/etc/.gitignore | 1 + .../radar-cp-hadoop-stack/etc/env.template | 2 + .../etc/gateway/classpath.xml | 10 ----- .../etc/gateway/radar-is.yml | 2 - .../etc/webserver/nginx.conf.template | 2 +- .../lib/keystore-util.sh | 45 +++++++++++++++++++ .../lib/perform-install.sh | 14 +----- .../radar-cp-hadoop-stack/lib/util.sh | 32 +++++++++++++ 9 files changed, 86 insertions(+), 26 deletions(-) delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/gateway/classpath.xml delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/gateway/radar-is.yml create mode 100644 dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 0b7605f85..6b9eeefaf 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -694,7 +694,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarbase/management-portal:0.3.8 + image: radarbase/management-portal:0.4.1 networks: - default - api @@ -715,6 +715,8 @@ services: MANAGEMENTPORTAL_OAUTH_CLIENTS_FILE: /mp-includes/config/oauth_client_details.csv MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT: ${MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT} MANAGEMENTPORTAL_CATALOGUE_SERVER_SERVER_URL: http://catalog-server:9010/source-types + MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0: ${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0} + MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1: ${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1} JHIPSTER_SLEEP: 10 # gives time for the database to boot before the application JAVA_OPTS: -Xmx256m # maximum heap size for the JVM running ManagementPortal, increase this as necessary volumes: diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index 66a66367f..cf98074a4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -12,3 +12,4 @@ /managementportal/config/oauth_client_details.csv /schema/commons/ /schema/specifications/ +/gateway/radar-is.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 04ba3f3f9..9e3851871 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -30,3 +30,5 @@ POSTGRES_VERSION=10.4-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD= PORTAINER_PASSWORD_HASH= +MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0= +MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1= diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/classpath.xml b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/classpath.xml deleted file mode 100644 index 408a20479..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/classpath.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/radar-is.yml b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/radar-is.yml deleted file mode 100644 index a6f64f516..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/radar-is.yml +++ /dev/null @@ -1,2 +0,0 @@ -resourceName: res_ManagementPortal -publicKeyEndpoint: http://managementportal-app:8080/managementportal/oauth/token_key diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 02ff9953a..05fe79721 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -59,7 +59,7 @@ http { location /kafka/ { include cors.conf; - proxy_pass http://gateway:8080/radar-gateway/; + proxy_pass http://gateway/radar-gateway/; proxy_set_header Host $host; proxy_http_version 1.1; proxy_set_header Connection ""; diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh new file mode 100644 index 000000000..0f0c8c786 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +function init_mp_keystore { + keystorefile=etc/managementportal/config/keystore.jks + KEYTOOL_OPTS="-keystore ${keystorefile} -storepass radarbase -keypass radarbase $KEYTOOL_OPTS" + if ! keytool -list $KEYTOOL_OPTS -alias radarbase-managementportal-ec >/dev/null 2>/dev/null; then + KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYTOOL_CREATE_OPTS" + if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then + KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" + fi + echo "--> Generating keystore to hold EC keypair for JWT signing" + keytool $KEYTOOL_CREATE_OPTS $KEYTOOL_OPTS + else + echo "--> ECDSA keypair for signing JWTs already exists. Not creating a new one." + fi + + chmod 400 "${keystorefile}" + + ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) + NUM_ALIASES=${#ALIASES[@]} + + GATEWAY_CONF=etc/gateway/radar-is.yml + echo "--> Updating gateway signature keys" + echo "resourceName: res_gateway" > $GATEWAY_CONF + echo "publicKeys:" >> $GATEWAY_CONF + + indent=4 + for (( i=0; i < ${#ALIASES[@]}; i++)); do + ALIAS=${ALIASES[$i]} + ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env + echo " - |-" >> $GATEWAY_CONF + cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" + if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then + REPLACE_PUBKEY="EC PUBLIC KEY" + else + REPLACE_PUBKEY="PUBLIC KEY" + fi + + while IFS='' read -r line && [ -n "$line" ]; do + line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) + printf "%*s%s\n" $indent '' "$line" >> $GATEWAY_CONF + done <<< "$cert" + done +} + diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 10742c8e6..6e181d93e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -115,19 +115,9 @@ inline_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-h echo "==> Configuring Management Portal" +. lib/keystore-util.sh -keystorefile=etc/managementportal/config/keystore.jks -if [ -f "$keystorefile" ]; then - echo "--> Keystore for signing JWTs already exists. Not creating a new one." -else - echo "--> Generating keystore to hold RSA keypair for JWT signing" - if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then - sudo-linux keytool -genkeypair -dname "${MANAGEMENTPORTAL_KEY_DNAME}" -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase - else - sudo-linux keytool -genkeypair -alias selfsigned -keyalg RSA -keystore "$keystorefile" -keysize 4096 -storepass radarbase -keypass radarbase - fi - sudo-linux chmod 400 "${keystorefile}" -fi +sudo-linux init_mp_keystore echo "==> Configuring REST-API" diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh index b4426d8ea..1322d4d3e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh @@ -68,6 +68,14 @@ inline_variable() { sed_i 's|^\([[:space:]]*'"$1"'\).*$|\1'"$2"'|' "$3" } +ensure_variable() { + if grep -q "$1" "$3"; then + inline_variable "$@" + else + echo "$1$2" >> "$3" + fi +} + # Copies the template (defined by the given config file with suffix # ".template") to intended configuration file, if the file does not # yet exist. @@ -182,6 +190,30 @@ request_certificate() { sudo-linux docker-compose kill -s HUP webserver 1>/dev/null 2>&1 } +query_password() { + echo $2 + stty -echo + printf "Password: " + read PASSWORD + stty echo + printf "\n" + eval "$1=$PASSWORD" +} + +ensure_env_default() { + if eval '[ -z $'$1' ]'; then + ensure_variable "$1=" "$2" .env + eval "$1=$2" + fi +} + +ensure_env_password() { + if eval '[ -z $'$1' ]'; then + query_password $1 "$2" + ensure_variable "$1=" "$PASSWORD" .env + fi +} + echo "OS version: $(uname -a)" check_command_exists docker check_command_exists docker-compose From b51d5e1724725b98e5ceeb17d08ee0cae999d6e0 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 15:59:28 +0200 Subject: [PATCH 100/194] Update Kafka manager --- .../radar-cp-hadoop-stack/docker-compose.yml | 2 +- .../etc/webserver/nginx.conf.template | 2 ++ images/kafka-manager/Dockerfile | 25 +++++++++++-------- images/kafka-manager/entrypoint.sh | 4 +++ 4 files changed, 21 insertions(+), 12 deletions(-) create mode 100755 images/kafka-manager/entrypoint.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 6b9eeefaf..9156ec8d1 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -750,7 +750,7 @@ services: # Kafka Manager # #---------------------------------------------------------------------------# kafka-manager: - image: radarbase/kafka-manager + image: radarbase/kafka-manager:1.3.3.18 networks: - zookeeper - kafka diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 05fe79721..19ccecddc 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -116,6 +116,8 @@ http { } location /kafkamanager/{ include ip-access-control.conf; + auth_basic "Kafka manager"; + auth_basic_user_file kafka-manager.htpasswd; proxy_pass http://kafka-manager:9000; proxy_set_header Host $host; diff --git a/images/kafka-manager/Dockerfile b/images/kafka-manager/Dockerfile index 28f2de9c3..2d2fe7cc4 100644 --- a/images/kafka-manager/Dockerfile +++ b/images/kafka-manager/Dockerfile @@ -1,19 +1,21 @@ -FROM hseeberger/scala-sbt as builder -ENV KM_VERSION=1.3.3.14 +FROM hseeberger/scala-sbt:8u171-2.12.6-1.2.0 as builder +ENV SBT_VERSION=0.13.9 -RUN mkdir -p /tmp && \ - cd /tmp && \ - wget https://github.com/yahoo/kafka-manager/archive/${KM_VERSION}.tar.gz && \ +RUN mkdir /code + +WORKDIR /code + +RUN sbt -sbt-version ${SBT_VERSION} + +ENV KM_VERSION=1.3.3.18 + +RUN wget https://github.com/yahoo/kafka-manager/archive/${KM_VERSION}.tar.gz && \ tar xxf ${KM_VERSION}.tar.gz && \ - cd /tmp/kafka-manager-${KM_VERSION} && \ + cd kafka-manager-${KM_VERSION} && \ sbt clean dist && \ unzip -d / ./target/universal/kafka-manager-${KM_VERSION}.zip && \ - rm -fr /tmp/${KM_VERSION} /tmp/kafka-manager-${KM_VERSION} && \ mv /kafka-manager-${KM_VERSION} /kafka-manager -COPY ./conf/application.conf /kafka-manager-${KM_VERSION}/conf/application.conf -WORKDIR /kafka-manager-${KM_VERSION} - FROM openjdk:8-alpine MAINTAINER Yatharth Ranjan @@ -22,7 +24,8 @@ RUN apk add --no-cache bash COPY --from=builder /kafka-manager /kafka-manager COPY ./conf/application.conf /kafka-manager/conf/application.conf +COPY ./entrypoint.sh /kafka-manager/ WORKDIR /kafka-manager EXPOSE 9000 -ENTRYPOINT ["./bin/kafka-manager","-Dconfig.file=conf/application.conf"] +ENTRYPOINT ["./entrypoint.sh"] diff --git a/images/kafka-manager/entrypoint.sh b/images/kafka-manager/entrypoint.sh new file mode 100755 index 000000000..42c3d104b --- /dev/null +++ b/images/kafka-manager/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rm -f RUNNING_PID +exec ./bin/kafka-manager -Dconfig.file=conf/application.conf From f1221c19b57e6623180769461a742c5af10f750b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 15:59:54 +0200 Subject: [PATCH 101/194] Install script cleanup --- .../radar-cp-hadoop-stack/etc/.gitignore | 1 + .../lib/perform-install.sh | 68 ++++++++----------- 2 files changed, 28 insertions(+), 41 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index cf98074a4..d55a08afb 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -2,6 +2,7 @@ /hdfs-connector/sink-hdfs.properties /webserver/nginx.conf /webserver/kafka-manager.htpasswd +/webserver/ip-access-control.conf /radar-backend/radar.yml /smtp.env /rest-api/device-catalog.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 6e181d93e..105ec53ec 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -4,14 +4,6 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." . lib/util.sh -sudo-linux chmod og-rw ./.env -sudo-linux chmod og-rwx ./etc -if [ -e ./output ]; then - sudo-linux chmod og-rwx ./output -else - sudo-linux mkdir -m 0700 ./output -fi - # Initialize and check all config files check_config_present .env etc/env.template check_config_present etc/radar-backend/radar.yml @@ -23,6 +15,15 @@ copy_template_if_absent etc/rest-api/radar.yml copy_template_if_absent etc/webserver/nginx.conf copy_template_if_absent etc/webserver/ip-access-control.conf +# Set permissions +sudo-linux chmod og-rw ./.env +sudo-linux chmod og-rwx ./etc +if [ -e ./output ]; then + sudo-linux chmod og-rwx ./output +else + sudo-linux mkdir -m 0700 ./output +fi + . ./.env # Check provided directories and configurations @@ -33,35 +34,21 @@ check_parent_exists HDFS_NAME_DIR_2 ${HDFS_NAME_DIR_2} check_parent_exists MONGODB_DIR ${MONGODB_DIR} check_parent_exists MP_POSTGRES_DIR ${MP_POSTGRES_DIR} -if [ -z ${SERVER_NAME} ]; then - echo "Set SERVER_NAME variable in .env" - exit 1 -fi +# Checking provided passwords and environment variables +ensure_env_default SERVER_NAME localhost -# Checking provided passwords -if [[ -z ${HOTSTORAGE_PASSWORD} ]]; then - echo "No Hotstorage Password specified in the .env file. Please enter the password now: " - read hotstorage_pass - inline_variable 'HOTSTORAGE_PASSWORD=' "${hotstorage_pass}" .env -fi +ensure_env_default HOTSTORAGE_USERNAME hotstorage +ensure_env_password HOTSTORAGE_PASSWORD "Hot storage (MongoDB) password not set in .env." +ensure_env_default HOTSTORAGE_NAME hotstorage -if [[ -z ${POSTGRES_PASSWORD} ]]; then - echo "No Postgres Password specified in the .env file. Please enter the password now: " - read postgres_pass - inline_variable 'POSTGRES_PASSWORD=' "${postgres_pass}" .env -fi - -if [[ -z ${KAFKA_MANAGER_PASSWORD} ]]; then - echo "No Kafka Manager Password specified in the .env file. Please enter the password now: " - read KAFKA_MANAGER_PASSWORD - inline_variable 'KAFKA_MANAGER_PASSWORD=' "${KAFKA_MANAGER_PASSWORD}" .env -fi +ensure_env_password POSTGRES_PASSWORD "PostgreSQL password not set in .env." +ensure_env_default KAFKA_MANAGER_USERNAME kafkamanager-user +ensure_env_password KAFKA_MANAGER_PASSWORD "Kafka Manager password not set in .env." -if [[ -z ${PORTAINER_PASSWORD_HASH} ]]; then - echo "No Portainer Password specified in the .env file. Please enter the password now: " - read portainer_pass - portainer_pass_hash=$(sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB admin ${portainer_pass} | cut -d ":" -f 2) - inline_variable 'PORTAINER_PASSWORD_HASH=' "${portainer_pass_hash}" .env +if [ -z ${PORTAINER_PASSWORD_HASH} ]; then + query_password PORTAINER_PASSWORD "Portainer password not set in .env." + PORTAINER_PASSWORD_HASH=$(sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB admin "${PORTAINER_PASSWORD}" | cut -d ":" -f 2) + ensure_variable 'PORTAINER_PASSWORD_HASH=' "${PORTAINER_PASSWORD_HASH}" .env fi # Create networks and volumes @@ -83,7 +70,6 @@ fi # Initializing Kafka echo "==> Setting up topics" sudo-linux bin/radar-docker up -d zookeeper-1 zookeeper-2 zookeeper-3 kafka-1 kafka-2 kafka-3 schema-registry-1 -sleep 60 sudo-linux bin/radar-docker run --rm kafka-init KAFKA_SCHEMA_RETENTION_MS=${KAFKA_SCHEMA_RETENTION_MS:-5400000000} KAFKA_SCHEMA_RETENTION_CMD='kafka-configs --zookeeper "${KAFKA_ZOOKEEPER_CONNECT}" --entity-type topics --entity-name _schemas --alter --add-config min.compaction.lag.ms='${KAFKA_SCHEMA_RETENTION_MS}',cleanup.policy=compact' @@ -91,9 +77,9 @@ sudo-linux bin/radar-docker exec -T kafka-1 bash -c "$KAFKA_SCHEMA_RETENTION_CMD echo "==> Configuring MongoDB Connector" # Update sink-mongo.properties -inline_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sink-mongo.properties -inline_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties -inline_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties +ensure_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sink-mongo.properties +ensure_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties +ensure_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties # Set topics if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then @@ -102,7 +88,7 @@ if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then COMBINED_AGG_TOPIC_LIST="${RADAR_AGG_TOPIC_LIST},${COMBINED_AGG_TOPIC_LIST}" fi fi -inline_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/mongodb-connector/sink-mongo.properties +ensure_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/mongodb-connector/sink-mongo.properties echo "==> Configuring HDFS Connector" if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then @@ -111,7 +97,7 @@ if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then COMBINED_RAW_TOPIC_LIST="${RADAR_RAW_TOPIC_LIST},${COMBINED_RAW_TOPIC_LIST}" fi fi -inline_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties +ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties echo "==> Configuring Management Portal" @@ -127,7 +113,7 @@ inline_variable 'password:[[:space:]]' "$HOTSTORAGE_PASSWORD" etc/rest-api/radar inline_variable 'database_name:[[:space:]]' "$HOTSTORAGE_NAME" etc/rest-api/radar.yml echo "==> Configuring Kafka-manager" -sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB "${KAFKA_MANAGER_USERNAME}" "${KAFKA_MANAGER_PASSWORD}" > etc/webserver/kafka-manager.passwd +sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB "${KAFKA_MANAGER_USERNAME}" "${KAFKA_MANAGER_PASSWORD}" > etc/webserver/kafka-manager.htpasswd echo "==> Configuring nginx" inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx.conf From 76bb6818a8825421672f0ee78e0c5547b7e0915f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 16:55:15 +0200 Subject: [PATCH 102/194] Fixed keystore util on Linux and updated radar-gateway --- .../radar-cp-hadoop-stack/bin/keystore-init | 48 +++++++++++++++++++ .../radar-cp-hadoop-stack/docker-compose.yml | 5 +- .../lib/keystore-util.sh | 45 ----------------- .../lib/perform-install.sh | 8 ++-- .../radar-cp-hadoop-stack/lib/util.sh | 3 -- 5 files changed, 56 insertions(+), 53 deletions(-) create mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init new file mode 100755 index 000000000..0c418228c --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init @@ -0,0 +1,48 @@ +#!/bin/bash + +cd "$( dirname "${BASH_SOURCE[0]}" )/.." + +. ./.env +. lib/util.sh + +keystorefile=etc/managementportal/config/keystore.jks +KEYTOOL_OPTS="-keystore ${keystorefile} -storepass radarbase -keypass radarbase $KEYTOOL_OPTS" +if ! keytool -list $KEYTOOL_OPTS -alias radarbase-managementportal-ec >/dev/null 2>/dev/null; then + KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYTOOL_CREATE_OPTS" + if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then + KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" + fi + echo "--> Generating keystore to hold EC keypair for JWT signing" + keytool $KEYTOOL_CREATE_OPTS $KEYTOOL_OPTS +else + echo "--> ECDSA keypair for signing JWTs already exists. Not creating a new one." +fi + +chmod 400 "${keystorefile}" + +ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) +NUM_ALIASES=${#ALIASES[@]} + +GATEWAY_CONF=etc/gateway/radar-is.yml +echo "--> Updating gateway signature keys" +echo "resourceName: res_gateway" > $GATEWAY_CONF +echo "publicKeys:" >> $GATEWAY_CONF + +indent=4 +for (( i=0; i < ${#ALIASES[@]}; i++)); do + ALIAS=${ALIASES[$i]} + ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env + echo " - |-" >> $GATEWAY_CONF + cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" + if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then + REPLACE_PUBKEY="EC PUBLIC KEY" + else + REPLACE_PUBKEY="PUBLIC KEY" + fi + + while IFS='' read -r line && [ -n "$line" ]; do + line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) + printf "%*s%s\n" $indent '' "$line" >> $GATEWAY_CONF + done <<< "$cert" +done + diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 9156ec8d1..3ae8230a5 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -773,16 +773,17 @@ services: # RADAR Gateway # #---------------------------------------------------------------------------# gateway: - image: radarbase/radar-gateway:0.2.1 + image: radarbase/radar-gateway:0.2.2 networks: - api - kafka depends_on: - rest-proxy-1 - - managementportal-app volumes: - ./etc/gateway:/etc/radar-gateway command: ["radar-gateway", "/etc/radar-gateway/gateway.yml"] + environment: + RADAR_IS_CONFIG_LOCATION: /etc/radar-gateway/radar-is.yml healthcheck: # should give an unauthenticated response, rather than a 404 test: ["CMD-SHELL", "wget --spider localhost/radar-gateway/topics 2>&1 | grep -q 401 || exit 1"] diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh deleted file mode 100644 index 0f0c8c786..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/keystore-util.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -function init_mp_keystore { - keystorefile=etc/managementportal/config/keystore.jks - KEYTOOL_OPTS="-keystore ${keystorefile} -storepass radarbase -keypass radarbase $KEYTOOL_OPTS" - if ! keytool -list $KEYTOOL_OPTS -alias radarbase-managementportal-ec >/dev/null 2>/dev/null; then - KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYTOOL_CREATE_OPTS" - if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then - KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" - fi - echo "--> Generating keystore to hold EC keypair for JWT signing" - keytool $KEYTOOL_CREATE_OPTS $KEYTOOL_OPTS - else - echo "--> ECDSA keypair for signing JWTs already exists. Not creating a new one." - fi - - chmod 400 "${keystorefile}" - - ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) - NUM_ALIASES=${#ALIASES[@]} - - GATEWAY_CONF=etc/gateway/radar-is.yml - echo "--> Updating gateway signature keys" - echo "resourceName: res_gateway" > $GATEWAY_CONF - echo "publicKeys:" >> $GATEWAY_CONF - - indent=4 - for (( i=0; i < ${#ALIASES[@]}; i++)); do - ALIAS=${ALIASES[$i]} - ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env - echo " - |-" >> $GATEWAY_CONF - cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" - if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then - REPLACE_PUBKEY="EC PUBLIC KEY" - else - REPLACE_PUBKEY="PUBLIC KEY" - fi - - while IFS='' read -r line && [ -n "$line" ]; do - line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) - printf "%*s%s\n" $indent '' "$line" >> $GATEWAY_CONF - done <<< "$cert" - done -} - diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 105ec53ec..151ac412f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -4,6 +4,10 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." . lib/util.sh +echo "OS version: $(uname -a)" +check_command_exists docker +check_command_exists docker-compose + # Initialize and check all config files check_config_present .env etc/env.template check_config_present etc/radar-backend/radar.yml @@ -101,9 +105,7 @@ ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-h echo "==> Configuring Management Portal" -. lib/keystore-util.sh - -sudo-linux init_mp_keystore +bin/keystore-init echo "==> Configuring REST-API" diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh index 1322d4d3e..d23345df1 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh @@ -214,6 +214,3 @@ ensure_env_password() { fi } -echo "OS version: $(uname -a)" -check_command_exists docker -check_command_exists docker-compose From dd3f58deefe051ea7d5cfda8511425bd37448ddf Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 17:00:40 +0200 Subject: [PATCH 103/194] Keystore simplifications --- dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init index 0c418228c..1aba031f3 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init @@ -20,29 +20,27 @@ fi chmod 400 "${keystorefile}" -ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) -NUM_ALIASES=${#ALIASES[@]} GATEWAY_CONF=etc/gateway/radar-is.yml echo "--> Updating gateway signature keys" echo "resourceName: res_gateway" > $GATEWAY_CONF echo "publicKeys:" >> $GATEWAY_CONF -indent=4 +ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) for (( i=0; i < ${#ALIASES[@]}; i++)); do ALIAS=${ALIASES[$i]} ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env echo " - |-" >> $GATEWAY_CONF - cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then REPLACE_PUBKEY="EC PUBLIC KEY" else REPLACE_PUBKEY="PUBLIC KEY" fi + cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" while IFS='' read -r line && [ -n "$line" ]; do line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) - printf "%*s%s\n" $indent '' "$line" >> $GATEWAY_CONF + echo " $line" >> $GATEWAY_CONF done <<< "$cert" done From b7d80c9f24f8d9c5ef1e4fdff0d2ede3a53d2b80 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Aug 2018 17:09:18 +0200 Subject: [PATCH 104/194] Setup keystore in travis --- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index 9c9ba065e..d69aadf7e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -14,4 +14,6 @@ cp ../etc/mongodb-connector/sink-mongo.properties.template ../etc/mongodb-connec cp ../etc/managementportal/config/oauth_client_details.csv.template ../etc/managementportal/config/oauth_client_details.csv cp ../etc/redcap-integration/radar.yml.template ../etc/redcap-integration/radar.yml +../bin/keystore-init + echo "Setting up mock configurations finished..." From b8f819af878ec667688f3ee18723d757ded35499 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Mon, 13 Aug 2018 12:10:04 +0100 Subject: [PATCH 105/194] Fix import for cert-renew --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index 5b7147fc9..2a5ae38d3 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -1,6 +1,8 @@ #!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")/.." +. ./lib/util.sh + stack=${DOCKER_COMPOSE:-docker-compose} function print_usage() { From ae2d557bcd2ff5177453e58ef4589936f69753b0 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 13 Aug 2018 15:52:40 +0200 Subject: [PATCH 106/194] Move util import to relevant switch case. --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index 2a5ae38d3..57b2d4b19 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -1,8 +1,6 @@ #!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")/.." -. ./lib/util.sh - stack=${DOCKER_COMPOSE:-docker-compose} function print_usage() { @@ -45,6 +43,7 @@ shopt -s extglob case "${CMD}" in cert-renew) + . ./lib/util.sh request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" force ;; install) From 768990f72212944c9e1e97753bd0ce09c9a3ba23 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 22 Aug 2018 17:32:12 +0200 Subject: [PATCH 107/194] Add rate limiting to ManagementPortal --- .../etc/webserver/nginx.conf.template | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 19ccecddc..b72e1422b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -17,6 +17,9 @@ http { # add nosniff header (https://www.owasp.org/index.php/List_of_useful_HTTP_headers) add_header X-Content-Type-Options nosniff; + # For logins, make 2 requests per second at most + limit_req_zone $binary_remote_addr zone=login_limit:10m rate=2r/s; + server { listen 80 default_server; listen [::]:80 default_server; @@ -114,6 +117,19 @@ http { proxy_pass http://managementportal-app:8080/managementportal/; proxy_set_header Host $host; } + location /managementportal/oauth/ { + # Allow 20 fast-following requests, like when authorizing a user. + limit_req zone=login_limit burst=20; + include cors.conf; + proxy_pass http://managementportal-app:8080/managementportal/oauth/; + proxy_set_header Host $host; + } + location /managementportal/api/meta-token/ { + limit_req zone=login_limit; + include cors.conf; + proxy_pass http://managementportal-app:8080/managementportal/api/meta-token/; + proxy_set_header Host $host; + } location /kafkamanager/{ include ip-access-control.conf; auth_basic "Kafka manager"; From 834e3200395769f90330ef0c48573dc22a560bf3 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 27 Aug 2018 15:38:09 +0200 Subject: [PATCH 108/194] Use correct Spring syntax for a list of data --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 3ae8230a5..2ee7a9893 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -715,8 +715,7 @@ services: MANAGEMENTPORTAL_OAUTH_CLIENTS_FILE: /mp-includes/config/oauth_client_details.csv MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT: ${MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT} MANAGEMENTPORTAL_CATALOGUE_SERVER_SERVER_URL: http://catalog-server:9010/source-types - MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0: ${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0} - MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1: ${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1} + SPRING_APPLICATION_JSON: '{"managementportal":{"oauth":{"checkingKeyAliases":["${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0}","${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1}"]}}}' JHIPSTER_SLEEP: 10 # gives time for the database to boot before the application JAVA_OPTS: -Xmx256m # maximum heap size for the JVM running ManagementPortal, increase this as necessary volumes: From c2377102b2f25cdd26fac012dcde455e96e4e9bb Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 29 Aug 2018 15:11:42 +0200 Subject: [PATCH 109/194] Update env.template --- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 9e3851871..86b458e4c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -3,7 +3,7 @@ MANAGEMENTPORTAL_KEY_DNAME=CN=localhost,OU=MyName,O=MyOrg,L=MyCity,S=MyState,C=M MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET= SELF_SIGNED_CERT=yes MAINTAINER_EMAIL=me@example.com -RADAR_RAW_TOPIC_LIST= +RADAR_RAW_TOPIC_LIST=android_phone_usage_event_aggregated,android_phone_usage_event_output RADAR_AGG_TOPIC_LIST= HOTSTORAGE_USERNAME=mongodb-user HOTSTORAGE_PASSWORD= From 18802dade242d4f8611da2cd21d96d9542e81c5e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 5 Sep 2018 15:47:40 +0200 Subject: [PATCH 110/194] Fixed sudo-linux and hdfs-restructure for unpriviledged user --- .../bin/hdfs-restructure | 25 +++++++++++++------ .../radar-cp-hadoop-stack/lib/util.sh | 5 ++-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index a9be0e739..5cfff74a1 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -1,13 +1,16 @@ #!/bin/bash # HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.1 +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.2 # HDFS restructure script flags HDFS_RESTRUCTURE_OPTS=( --compression gzip --deduplicate --num-threads 3 ) +OUTPUT_USER=${OUTPUT_USER:-$(id -un)} +OUTPUT_GROUP=${OUTPUT_GROUP:-$(id -gn)} + # HDFS restructure JVM flags export RADAR_HDFS_RESTRUCTURE_OPTS="$JAVA_OPTS -Xmx4g" # Without DOCKER_OPTS, run in interactive mode. @@ -56,11 +59,17 @@ HDFS_FILE=$1 HDFS_OUTPUT_DIR=/output # HDFS command to run HDFS_COMMAND=( - "${HDFS_RESTRUCTURE_OPTS[@]}" - -n hdfs-namenode-1 - -o "$HDFS_OUTPUT_DIR" - --tmp-dir "$HDFS_OUTPUT_DIR/+tmp" - "$HDFS_FILE" ) + "${HDFS_RESTRUCTURE_OPTS[@]}" + -p local-uid=$(id -u ${OUTPUT_USER}) + -p local-gid=$(grep "^${OUTPUT_GROUP}:" /etc/group | cut -d: -f3) + -n hdfs-namenode-1 + -o "$HDFS_OUTPUT_DIR" + --tmp-dir "$HDFS_OUTPUT_DIR/+tmp" + "$HDFS_FILE" + ) -mkdir -p $OUTPUT_DIR -sudo-linux docker run ${DOCKER_OPTS} -t --rm --network hadoop -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" -e RADAR_HDFS_RESTRUCTURE_OPTS $DOCKER_IMAGE "${HDFS_COMMAND[@]}" +sudo-linux docker run ${DOCKER_OPTS} \ + -t --rm --network hadoop \ + -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" \ + -e RADAR_HDFS_RESTRUCTURE_OPTS \ + $DOCKER_IMAGE "${HDFS_COMMAND[@]}" diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh index d23345df1..87d4db50c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh @@ -41,7 +41,7 @@ check_parent_exists() { # sudo if on Linux, not on OS X # useful for docker, which doesn't need sudo on OS X sudo-linux() { - if [ $(uname) == "Darwin" ]; then + if [ $(uname) == "Darwin" ] || id -Gn | grep -qve '\'; then "$@" else sudo "$@" @@ -53,7 +53,7 @@ sed_i() { if [ $(uname) == "Darwin" ]; then sed -i '' "$@" else - sudo sed -i -- "$@" + sudo-linux sed -i -- "$@" fi } @@ -213,4 +213,3 @@ ensure_env_password() { ensure_variable "$1=" "$PASSWORD" .env fi } - From 8f765bc81b2d655a50736099200b9dcac4d2e4cc Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 5 Sep 2018 16:01:41 +0200 Subject: [PATCH 111/194] Unpriveledged systemd scripts --- .../lib/install-systemd-wrappers.sh | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index 9b8e113ca..a7df08c76 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -3,35 +3,44 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." echo $(pwd) . lib/util.sh +if id -Gn | grep -qe '\'; then + BASE=/etc/systemd/system + SYSTEMCTL_OPTS=() +else + BASE=$HOME/.config/systemd/user + mkdir -p $BASE + SYSTEMCTL_OPTS=(--user) +fi + echo "==> Copying templates" -copy_template_if_absent /etc/systemd/system/radar-docker.service lib/systemd/radar-docker.service.template -copy_template_if_absent /etc/systemd/system/radar-output.service lib/systemd/radar-output.service.template -copy_template_if_absent /etc/systemd/system/radar-output.timer lib/systemd/radar-output.timer.template -copy_template_if_absent /etc/systemd/system/radar-check-health.service lib/systemd/radar-check-health.service.template -copy_template_if_absent /etc/systemd/system/radar-check-health.timer lib/systemd/radar-check-health.timer.template -copy_template_if_absent /etc/systemd/system/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template -copy_template_if_absent /etc/systemd/system/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template +copy_template_if_absent $BASE/radar-docker.service lib/systemd/radar-docker.service.template +copy_template_if_absent $BASE/radar-output.service lib/systemd/radar-output.service.template +copy_template_if_absent $BASE/radar-output.timer lib/systemd/radar-output.timer.template +copy_template_if_absent $BASE/radar-check-health.service lib/systemd/radar-check-health.service.template +copy_template_if_absent $BASE/radar-check-health.timer lib/systemd/radar-check-health.timer.template +copy_template_if_absent $BASE/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template +copy_template_if_absent $BASE/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template echo "==> Inlining variables" -inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-docker.service -inline_variable 'ExecStart=' "$PWD/bin/radar-docker foreground" /etc/systemd/system/radar-docker.service +inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-docker.service +inline_variable 'ExecStart=' "$PWD/bin/radar-docker foreground" $BASE/radar-docker.service -inline_variable 'WorkingDirectory=' "$PWD/hdfs" /etc/systemd/system/radar-output.service -inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure-process" /etc/systemd/system/radar-output.service +inline_variable 'WorkingDirectory=' "$PWD/hdfs" $BASE/radar-output.service +inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure-process" $BASE/radar-output.service -inline_variable 'WorkingDirectory=' "$PWD" /etc/systemd/system/radar-check-health.service -inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" /etc/systemd/system/radar-check-health.service +inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-check-health.service +inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" $BASE/radar-check-health.service -inline_variable 'WorkingDirectory=' "$DIR" /etc/systemd/system/radar-renew-certificate.service -inline_variable 'ExecStart=' "$PWD/bin/radar-docker cert-renew" /etc/systemd/system/radar-renew-certificate.service +inline_variable 'WorkingDirectory=' "$DIR" $BASE/radar-renew-certificate.service +inline_variable 'ExecStart=' "$PWD/bin/radar-docker cert-renew" $BASE/radar-renew-certificate.service echo "==> Reloading systemd" -systemctl daemon-reload -systemctl enable radar-docker -systemctl enable radar-output.timer -systemctl enable radar-check-health.timer -systemctl enable radar-renew-certificate.timer -systemctl start radar-docker -systemctl start radar-output.timer -systemctl start radar-check-health.timer -systemctl start radar-renew-certificate.timer +systemctl "${SYSTEMCTL_OPTS[@]}" daemon-reload +systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-docker +systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-output.timer +systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-check-health.timer +systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-renew-certificate.timer +systemctl "${SYSTEMCTL_OPTS[@]}" start radar-docker +systemctl "${SYSTEMCTL_OPTS[@]}" start radar-output.timer +systemctl "${SYSTEMCTL_OPTS[@]}" start radar-check-health.timer +systemctl "${SYSTEMCTL_OPTS[@]}" start radar-renew-certificate.timer From 680f409dc1e1622b444112d13aab5bf3ff975ef0 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 5 Sep 2018 16:48:01 +0200 Subject: [PATCH 112/194] Redesign for user-local systemd scripts --- .../radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh | 1 + .../lib/systemd/radar-check-health.service.template | 1 - .../lib/systemd/radar-check-health.timer.template | 2 +- .../lib/systemd/radar-docker.service.template | 4 +--- .../lib/systemd/radar-output.service.template | 1 - .../lib/systemd/radar-output.timer.template | 2 +- .../lib/systemd/radar-renew-certificate.service.template | 1 - .../lib/systemd/radar-renew-certificate.timer.template | 2 +- 8 files changed, 5 insertions(+), 9 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index a7df08c76..e6e7cd8cd 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -10,6 +10,7 @@ else BASE=$HOME/.config/systemd/user mkdir -p $BASE SYSTEMCTL_OPTS=(--user) + export XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-/run/user/$UID} fi echo "==> Copying templates" diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template index e26609aad..5150878fa 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template @@ -1,7 +1,6 @@ [Unit] Description=RADAR-Docker service After=radar-docker.service -Requires=docker.service [Service] diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.timer.template index be08892cc..b309d5119 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.timer.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.timer.template @@ -8,4 +8,4 @@ OnUnitActiveSec=5min Unit=radar-check-health.service [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=default.target \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template index ad02b25d1..fcb1f83f2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template @@ -1,7 +1,5 @@ [Unit] Description=RADAR-Docker service -BindsTo=docker.service -After=docker.service [Service] TimeoutStartSec=0 @@ -20,4 +18,4 @@ ExecStop=/usr/local/bin/docker-compose down NotifyAccess=all [Install] -WantedBy=multi-user.target +WantedBy=default.target diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template index d6f68b75c..547384bfe 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template @@ -1,7 +1,6 @@ [Unit] Description=RADAR-Docker service After=radar-docker.service -Requires=docker.service [Service] diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template index 3ef981a94..abc8edb30 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template @@ -8,4 +8,4 @@ OnUnitActiveSec=1h Unit=radar-output.service [Install] -WantedBy=multi-user.target +WantedBy=default.target diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template index e26609aad..5150878fa 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template @@ -1,7 +1,6 @@ [Unit] Description=RADAR-Docker service After=radar-docker.service -Requires=docker.service [Service] diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template index 2c3939f1a..4af86edb0 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template @@ -9,7 +9,7 @@ Persistent=true Unit=radar-check-health.service [Install] -WantedBy=multi-user.target +WantedBy=default.target From 9accda529d8395429d3af060577f80f12e869678 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 6 Sep 2018 12:04:11 +0200 Subject: [PATCH 113/194] Small bug fixes and simplifications --- .../bin/hdfs-restructure-process | 31 -------------- .../radar-cp-hadoop-stack/bin/radar-docker | 2 +- .../radar-cp-hadoop-stack/etc/env.template | 1 + .../lib/install-systemd-wrappers.sh | 5 ++- .../radar-cp-hadoop-stack/lib/lock-util.sh | 41 ------------------- .../lib/perform-install.sh | 1 + .../radar-renew-certificate.timer.template | 4 -- 7 files changed, 6 insertions(+), 79 deletions(-) delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/lib/lock-util.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process deleted file mode 100755 index bae59f7b6..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure-process +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -cd "$( dirname "${BASH_SOURCE[0]}" )/.." - -# log file -logfile=hdfs_output.log - -# landing folder. Specify if not -storage_directory=output - -# lock file -lockfile=.LOCKFILE - -# involved HDFS directory -hdfs_directory=/topicAndroidNew - -. ./lib/lock-util.sh - -# extract file from hdfs to backup storage if no other instance is running -if [ ! -f $lockfile ]; then - log_info "Creating lock ..." - touch $lockfile - (DOCKER_OPTS="" ./bin/hdfs-restructure $hdfs_directory $storage_directory >> $logfile 2>&1) - log_info "Removing lock ..." - rm $lockfile -else - log_info "Another instance is already running ... " -fi -log_info "### DONE ###" - -# check if log size exceeds the limit. If so, it rotates the log file -rolloverLog diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index 57b2d4b19..ec5f2ecfb 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -61,7 +61,7 @@ quit) exit 0 || exit 1 ;; health) - . lib/radar-health.sh + . lib/check-health.sh ;; hdfs) exec $stack up -d --remove-orphans hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 9e3851871..ac886367d 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -32,3 +32,4 @@ KAFKA_MANAGER_PASSWORD= PORTAINER_PASSWORD_HASH= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1= +RESTRUCTURE_OUTPUT_DIR=output diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index e6e7cd8cd..96db3d45a 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -2,6 +2,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." echo $(pwd) . lib/util.sh +. .env if id -Gn | grep -qe '\'; then BASE=/etc/systemd/system @@ -26,8 +27,8 @@ echo "==> Inlining variables" inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-docker.service inline_variable 'ExecStart=' "$PWD/bin/radar-docker foreground" $BASE/radar-docker.service -inline_variable 'WorkingDirectory=' "$PWD/hdfs" $BASE/radar-output.service -inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure-process" $BASE/radar-output.service +inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-output.service +inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure /topicAndroidNew ${RESTRUCTURE_OUTPUT_DIR:-output}" $BASE/radar-output.service inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-check-health.service inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" $BASE/radar-check-health.service diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/lock-util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/lock-util.sh deleted file mode 100644 index aff2ccef1..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/lock-util.sh +++ /dev/null @@ -1,41 +0,0 @@ -PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin - -# maximum file size in byte to rotate log -minimumsize=10000000 - -# current time -timestamp=$(date '+%Y-%m-%d %H:%M:%S'); - -# Write message in the log file -log_info() { - echo "$timestamp - $@" >> $logfile 2>&1 -} - -# Remove old lock -checkLock() { - uptime=$( $logfile - fi -} - -# Entry point -touch $logfile -log_info "### $timestamp ###" -log_info "Checking lock ..." -checkLock diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 151ac412f..238a3e40c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -10,6 +10,7 @@ check_command_exists docker-compose # Initialize and check all config files check_config_present .env etc/env.template +check_config_present etc/smtp.env check_config_present etc/radar-backend/radar.yml check_config_present etc/managementportal/config/oauth_client_details.csv check_config_present etc/redcap-integration/radar.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template index 4af86edb0..60a27325e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.timer.template @@ -10,7 +10,3 @@ Unit=radar-check-health.service [Install] WantedBy=default.target - - - - From 4e7ff6f026f21fdea1016d7df2aa6b06ee468b4b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 6 Sep 2018 12:37:57 +0200 Subject: [PATCH 114/194] Fix Mongo/HDFS initialization --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 8 ++++---- .../radar-cp-hadoop-stack/lib/perform-install.sh | 9 +++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 3ae8230a5..7f88ca2aa 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -388,7 +388,7 @@ services: healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m - timeout: 10s + timeout: 15s retries: 3 hdfs-datanode-2: @@ -416,7 +416,7 @@ services: healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m - timeout: 10s + timeout: 15s retries: 3 hdfs-datanode-3: @@ -444,7 +444,7 @@ services: healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m - timeout: 10s + timeout: 15s retries: 3 hdfs-namenode-1: @@ -470,7 +470,7 @@ services: healthcheck: test: ["CMD", "hdfs", "dfs", "-test", "-e", "/"] interval: 1m - timeout: 10s + timeout: 15s retries: 3 #---------------------------------------------------------------------------# diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 238a3e40c..8be44370c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -86,9 +86,14 @@ ensure_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sin ensure_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties ensure_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties +KAFKA_INIT_OPTS=( + --rm -v "$PWD/etc/schema:/schema/conf" + radarbase/kafka-init:0.3.6 + ) + # Set topics if [ -z "${COMBINED_AGG_TOPIC_LIST}"]; then - COMBINED_AGG_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_aggregated.sh 2>/dev/null | tail -n 1) + COMBINED_AGG_TOPIC_LIST=$(sudo-linux docker run "${KAFKA_INIT_OPTS[@]}" list_aggregated.sh 2>/dev/null | tail -n 1) if [ -n "${RADAR_AGG_TOPIC_LIST}" ]; then COMBINED_AGG_TOPIC_LIST="${RADAR_AGG_TOPIC_LIST},${COMBINED_AGG_TOPIC_LIST}" fi @@ -97,7 +102,7 @@ ensure_variable 'topics=' "${COMBINED_AGG_TOPIC_LIST}" etc/mongodb-connector/sin echo "==> Configuring HDFS Connector" if [ -z "${COMBINED_RAW_TOPIC_LIST}"]; then - COMBINED_RAW_TOPIC_LIST=$(sudo-linux docker run --rm radarcns/kafka-init list_raw.sh 2>/dev/null | tail -n 1) + COMBINED_RAW_TOPIC_LIST=$(sudo-linux docker run "${KAFKA_INIT_OPTS[@]}" list_raw.sh 2>/dev/null | tail -n 1) if [ -n "${RADAR_RAW_TOPIC_LIST}" ]; then COMBINED_RAW_TOPIC_LIST="${RADAR_RAW_TOPIC_LIST},${COMBINED_RAW_TOPIC_LIST}" fi From ee6897291ea0f073e20693683a983f63a0803ad3 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 6 Sep 2018 13:48:48 +0200 Subject: [PATCH 115/194] Updated README --- dcompose-stack/radar-cp-hadoop-stack/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 6a781d56d..51f3e114a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -58,7 +58,7 @@ To enable a `systemd` service to control the platform, run ```shell bin/radar-docker install-systemd ``` -After that command, the RADAR platform should be controlled via `systemctl`. +After that command, the RADAR platform should be controlled via `systemctl`. When running as a user without `sudo` rights, in the following commands replace `sudo systemctl` with `systemctl --user`. ```shell # query the latest status and logs sudo systemctl status radar-docker From 706b9023b79e8ab98dc33716ff34a7e2e9bed7ce Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 6 Sep 2018 14:02:43 +0200 Subject: [PATCH 116/194] Attempt to fix travis environment --- .../radar-cp-hadoop-stack/travis/setup-env.sh | 2 +- .../travis/travis-env.template | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index d69aadf7e..c9c238336 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -2,7 +2,7 @@ echo "Starting to configure mock configurations for test" # create folder for docker volumes -sudo mkdir -p /usr/local/var/lib/docker/ +mkdir -p /home/travis/data # setup mock configs cp ./travis-env.template ../.env diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 123fa4837..99d5321cc 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -8,8 +8,8 @@ RADAR_AGG_TOPIC_LIST= HOTSTORAGE_USERNAME=mongodb-user HOTSTORAGE_PASSWORD=mongo-pwd HOTSTORAGE_NAME=mongodb-database -MONGODB_DIR=/usr/local/var/lib/docker/mongodb -MP_POSTGRES_DIR=/usr/local/var/lib/docker/postgres +MONGODB_DIR=/home/travis/data/mongodb +MP_POSTGRES_DIR=/home/travis/data/postgres POSTGRES_USER=postgresdb-user POSTGRES_PASSWORD=postgresdb-pwd FROM_EMAIL=from-email @@ -18,14 +18,14 @@ HEALTHCHECK_HIPCHAT_NOTIFY=no HEALTHCHECK_HIPCHAT_ROOM_ID="" HEALTHCHECK_HIPCHAT_TOKEN="" HDFS_BASE_VERSION=3.0.3-alpine -HDFS_DATA_DIR_1=/usr/local/var/lib/docker/hdfs-data-1 -HDFS_DATA_DIR_2=/usr/local/var/lib/docker/hdfs-data-2 -HDFS_DATA_DIR_3=/usr/local/var/lib/docker/hdfs-data-3 -HDFS_NAME_DIR_1=/usr/local/var/lib/docker/hdfs-name-1 -HDFS_NAME_DIR_2=/usr/local/var/lib/docker/hdfs-name-2 -HDFS_JOURNAL_DIR_1=/usr/local/var/lib/docker/hdfs-journal-1 -HDFS_JOURNAL_DIR_2=/usr/local/var/lib/docker/hdfs-journal-2 -HDFS_JOURNAL_DIR_3=/usr/local/var/lib/docker/hdfs-journal-3 +HDFS_DATA_DIR_1=/home/travis/data/hdfs-data-1 +HDFS_DATA_DIR_2=/home/travis/data/hdfs-data-2 +HDFS_DATA_DIR_3=/home/travis/data/hdfs-data-3 +HDFS_NAME_DIR_1=/home/travis/data/hdfs-name-1 +HDFS_NAME_DIR_2=/home/travis/data/hdfs-name-2 +HDFS_JOURNAL_DIR_1=/home/travis/data/hdfs-journal-1 +HDFS_JOURNAL_DIR_2=/home/travis/data/hdfs-journal-2 +HDFS_JOURNAL_DIR_3=/home/travis/data/hdfs-journal-3 POSTGRES_VERSION=10.4-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD=kafkamanager-pwd From d55aa66a5fbfab976f84f6eb67f5e1f9a9438032 Mon Sep 17 00:00:00 2001 From: nivethika Date: Wed, 12 Sep 2018 17:08:33 +0200 Subject: [PATCH 117/194] add MP 0.5.0 to the stack --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 7 +++++-- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 2 ++ .../radar-cp-hadoop-stack/lib/perform-install.sh | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 59269a5b4..39af8f6a5 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -694,7 +694,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarbase/management-portal:0.4.1 + image: radarbase/management-portal:0.5.0 networks: - default - api @@ -710,11 +710,14 @@ services: SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER} SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD} MANAGEMENTPORTAL_MAIL_FROM: ${FROM_EMAIL} - MANAGEMENTPORTAL_MAIL_BASEURL: https://${SERVER_NAME}/managementportal + MANAGEMENTPORTAL_COMMON_BASEURL: https://${SERVER_NAME}/ + MANAGEMENTPORTAL_COMMON_MANAGEMENT_PORTAL_BASE_URL: https://${SERVER_NAME}/managementportal MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET: ${MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET} MANAGEMENTPORTAL_OAUTH_CLIENTS_FILE: /mp-includes/config/oauth_client_details.csv MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT: ${MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT} MANAGEMENTPORTAL_CATALOGUE_SERVER_SERVER_URL: http://catalog-server:9010/source-types + MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD: ${MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD} + MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL: ${MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL} SPRING_APPLICATION_JSON: '{"managementportal":{"oauth":{"checkingKeyAliases":["${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0}","${MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1}"]}}}' JHIPSTER_SLEEP: 10 # gives time for the database to boot before the application JAVA_OPTS: -Xmx256m # maximum heap size for the JVM running ManagementPortal, increase this as necessary diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index f4822837d..4cd22963a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -33,3 +33,5 @@ PORTAINER_PASSWORD_HASH= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1= RESTRUCTURE_OUTPUT_DIR=output +MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD= +MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 8be44370c..66483f9b6 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -110,6 +110,7 @@ fi ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties echo "==> Configuring Management Portal" +ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." bin/keystore-init From f298fd34b75c0ce251de3e9aed2e657a295515d5 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 13 Sep 2018 11:52:57 +0200 Subject: [PATCH 118/194] upgrade gateway to 0.3.0 --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 39af8f6a5..588e0804c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -775,7 +775,7 @@ services: # RADAR Gateway # #---------------------------------------------------------------------------# gateway: - image: radarbase/radar-gateway:0.2.2 + image: radarbase/radar-gateway:0.3.0 networks: - api - kafka From 4e5037c964a0b266b8178cf8eccc48d878cb3749 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 13 Sep 2018 11:55:55 +0200 Subject: [PATCH 119/194] update travis env with mandatory password var --- dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 99d5321cc..40379e43b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -30,3 +30,5 @@ POSTGRES_VERSION=10.4-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD=kafkamanager-pwd PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 +MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD=notadmin +MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy \ No newline at end of file From 04bdec94d11771ece3366f179040fb53965e9f23 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 13 Sep 2018 12:26:46 +0200 Subject: [PATCH 120/194] update gateway.yml --- dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml index 99993dead..bcada2f56 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/gateway/gateway.yml @@ -1,3 +1,4 @@ baseUri: http://0.0.0.0/radar-gateway/ managementPortalUrl: http://managementportal-app:8080/managementportal restProxyUrl: http://rest-proxy-1:8082 +schemaRegistryUrl: http://schema-registry-1:8081 From 01e9daaaa7642c24ff9caba481f8956a7a51a9a9 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 13 Sep 2018 12:47:51 +0200 Subject: [PATCH 121/194] update rest-api 0.3 and add radar-is.yml for rest-api --- .../radar-cp-hadoop-stack/bin/keystore-init | 24 +++++++++++++++++++ .../radar-cp-hadoop-stack/docker-compose.yml | 4 +++- .../lib/perform-install.sh | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init index 1aba031f3..fff66133e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init @@ -44,3 +44,27 @@ for (( i=0; i < ${#ALIASES[@]}; i++)); do done <<< "$cert" done + +REST_API_CONF=etc/rest-api/radar-is.yml +echo "--> Updating Rest-API signature keys" +echo "resourceName: res_RestApi" > $REST_API_CONF +echo "publicKeys:" >> $REST_API_CONF + +ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) +for (( i=0; i < ${#ALIASES[@]}; i++)); do + ALIAS=${ALIASES[$i]} + ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env + echo " - |-" >> $REST_API_CONF + if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then + REPLACE_PUBKEY="EC PUBLIC KEY" + else + REPLACE_PUBKEY="PUBLIC KEY" + fi + + cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" + while IFS='' read -r line && [ -n "$line" ]; do + line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) + echo " $line" >> $REST_API_CONF + done <<< "$cert" +done + diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 588e0804c..e927445d3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -304,7 +304,7 @@ services: # RADAR REST API # #---------------------------------------------------------------------------# rest-api: - image: radarbase/radar-restapi:0.2.1 + image: radarbase/radar-restapi:0.3 networks: - hotstorage - api @@ -315,6 +315,8 @@ services: restart: always volumes: - "./etc/rest-api:/usr/local/conf/radar/rest-api" + environment: + RADAR_IS_CONFIG_LOCATION: usr/local/conf/radar/rest-api/radar-is.yml healthcheck: test: ["CMD", "wget", "--spider", "http://localhost:8080/api/openapi.json"] interval: 1m diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 66483f9b6..293e8eaf8 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -110,6 +110,7 @@ fi ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties echo "==> Configuring Management Portal" +ensure_env_password MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET "ManagementPortal front-end client secret is not set in .env" ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." bin/keystore-init From d3ce39ffc47d51681ee08a8db8a6d7492e18df91 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 13 Sep 2018 15:13:06 +0200 Subject: [PATCH 122/194] update radar-backend 0.4.0 and update radar.yml with config --- .../radar-cp-hadoop-stack/docker-compose.yml | 4 +- .../etc/radar-backend/radar.yml.template | 83 +++++++++++++------ 2 files changed, 59 insertions(+), 28 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index e927445d3..e31c67d43 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -595,7 +595,7 @@ services: # RADAR backend streams # #---------------------------------------------------------------------------# radar-backend-stream: - image: radarbase/radar-backend:0.2.3 + image: radarbase/radar-backend:0.4.0 command: - stream networks: @@ -622,7 +622,7 @@ services: # RADAR backend monitor # #---------------------------------------------------------------------------# radar-backend-monitor: - image: radarbase/radar-backend:0.2.3 + image: radarbase/radar-backend:0.4.0 command: monitor networks: - zookeeper diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index 5a116cf4d..484e1efd2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -1,9 +1,5 @@ version: 1.0 -released: 2016-11-27 - -#============================= Application =============================# -#Possible value are standalone or high_performance -mode: standalone +released: 2018-09-13 #============================== Zookeeper ==============================# #List of Zookeeper instances @@ -17,33 +13,73 @@ broker: - host: kafka-1 port: 9092 -#Kafka internal parameters -stream_properties: - max.request.size: 3500042 #Set message.max.bytes for kafka brokers higher than or equal to this value - retries: 15 - session.timeout.ms: 500000 - commit.interval.ms: 300000 - request.timeout.ms: 10860000 - state.cleanup.delay.ms: 10860000 -#============================ Kafka Streams ============================# -#The number of threads that a stream must be run according is priority -stream_priority: - low: 1 - normal: 2 - high: 4 - #=========================== Schema Registry ===========================# #List of Schema Registry instances schema_registry: - host: schema-registry-1 port: 8081 protocol: http - +#=========================== Kafka REST Proxy ===========================# +#List of kafka Rest proxy instances rest_proxy: host: rest-proxy-1 port: 8082 protocol: http +#Kafka internal parameters +#============================ Kafka Streams ============================# +#The number of threads that a stream must be run according is priority +stream: + stream_priority: + low: 1 + normal: 2 + high: 4 + + properties: + max.request.size: 3500042 #Set message.max.bytes for kafka brokers higher than or equal to this value + retries: 15 + session.timeout.ms: 500000 + commit.interval.ms: 300000 + request.timeout.ms: 10860000 + state.cleanup.delay.ms: 10860000 + + streams: + - class: org.radarcns.stream.empatica.E4AccelerationStream + - class: org.radarcns.stream.empatica.E4BatteryLevelStream + - class: org.radarcns.stream.empatica.E4BloodVolumePulseStream + - class: org.radarcns.stream.empatica.E4ElectroDermalActivityStream + - class: org.radarcns.stream.empatica.E4HeartRateStream + - class: org.radarcns.stream.empatica.E4InterBeatIntervalStream + - class: org.radarcns.stream.empatica.E4TemperatureStream + - class: org.radarcns.stream.phone.PhoneAccelerationStream + - class: org.radarcns.stream.phone.PhoneBatteryStream + - class: org.radarcns.stream.phone.PhoneUsageStream + - class: org.radarcns.stream.phone.PhoneUsageAggregationStream + + + #====================== Source statistics monitor ======================# + source_statistics: + - name: Empatica E4 + topics: + - android_empatica_e4_blood_volume_pulse_1min + output_topic: source_statistics_empatica_e4 + - name: Biovotion VSM1 + topics: + - android_biovotion_vsm1_acceleration_1min + output_topic: source_statistics_biovotion_vsm1 + - name: RADAR pRMT + topics: + - android_phone_acceleration_1min + - android_phone_bluetooth_devices + - android_phone_sms + - android_phone_call + - android_phone_contacts + - android_phone_usage_event + - android_phone_relative_location + output_topic: source_statistics_android_phone + + + #======================== Battery level monitor ========================# battery_monitor: level: LOW @@ -98,9 +134,4 @@ statistics_monitors: - android_phone_relative_location output_topic: source_statistics_android_phone -#========================= Kafka stream ================================# -stream_masters: - - org.radarcns.stream.empatica.E4StreamMaster - - org.radarcns.stream.phone.PhoneStreamMaster - persistence_path: /var/lib/radar/data From 5c6bf55023b83591ac30955adbfebe207d823487 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Mon, 17 Sep 2018 11:40:27 +0100 Subject: [PATCH 123/194] Add fitbit source connector to the stack --- .../radar-cp-hadoop-stack/README.md | 20 ++++++--- .../radar-cp-hadoop-stack/docker-compose.yml | 44 +++++++++++++++++++ .../radar-cp-hadoop-stack/etc/env.template | 2 + .../docker/source-fitbit.properties.template | 8 ++++ .../docker/users/fitbit-user.yml.template | 24 ++++++++++ .../lib/perform-install.sh | 5 +++ .../travis/travis-env.template | 2 + .../travis/travis-smtp.template | 2 +- 8 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 51f3e114a..191fdc338 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -4,6 +4,9 @@ This docker-compose stack contains the full operational RADAR platform. Once con ## Configuration +### Required +This is the set of minimal configuration required to run the stack. + 1. First copy `etc/env.template` file to `./.env` and check and modify all its variables. @@ -21,18 +24,21 @@ This docker-compose stack contains the full operational RADAR platform. Once con 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. -5. (Optional) Next copy the `etc/webserver/ip-access-control.conf.template` to `etc/webserver/ip-access-control.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two examples are included in the comments. By default all IPs are allowed. +5. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. + +### Optional +This is a set of optional configuration which is not required but could be useful. -6. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. +1. For added security, copy the `etc/webserver/ip-access-control.conf.template` to `etc/webserver/ip-access-control.conf` and configure restriction of admin tools (like portainer and kafka-manager) to certain known IP addresses. For easy configuration two examples are included in the comments. By default all IPs are allowed. -7. (Optional) Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, +2. Note: To have different flush.size for different topics, you can create multipe property configurations for a single connector. To do that, - 7.1 Create multiple property files that have different `flush.size` for given topics. + 2.1 Create multiple property files that have different `flush.size` for given topics. Examples [sink-hdfs-high.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-high.properties) , [sink-hdfs-low.properties](https://github.com/RADAR-base/RADAR-Docker/blob/dev/dcompose-stack/radar-cp-hadoop-stack/etc/sink-hdfs-low.properties) - 7.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` environment variable to `radar-hdfs-connector` service in `docker-compose` file. + 2.2 Add `CONNECTOR_PROPERTY_FILE_PREFIX: ` environment variable to `radar-hdfs-connector` service in `docker-compose` file. - 7.3 Add created property files to the `radar-hdfs-connector` service in `docker-compose` with name abides to prefix-value mentioned in `CONNECTOR_PROPERTY_FILE_PREFIX` + 2.3 Add created property files to the `radar-hdfs-connector` service in `docker-compose` with name abides to prefix-value mentioned in `CONNECTOR_PROPERTY_FILE_PREFIX` ```ini radar-hdfs-connector: @@ -46,6 +52,8 @@ This docker-compose stack contains the full operational RADAR platform. Once con CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" ``` +3. If using the Fitbit Connector, please specify the `FITBIT_API_CLIENT_ID` and `FITBIT_API_CLIENT_SECRET` in the .env file. Then copy the `etc/fitbit/docker/users/fitbit-user.yml.template` to `etc/fitbit/docker/users/fitbit-user.yml` and fill out all the details of the fitbit user. If multiple users, then for each user create a separate file in the `etc/fitbit/docker/users/` directory containing all the fields as in the template. For more information about users configuration for fitbit, read [here](https://github.com/RADAR-base/RADAR-REST-Connector#usage). + ## Usage Run diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 59269a5b4..7f47fcdfa 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -41,6 +41,7 @@ volumes: external: true certs-data: external: true + fitbit-logs: {} services: #---------------------------------------------------------------------------# @@ -812,3 +813,46 @@ services: interval: 1m30s timeout: 5s retries: 3 + + #---------------------------------------------------------------------------# + # RADAR Fitbit connector # + #---------------------------------------------------------------------------# + radar-fitbit-connector: + image: radarbase/kafka-connect-rest-fitbit-source:0.1.0 + restart: on-failure + volumes: + - ./etc/fitbit/docker/source-fitbit.properties:/etc/kafka-connect/source-fitbit.properties + - ./etc/fitbit/docker/users:/var/lib/kafka-connect-fitbit-source/users + - fitbit-logs:/var/lib/kafka-connect-fitbit-source/logs + networks: + - zookeeper + - kafka + - default + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "default" + CONNECT_CONFIG_STORAGE_TOPIC: "default.config" + CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" + CONNECT_STATUS_STORAGE_TOPIC: "default.status" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-fitbit-source/logs/connect.offsets" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-fitbit-connector" + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + CONNECTOR_PROPERTY_FILE_PREFIX: "source-fitbit" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index f4822837d..52d684847 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -33,3 +33,5 @@ PORTAINER_PASSWORD_HASH= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1= RESTRUCTURE_OUTPUT_DIR=output +FITBIT_API_CLIENT_ID=fitbit-client +FITBIT_API_CLIENT_SECRET=fitbit-secret diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template new file mode 100644 index 000000000..631321bb2 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template @@ -0,0 +1,8 @@ +name=radar-fitbit-source +connector.class=org.radarbase.connect.rest.fitbit.FitbitSourceConnector +tasks.max=4 +rest.source.base.url=https://api.fitbit.com +rest.source.poll.interval.ms=5000 +rest.source.request.generator.class=org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator +fitbit.api.client= +fitbit.api.secret= diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template new file mode 100644 index 000000000..56f4eeea6 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template @@ -0,0 +1,24 @@ +--- +# Unique user key +id: test +# Project ID to be used in org.radarcns.kafka.ObservationKey record keys +projectId: radar-test +# User ID to be used in org.radarcns.kafka.ObservationKey record keys +userId: test +# Source ID to be used in org.radarcns.kafka.ObservationKey record keys +sourceId: charge-2 +# Date from when to collect data. +startDate: 2018-08-06T00:00:00Z +# Date until when to collect data. +endDate: 2019-01-01T00:00:00Z +# Fitbit user ID as returned by the Fitbit authentication procedure +fitbitUserId: ? +oauth2: + # Fitbit OAuth 2.0 access token as returned by the Fitbit authentication procedure + accessToken: ? + # Fitbit OAuth 2.0 refresh token as returned by the Fitbit authentication procedure + refreshToken: ? + # Optional expiry time of the access token. If absent, it will be estimated to one hour + # when the source connector starts. When an authentication error occurs, a new access token will + # be fetched regardless of the value in this field. + #expiresAt: 2018-08-06T00:00:00Z diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 8be44370c..6e575d699 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -19,6 +19,7 @@ copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml copy_template_if_absent etc/webserver/nginx.conf copy_template_if_absent etc/webserver/ip-access-control.conf +copy_template_if_absent etc/fitbit/docker/source-fitbit.properties # Set permissions sudo-linux chmod og-rw ./.env @@ -86,6 +87,10 @@ ensure_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sin ensure_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties ensure_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties +echo "==> Configuring Fitbit Connector" +ensure_variable 'fitbit.api.client=' $FITBIT_API_CLIENT_ID etc/fitbit/docker/source-fitbit.properties +ensure_variable 'fitbit.api.secret=' $FITBIT_API_CLIENT_SECRET etc/fitbit/docker/source-fitbit.properties + KAFKA_INIT_OPTS=( --rm -v "$PWD/etc/schema:/schema/conf" radarbase/kafka-init:0.3.6 diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 99d5321cc..0e5fd8a61 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -30,3 +30,5 @@ POSTGRES_VERSION=10.4-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD=kafkamanager-pwd PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 +FITBIT_API_CLIENT_ID=fitbit-client +FITBIT_API_CLIENT_SECRET=fitbit-secret diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template index 58a8cdb8b..ab7fe491d 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-smtp.template @@ -1,4 +1,4 @@ SMARTHOST_ADDRESS=mail.example.com SMARTHOST_PORT=587 SMARTHOST_USER=user@example.com -SMARTHOST_PASSWORD=XXXXXXXX \ No newline at end of file +SMARTHOST_PASSWORD=XXXXXXXX From 6fda2bd439ac4afb5089d8b7e8124fe25caa37b6 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Mon, 17 Sep 2018 12:08:46 +0100 Subject: [PATCH 124/194] Increase travis sleep time --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 70ac0db89..6f55760e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,4 @@ script: - cd ../radar-cp-hadoop-stack/travis - ./setup-env.sh - cd ../ - - bin/radar-docker install && sleep 60 && [ -z "$($DOCKER_COMPOSE ps | tail -n +3 | grep " Exit ")" ] + - bin/radar-docker install && sleep 120 && [ -z "$($DOCKER_COMPOSE ps | tail -n +3 | grep " Exit ")" ] From 6f5418d15992aa3468e3ddc13d59c9186088aa37 Mon Sep 17 00:00:00 2001 From: Denny Verbeeck Date: Thu, 20 Sep 2018 09:13:33 +0200 Subject: [PATCH 125/194] Remove OAuth secret from public clients --- .../config/oauth_client_details.csv.template | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template index ad005bea3..4aeafecf8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template @@ -1,7 +1,7 @@ client_id;resource_ids;client_secret;scope;authorized_grant_types;redirect_uri;authorities;access_token_validity;refresh_token_validity;additional_information;autoapprove -pRMT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; -aRMT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; -THINC-IT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +pRMT;res_ManagementPortal,res_gateway;;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +aRMT;res_ManagementPortal,res_gateway;;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +THINC-IT;res_ManagementPortal,res_gateway;;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; radar_restapi;res_ManagementPortal;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ;client_credentials;;;43200;259200;{}; radar_redcap_integrator;res_ManagementPortal;secret;PROJECT.READ,SUBJECT.CREATE,SUBJECT.READ,SUBJECT.UPDATE;client_credentials;;;43200;259200;{}; -radar_dashboard;res_ManagementPortal,res_RestApi;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,MEASUREMENT.READ;client_credentials;;;43200;259200;{}; +radar_dashboard;res_ManagementPortal,res_RestApi;;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,MEASUREMENT.READ;client_credentials;;;43200;259200;{}; From c32f80df9b4ddbba25c86a8c797d4ee34405c6fd Mon Sep 17 00:00:00 2001 From: Denny Verbeeck Date: Thu, 20 Sep 2018 10:44:41 +0200 Subject: [PATCH 126/194] Update dashboard allowed grant types --- .../managementportal/config/oauth_client_details.csv.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template index 4aeafecf8..89ad8e747 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/managementportal/config/oauth_client_details.csv.template @@ -4,4 +4,4 @@ aRMT;res_ManagementPortal,res_gateway;;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT THINC-IT;res_ManagementPortal,res_gateway;;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; radar_restapi;res_ManagementPortal;secret;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ;client_credentials;;;43200;259200;{}; radar_redcap_integrator;res_ManagementPortal;secret;PROJECT.READ,SUBJECT.CREATE,SUBJECT.READ,SUBJECT.UPDATE;client_credentials;;;43200;259200;{}; -radar_dashboard;res_ManagementPortal,res_RestApi;;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,MEASUREMENT.READ;client_credentials;;;43200;259200;{}; +radar_dashboard;res_ManagementPortal,res_RestApi;;SUBJECT.READ,PROJECT.READ,SOURCE.READ,SOURCETYPE.READ,MEASUREMENT.READ;refresh_token,authorization_code;;;43200;259200;{}; From 18ed3e96d0f148e5f2fcd2243756d35b53510f6e Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 21 Sep 2018 13:56:48 +0100 Subject: [PATCH 127/194] Add optional services and config. Add healthcheck for fitbit connector. --- .travis.yml | 2 +- .../radar-cp-hadoop-stack/README.md | 8 ++- .../radar-cp-hadoop-stack/bin/radar-docker | 9 ++- .../radar-cp-hadoop-stack/docker-compose.yml | 65 +---------------- .../radar-cp-hadoop-stack/etc/.gitignore | 2 + .../radar-cp-hadoop-stack/etc/env.template | 1 + .../lib/perform-install.sh | 11 +-- .../optional-services.yml | 70 +++++++++++++++++++ 8 files changed, 95 insertions(+), 73 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/optional-services.yml diff --git a/.travis.yml b/.travis.yml index 6f55760e6..9df4ac878 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,4 @@ script: - cd ../radar-cp-hadoop-stack/travis - ./setup-env.sh - cd ../ - - bin/radar-docker install && sleep 120 && [ -z "$($DOCKER_COMPOSE ps | tail -n +3 | grep " Exit ")" ] + - bin/radar-docker install && sleep 120 && $DOCKER_COMPOSE ps && [ -z "$($DOCKER_COMPOSE ps | tail -n +3 | grep " Exit ")" ] diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 191fdc338..53987200a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -20,8 +20,6 @@ This is the set of minimal configuration required to run the stack. 2. Copy `etc/smtp.env.template` to `etc/smtp.env` and configure your email settings. Configure alternative mail providers like Amazon SES or Gmail by using the parameters of the [`namshi/smtp` Docker image](https://hub.docker.com/r/namshi/smtp/). -3. Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` - 4. Copy `etc/managementportal/config/oauth_client_details.csv.template` to `etc/managementportal/config/oauth_client_details.csv` and change OAuth client credentials for production MP. The OAuth client for the frontend will be loaded automatically and does not need to be listed in this file. This file will be read at each startup. The current implementation overwrites existing clients with the same client ID, so be aware of this if you have made changes to a client listed in this file using the Management Portal frontend. This behaviour might change in the future. 5. Finally, copy `etc/radar-backend/radar.yml.template` to `etc/radar-backend/radar.yml` and edit it, especially concerning the monitor email address configuration. @@ -52,7 +50,11 @@ This is a set of optional configuration which is not required but could be usefu CONNECTOR_PROPERTY_FILE_PREFIX: "sink-hdfs" ``` -3. If using the Fitbit Connector, please specify the `FITBIT_API_CLIENT_ID` and `FITBIT_API_CLIENT_SECRET` in the .env file. Then copy the `etc/fitbit/docker/users/fitbit-user.yml.template` to `etc/fitbit/docker/users/fitbit-user.yml` and fill out all the details of the fitbit user. If multiple users, then for each user create a separate file in the `etc/fitbit/docker/users/` directory containing all the fields as in the template. For more information about users configuration for fitbit, read [here](https://github.com/RADAR-base/RADAR-REST-Connector#usage). +3. To enable optional services, please set the `ENABLE_OPTIONAL_SERVICES` parameter in `.env` file to `true`. By default optional services are disabled. You can check which service are optional in the file `optional-services.yml` + + 3.1 Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` + + 3.2 For the Fitbit Connector, please specify the `FITBIT_API_CLIENT_ID` and `FITBIT_API_CLIENT_SECRET` in the .env file. Then copy the `etc/fitbit/docker/users/fitbit-user.yml.template` to `etc/fitbit/docker/users/fitbit-user.yml` and fill out all the details of the fitbit user. If multiple users, then for each user create a separate file in the `etc/fitbit/docker/users/` directory containing all the fields as in the template. For more information about users configuration for fitbit, read [here](https://github.com/RADAR-base/RADAR-REST-Connector#usage). ## Usage diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker index ec5f2ecfb..92d6a60d7 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-docker @@ -1,7 +1,14 @@ #!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")/.." -stack=${DOCKER_COMPOSE:-docker-compose} +. .env + +if [[ "${ENABLE_OPTIONAL_SERVICES}" = "true" ]]; then + stack="${DOCKER_COMPOSE:-docker-compose} -f docker-compose.yml -f optional-services.yml" +else + stack=${DOCKER_COMPOSE:-docker-compose} +fi + function print_usage() { echo 'Control the RADAR-base platform directly' diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 7f47fcdfa..99c5b7a01 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -1,5 +1,5 @@ --- -version: '2.4' +version: '2.2' networks: zookeeper: @@ -322,26 +322,6 @@ services: timeout: 5s retries: 3 - - #---------------------------------------------------------------------------# - # RADAR REDCap Integration # - #---------------------------------------------------------------------------# - radar-integration: - image: radarbase/radar-redcapintegration:latest - networks: - - api - - default - depends_on: - - managementportal-app - restart: always - volumes: - - "./etc/redcap-integration:/usr/local/tomcat/conf/radar" - healthcheck: - test: ["CMD", "curl", "-IX", "POST", "http://localhost:8080/redcap/trigger"] - interval: 1m - timeout: 5s - retries: 3 - #---------------------------------------------------------------------------# # RADAR Dashboard # #---------------------------------------------------------------------------# @@ -813,46 +793,3 @@ services: interval: 1m30s timeout: 5s retries: 3 - - #---------------------------------------------------------------------------# - # RADAR Fitbit connector # - #---------------------------------------------------------------------------# - radar-fitbit-connector: - image: radarbase/kafka-connect-rest-fitbit-source:0.1.0 - restart: on-failure - volumes: - - ./etc/fitbit/docker/source-fitbit.properties:/etc/kafka-connect/source-fitbit.properties - - ./etc/fitbit/docker/users:/var/lib/kafka-connect-fitbit-source/users - - fitbit-logs:/var/lib/kafka-connect-fitbit-source/logs - networks: - - zookeeper - - kafka - - default - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 - - kafka-1 - - kafka-2 - - kafka-3 - - schema-registry-1 - environment: - CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 - CONNECT_REST_PORT: 8083 - CONNECT_GROUP_ID: "default" - CONNECT_CONFIG_STORAGE_TOPIC: "default.config" - CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" - CONNECT_STATUS_STORAGE_TOPIC: "default.status" - CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" - CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" - CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-fitbit-source/logs/connect.offsets" - CONNECT_REST_ADVERTISED_HOST_NAME: "radar-fitbit-connector" - CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 - CONNECTOR_PROPERTY_FILE_PREFIX: "source-fitbit" - KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" - KAFKA_BROKERS: 3 - CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index d55a08afb..b5fcd2da4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -14,3 +14,5 @@ /schema/commons/ /schema/specifications/ /gateway/radar-is.yml +/fitbit/docker/users/*.yml +/fitbit/docker/source-fitbit.properties diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 52d684847..b0f618570 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -33,5 +33,6 @@ PORTAINER_PASSWORD_HASH= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1= RESTRUCTURE_OUTPUT_DIR=output +ENABLE_OPTIONAL_SERVICES=false FITBIT_API_CLIENT_ID=fitbit-client FITBIT_API_CLIENT_SECRET=fitbit-secret diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 6e575d699..5d4d4f566 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -87,10 +87,6 @@ ensure_variable 'mongo.username=' $HOTSTORAGE_USERNAME etc/mongodb-connector/sin ensure_variable 'mongo.password=' $HOTSTORAGE_PASSWORD etc/mongodb-connector/sink-mongo.properties ensure_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mongo.properties -echo "==> Configuring Fitbit Connector" -ensure_variable 'fitbit.api.client=' $FITBIT_API_CLIENT_ID etc/fitbit/docker/source-fitbit.properties -ensure_variable 'fitbit.api.secret=' $FITBIT_API_CLIENT_SECRET etc/fitbit/docker/source-fitbit.properties - KAFKA_INIT_OPTS=( --rm -v "$PWD/etc/schema:/schema/conf" radarbase/kafka-init:0.3.6 @@ -133,6 +129,13 @@ inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx. sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf init_certificate "${SERVER_NAME}" +# Configure Optional services +if [[ "${ENABLE_OPTIONAL_SERVICES}" = "true" ]]; then + echo "==> Configuring Fitbit Connector" + ensure_variable 'fitbit.api.client=' $FITBIT_API_CLIENT_ID etc/fitbit/docker/source-fitbit.properties + ensure_variable 'fitbit.api.secret=' $FITBIT_API_CLIENT_SECRET etc/fitbit/docker/source-fitbit.properties +fi + echo "==> Starting RADAR-base Platform" sudo-linux bin/radar-docker up -d --remove-orphans "$@" diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml new file mode 100644 index 000000000..d9052b156 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -0,0 +1,70 @@ +--- +version: '2.2' + +services: + #---------------------------------------------------------------------------# + # RADAR REDCap Integration # + #---------------------------------------------------------------------------# + radar-integration: + image: radarbase/radar-redcapintegration:latest + networks: + - api + - default + depends_on: + - managementportal-app + restart: always + volumes: + - "./etc/redcap-integration:/usr/local/tomcat/conf/radar" + healthcheck: + test: ["CMD", "curl", "-IX", "POST", "http://localhost:8080/redcap/trigger"] + interval: 1m + timeout: 5s + retries: 3 + + #---------------------------------------------------------------------------# + # RADAR Fitbit connector # + #---------------------------------------------------------------------------# + radar-fitbit-connector: + image: radarbase/kafka-connect-rest-fitbit-source:0.1.0 + restart: on-failure + volumes: + - ./etc/fitbit/docker/source-fitbit.properties:/etc/kafka-connect/source-fitbit.properties + - ./etc/fitbit/docker/users:/var/lib/kafka-connect-fitbit-source/users + - fitbit-logs:/var/lib/kafka-connect-fitbit-source/logs + networks: + - zookeeper + - kafka + - default + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "default" + CONNECT_CONFIG_STORAGE_TOPIC: "default.config" + CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" + CONNECT_STATUS_STORAGE_TOPIC: "default.status" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-fitbit-source/logs/connect.offsets" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-fitbit-connector" + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + CONNECTOR_PROPERTY_FILE_PREFIX: "source-fitbit" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" + healthcheck: + test: ["CMD-SHELL", "curl -sf localhost:8083/connectors/radar-fitbit-source/status | grep -o '\"state\":\"[^\"]*\"' | tr '\\n' ',' | grep -vq FAILED || exit 1"] + interval: 1m + timeout: 5s + retries: 3 From 32d8ce5c4a03c8deb8c9c28df7248cef13478908 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 21 Sep 2018 13:57:33 +0100 Subject: [PATCH 128/194] Redcap integration as optional and fixes for latest version --- dcompose-stack/radar-cp-hadoop-stack/optional-services.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index d9052b156..62c50e098 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -6,7 +6,7 @@ services: # RADAR REDCap Integration # #---------------------------------------------------------------------------# radar-integration: - image: radarbase/radar-redcapintegration:latest + image: radarbase/radar-redcapintegration:0.1.0 networks: - api - default @@ -14,9 +14,9 @@ services: - managementportal-app restart: always volumes: - - "./etc/redcap-integration:/usr/local/tomcat/conf/radar" + - "./etc/redcap-integration:/usr/local/etc/radar-redcap-int" healthcheck: - test: ["CMD", "curl", "-IX", "POST", "http://localhost:8080/redcap/trigger"] + test: ["CMD-SHELL", "wget --spider http://localhost:8080/redcap/trigger 2>&1 | grep -q 500 || exit 1"] interval: 1m timeout: 5s retries: 3 From 65af447387b9de89a8dbb0231098f4740946a52d Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 21 Sep 2018 14:26:48 +0100 Subject: [PATCH 129/194] Make webserver independent of the optional services --- dcompose-stack/radar-cp-hadoop-stack/README.md | 2 +- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- .../radar-cp-hadoop-stack/etc/webserver/nginx.conf.template | 5 +---- .../etc/webserver/optional-services.conf.template | 5 +++++ dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 1 + .../radar-cp-hadoop-stack/travis/travis-env.template | 1 + 6 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 53987200a..927f4e131 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -52,7 +52,7 @@ This is a set of optional configuration which is not required but could be usefu 3. To enable optional services, please set the `ENABLE_OPTIONAL_SERVICES` parameter in `.env` file to `true`. By default optional services are disabled. You can check which service are optional in the file `optional-services.yml` - 3.1 Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger` + 3.1 Copy `etc/redcap-integration/radar.yml.template` to `etc/redcap-integration/radar.yml` and modify it to configure the properties of Redcap instance and the management portal. For reference on configuration of this file look at the Readme file here - . In the REDcap portal under Project Setup, define the Data Trigger as `https:///redcapint/trigger`. Also need to configure the webserver config, just uncomment the location block at `etc/webserver/optional-services.conf.template` and copy it to `etc/webserver/optional-services.conf`. 3.2 For the Fitbit Connector, please specify the `FITBIT_API_CLIENT_ID` and `FITBIT_API_CLIENT_SECRET` in the .env file. Then copy the `etc/fitbit/docker/users/fitbit-user.yml.template` to `etc/fitbit/docker/users/fitbit-user.yml` and fill out all the details of the fitbit user. If multiple users, then for each user create a separate file in the `etc/fitbit/docker/users/` directory containing all the fields as in the template. For more information about users configuration for fitbit, read [here](https://github.com/RADAR-base/RADAR-REST-Connector#usage). diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 99c5b7a01..951966504 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -654,7 +654,6 @@ services: - schema-registry-1 - gateway - dashboard - - radar-integration - managementportal-app - kafka-manager ports: @@ -667,6 +666,7 @@ services: - "./etc/webserver/cors.conf:/etc/nginx/cors.conf:ro" - "./etc/webserver/ip-access-control.conf:/etc/nginx/ip-access-control.conf:ro" - "./etc/webserver/kafka-manager.htpasswd:/etc/nginx/kafka-manager.htpasswd:ro" + - "./etc/webserver/optional-services.conf:/etc/nginx/optional-services.conf" # healthcheck hard to do, however, it is possible to monitor this externally # with # docker logs --since 2m radarcphadoopstack_webserver_1 | grep "connect() failed" diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index b72e1422b..43bf46f39 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -108,10 +108,6 @@ http { proxy_pass http://rest-api:8080/api/; proxy_set_header Host $host; } - location /redcapint/ { - proxy_pass http://radar-integration:8080/redcap/; - proxy_set_header Host $host; - } location /managementportal/ { include cors.conf; proxy_pass http://managementportal-app:8080/managementportal/; @@ -138,5 +134,6 @@ http { proxy_pass http://kafka-manager:9000; proxy_set_header Host $host; } + include optional-services.conf; } } diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template new file mode 100644 index 000000000..ce1032d97 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template @@ -0,0 +1,5 @@ +# uncomment and add according to which optional services need to use the webserver. +#location /redcapint/ { +# proxy_pass http://radar-integration:8080/redcap/; +# proxy_set_header Host $host; +#} diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 5d4d4f566..a7c06e68e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -19,6 +19,7 @@ copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml copy_template_if_absent etc/webserver/nginx.conf copy_template_if_absent etc/webserver/ip-access-control.conf +copy_template_if_absent etc/webserver/optional-services.conf copy_template_if_absent etc/fitbit/docker/source-fitbit.properties # Set permissions diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 0e5fd8a61..dba4143fb 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -30,5 +30,6 @@ POSTGRES_VERSION=10.4-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD=kafkamanager-pwd PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 +ENABLE_OPTIONAL_SERVICES=true FITBIT_API_CLIENT_ID=fitbit-client FITBIT_API_CLIENT_SECRET=fitbit-secret From 66b7f4db1ae72e6e94f20d75c0716ede29af6ad8 Mon Sep 17 00:00:00 2001 From: nivethika Date: Fri, 21 Sep 2018 16:19:36 +0200 Subject: [PATCH 130/194] update MP to 0.5.1 --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index e31c67d43..a444652de 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -328,7 +328,7 @@ services: # RADAR REDCap Integration # #---------------------------------------------------------------------------# radar-integration: - image: radarbase/radar-redcapintegration:latest + image: radarbase/radar-redcapintegration:0.1.0 networks: - api - default @@ -338,7 +338,7 @@ services: volumes: - "./etc/redcap-integration:/usr/local/tomcat/conf/radar" healthcheck: - test: ["CMD", "curl", "-IX", "POST", "http://localhost:8080/redcap/trigger"] + test: ["CMD", "wget", "-IX", "POST", "http://localhost:8080/redcap/trigger"] interval: 1m timeout: 5s retries: 3 @@ -696,7 +696,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarbase/management-portal:0.5.0 + image: radarbase/management-portal:0.5.1 networks: - default - api From 0af17ce3da3179825ab6c636d21b827efe981a9c Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 21 Sep 2018 18:09:58 +0100 Subject: [PATCH 131/194] More separation of optional services --- .travis.yml | 2 +- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 1 - dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 3 ++- dcompose-stack/radar-cp-hadoop-stack/optional-services.yml | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9df4ac878..b11bc3478 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,4 @@ script: - cd ../radar-cp-hadoop-stack/travis - ./setup-env.sh - cd ../ - - bin/radar-docker install && sleep 120 && $DOCKER_COMPOSE ps && [ -z "$($DOCKER_COMPOSE ps | tail -n +3 | grep " Exit ")" ] + - bin/radar-docker install && sleep 120 && bin/radar-docker ps && [ -z "$(bin/radar-docker ps | tail -n +3 | grep " Exit ")" ] diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 951966504..ab81f8a12 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -41,7 +41,6 @@ volumes: external: true certs-data: external: true - fitbit-logs: {} services: #---------------------------------------------------------------------------# diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index a7c06e68e..ab76541d7 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -13,7 +13,6 @@ check_config_present .env etc/env.template check_config_present etc/smtp.env check_config_present etc/radar-backend/radar.yml check_config_present etc/managementportal/config/oauth_client_details.csv -check_config_present etc/redcap-integration/radar.yml copy_template_if_absent etc/mongodb-connector/sink-mongo.properties copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml @@ -135,6 +134,8 @@ if [[ "${ENABLE_OPTIONAL_SERVICES}" = "true" ]]; then echo "==> Configuring Fitbit Connector" ensure_variable 'fitbit.api.client=' $FITBIT_API_CLIENT_ID etc/fitbit/docker/source-fitbit.properties ensure_variable 'fitbit.api.secret=' $FITBIT_API_CLIENT_SECRET etc/fitbit/docker/source-fitbit.properties + + check_config_present etc/redcap-integration/radar.yml fi echo "==> Starting RADAR-base Platform" diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index 62c50e098..a59271d6f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -1,6 +1,10 @@ --- version: '2.2' + +volumes: + fitbit-logs: {} + services: #---------------------------------------------------------------------------# # RADAR REDCap Integration # From 147f3a554af0570b430322a0c1d83a39ca8dc7e3 Mon Sep 17 00:00:00 2001 From: yatharthranjan Date: Fri, 21 Sep 2018 18:56:57 +0100 Subject: [PATCH 132/194] Fix travis. Do not check for kafka_init as its intended to exit. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b11bc3478..82559599c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,4 @@ script: - cd ../radar-cp-hadoop-stack/travis - ./setup-env.sh - cd ../ - - bin/radar-docker install && sleep 120 && bin/radar-docker ps && [ -z "$(bin/radar-docker ps | tail -n +3 | grep " Exit ")" ] + - bin/radar-docker install && sleep 120 && bin/radar-docker ps && [ -z "$(bin/radar-docker ps | tail -n +3 | grep " Exit " | grep -v "kafka-init_1")" ] From 06b5dc5d889d3e440e288526b6b3665afad9d66e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 10 Oct 2018 10:43:31 +0200 Subject: [PATCH 133/194] Add configuration for enable-ssl --- .../radar-cp-hadoop-stack/etc/env.template | 1 + .../etc/webserver/nginx.nossl.conf.template | 104 ++++++++++++++++++ .../lib/perform-install.sh | 27 ++++- 3 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index b0f618570..a7f57f817 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -1,6 +1,7 @@ SERVER_NAME=localhost MANAGEMENTPORTAL_KEY_DNAME=CN=localhost,OU=MyName,O=MyOrg,L=MyCity,S=MyState,C=MyCountryCode MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET= +ENABLE_HTTPS=yes SELF_SIGNED_CERT=yes MAINTAINER_EMAIL=me@example.com RADAR_RAW_TOPIC_LIST=android_phone_usage_event_aggregated,android_phone_usage_event_output diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template new file mode 100644 index 000000000..0f06cac4d --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template @@ -0,0 +1,104 @@ +worker_rlimit_nofile 8192; + +events { + worker_connections 4096; ## Default: 1024 +} + +http { + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + tcp_nodelay on; + + # add nosniff header (https://www.owasp.org/index.php/List_of_useful_HTTP_headers) + add_header X-Content-Type-Options nosniff; + + # For logins, make 2 requests per second at most + limit_req_zone $http_x_forwarded_for zone=login_limit:10m rate=2r/s; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name localhost; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + location /kafka/ { + include cors.conf; + proxy_pass http://gateway/radar-gateway/; + proxy_set_header Host $host; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + location ^~ /kafka/consumers { + deny all; + } + location ^~ /kafka/brokers { + deny all; + } + location ~* /kafka/topics/.+/partitions { + deny all; + } + location /schema/ { + limit_except GET { + deny all; + } + proxy_pass http://schema-registry-1:8081/; + proxy_set_header Host $host; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + location /dashboard/ { + proxy_pass http://dashboard:80/; + proxy_set_header Host $host; + } + location /portainer/ { + include ip-access-control.conf; + + proxy_pass http://portainer:9000/; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + location /portainer/api/websocket/ { + include ip-access-control.conf; + proxy_pass http://portainer:9000/api/websocket/; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_http_version 1.1; + } + location /api/ { + include cors.conf; + proxy_pass http://rest-api:8080/api/; + proxy_set_header Host $host; + } + location /managementportal/ { + include cors.conf; + proxy_pass http://managementportal-app:8080/managementportal/; + proxy_set_header Host $host; + } + location /managementportal/oauth/ { + # Allow 20 fast-following requests, like when authorizing a user. + limit_req zone=login_limit burst=20; + include cors.conf; + proxy_pass http://managementportal-app:8080/managementportal/oauth/; + proxy_set_header Host $host; + } + location /managementportal/api/meta-token/ { + limit_req zone=login_limit; + include cors.conf; + proxy_pass http://managementportal-app:8080/managementportal/api/meta-token/; + proxy_set_header Host $host; + } + location /kafkamanager/{ + include ip-access-control.conf; + auth_basic "Kafka manager"; + auth_basic_user_file kafka-manager.htpasswd; + + proxy_pass http://kafka-manager:9000; + proxy_set_header Host $host; + } + include optional-services.conf; + } +} diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index ab76541d7..dcf253f65 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -16,7 +16,6 @@ check_config_present etc/managementportal/config/oauth_client_details.csv copy_template_if_absent etc/mongodb-connector/sink-mongo.properties copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml -copy_template_if_absent etc/webserver/nginx.conf copy_template_if_absent etc/webserver/ip-access-control.conf copy_template_if_absent etc/webserver/optional-services.conf copy_template_if_absent etc/fitbit/docker/source-fitbit.properties @@ -32,6 +31,22 @@ fi . ./.env +if [ "${ENABLE_HTTPS:-yes}" = yes ]; then + copy_template_if_absent etc/webserver/nginx.conf + if ! grep -q 443 etc/webserver/nginx.conf; then + echo "NGINX configuration does not contain HTTPS configuration. Update the config" + echo "to template etc/webserver/nginx.conf.template or set ENABLE_HTTPS=no in .env." + exit 1 + fi +else + copy_template_if_absent etc/webserver/nginx.conf etc/webserver/nginx.nossl.conf.template + if grep -q 443 etc/webserver/nginx.conf; then + echo "NGINX configuration does contains HTTPS configuration. Update the config" + echo "to template etc/webserver/nginx.nossl.conf.template or set ENABLE_HTTPS=yes in .env." + exit 1 + fi +fi + # Check provided directories and configurations check_parent_exists HDFS_DATA_DIR_1 ${HDFS_DATA_DIR_1} check_parent_exists HDFS_DATA_DIR_2 ${HDFS_DATA_DIR_2} @@ -126,8 +141,10 @@ sudo-linux docker run --rm httpd:2.4-alpine htpasswd -nbB "${KAFKA_MANAGER_USERN echo "==> Configuring nginx" inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx.conf -sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf -init_certificate "${SERVER_NAME}" +if [ "${ENABLE_HTTPS:-yes}" = yes ]; then + sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf + init_certificate "${SERVER_NAME}" +fi # Configure Optional services if [[ "${ENABLE_OPTIONAL_SERVICES}" = "true" ]]; then @@ -141,5 +158,7 @@ fi echo "==> Starting RADAR-base Platform" sudo-linux bin/radar-docker up -d --remove-orphans "$@" -request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" +if [ "${ENABLE_HTTPS:-yes}" = yes ]; then + request_certificate "${SERVER_NAME}" "${SELF_SIGNED_CERT:-yes}" +fi echo "### SUCCESS ###" From a6289e77fe1a2ab9047a8f6af9846483ae3c6ff9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 10 Oct 2018 10:47:14 +0200 Subject: [PATCH 134/194] Do not try to renew certificate without HTTPS --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew index 72fac5939..eeb6aa7dd 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-cert-renew @@ -5,6 +5,11 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/.." . ./lib/util.sh . ./.env +if [ "${ENABLE_HTTPS:-yes}" != yes ]; then + echo "HTTPS is disabled. Not renewing certificate." + exit 0 +fi + if [ -z ${SERVER_NAME} ]; then echo "Set SERVER_NAME variable in .env" exit 1 From 964e85a7df97a671b6107897c43fc81e59fee14c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 10 Oct 2018 11:17:28 +0200 Subject: [PATCH 135/194] Explicitly set forwarded IP address --- dcompose-stack/radar-cp-hadoop-stack/README.md | 4 ++-- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 1 + .../etc/webserver/nginx.nossl.conf.template | 8 +++++++- .../radar-cp-hadoop-stack/lib/perform-install.sh | 7 +++++++ 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 927f4e131..36ca0b81c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -9,8 +9,8 @@ This is the set of minimal configuration required to run the stack. 1. First copy `etc/env.template` file to `./.env` and check and modify all its variables. - - 1.1. To have a valid HTTPS connection for a public host, set `SELF_SIGNED_CERT=no`. You need to provide a public valid DNS name as `SERVER_NAME` for SSL certificate to work. IP addresses will not work. + 1.1. To have a valid HTTPS connection for a public host, set `SELF_SIGNED_CERT=no`. You need to provide a public valid DNS name as `SERVER_NAME` for SSL certificate to work. IP addresses will not work. For a locally signed certificate, set `SELF_SIGNED_CERT=yes`. If HTTPS is to be disabled altogether, set `ENABLE_HTTPS=no`. If that is because the server is + behind a reverse proxy or load balancer, set `NGINX_PROXIES=1.2.3.4 5.6.7.8` as a space-separated list of proxy server IP addresses as forwarded in the `X-Forwarded-For` header. 1.2. Set `MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET` to a secret to be used by the Management Portal frontend. diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index ab81f8a12..91107fe15 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -641,7 +641,7 @@ services: # Webserver # #---------------------------------------------------------------------------# webserver: - image: nginx:1.13.1-alpine + image: nginx:1.14.0-alpine restart: always networks: - api diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index a7f57f817..9794050ea 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -37,3 +37,4 @@ RESTRUCTURE_OUTPUT_DIR=output ENABLE_OPTIONAL_SERVICES=false FITBIT_API_CLIENT_ID=fitbit-client FITBIT_API_CLIENT_SECRET=fitbit-secret +NGINX_PROXIES= diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template index 0f06cac4d..dbb3cdbd7 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template @@ -5,6 +5,12 @@ events { } http { + real_ip_header X-Forwarded-For; + # Updated from NGINX_PROXIES .env variable by `bin/radar-docker install`. + # Do not change the next line! It is autogenerated. + # NGINX_PROXIES + real_ip_recursive on; + default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] $status ' '"$request" $body_bytes_sent "$http_referer" ' @@ -15,7 +21,7 @@ http { add_header X-Content-Type-Options nosniff; # For logins, make 2 requests per second at most - limit_req_zone $http_x_forwarded_for zone=login_limit:10m rate=2r/s; + limit_req_zone $binary_remote_addr zone=login_limit:10m rate=2r/s; server { listen 80 default_server; diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index dcf253f65..d3371522b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -144,6 +144,13 @@ inline_variable 'server_name[[:space:]]*' "${SERVER_NAME};" etc/webserver/nginx. if [ "${ENABLE_HTTPS:-yes}" = yes ]; then sed_i 's|\(/etc/letsencrypt/live/\)[^/]*\(/.*\.pem\)|\1'"${SERVER_NAME}"'\2|' etc/webserver/nginx.conf init_certificate "${SERVER_NAME}" +else + # Fill in reverse proxy servers + proxies= + for PROXY in ${NGINX_PROXIES:-}; do + proxies="${proxies}set_real_ip_from ${PROXY}; " + done + sed_i "s/^\(\s*\).*# NGINX_PROXIES/\1$proxies# NGINX_PROXIES/" etc/webserver/nginx.conf fi # Configure Optional services From a7c9e90ca2c6872d9e95746ccdb3465a66e85231 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 23 Oct 2018 10:04:24 +0200 Subject: [PATCH 136/194] Use function to manage keystore init --- .../radar-cp-hadoop-stack/bin/keystore-init | 102 ++++++++---------- .../radar-cp-hadoop-stack/etc/.gitignore | 2 +- 2 files changed, 46 insertions(+), 58 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init index fff66133e..a33f8862f 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init @@ -5,66 +5,54 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/.." . ./.env . lib/util.sh -keystorefile=etc/managementportal/config/keystore.jks -KEYTOOL_OPTS="-keystore ${keystorefile} -storepass radarbase -keypass radarbase $KEYTOOL_OPTS" -if ! keytool -list $KEYTOOL_OPTS -alias radarbase-managementportal-ec >/dev/null 2>/dev/null; then - KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYTOOL_CREATE_OPTS" - if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then - KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" - fi - echo "--> Generating keystore to hold EC keypair for JWT signing" - keytool $KEYTOOL_CREATE_OPTS $KEYTOOL_OPTS -else - echo "--> ECDSA keypair for signing JWTs already exists. Not creating a new one." -fi - -chmod 400 "${keystorefile}" - - -GATEWAY_CONF=etc/gateway/radar-is.yml -echo "--> Updating gateway signature keys" -echo "resourceName: res_gateway" > $GATEWAY_CONF -echo "publicKeys:" >> $GATEWAY_CONF - -ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) -for (( i=0; i < ${#ALIASES[@]}; i++)); do - ALIAS=${ALIASES[$i]} - ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env - echo " - |-" >> $GATEWAY_CONF - if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then - REPLACE_PUBKEY="EC PUBLIC KEY" +function createKeyStore() { + file=$1 + + if ! keytool -list $KEYTOOL_OPTS -alias radarbase-managementportal-ec >/dev/null 2>/dev/null; then + KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYTOOL_CREATE_OPTS" + if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then + KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" + fi + echo "--> Generating keystore to hold EC keypair for JWT signing" + keytool $KEYTOOL_CREATE_OPTS $KEYTOOL_OPTS else - REPLACE_PUBKEY="PUBLIC KEY" + echo "--> ECDSA keypair for signing JWTs already exists. Not creating a new one." fi - cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" - while IFS='' read -r line && [ -n "$line" ]; do - line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) - echo " $line" >> $GATEWAY_CONF - done <<< "$cert" -done + chmod 400 "$file" +} + +function writeKeys() { + FILE=$1 + RES=$2 + + echo "--> Updating gateway signature keys" + echo "resourceName: $RES" > "$FILE" + echo "publicKeys:" >> "$FILE" + + ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) + for (( i=0; i < ${#ALIASES[@]}; i++)); do + ALIAS=${ALIASES[$i]} + ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env + echo " - |-" >> "$FILE" + if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then + REPLACE_PUBKEY="EC PUBLIC KEY" + else + REPLACE_PUBKEY="PUBLIC KEY" + fi + + cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" + while IFS='' read -r line && [ -n "$line" ]; do + line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) + echo " $line" >> "$FILE" + done <<< "$cert" + done +} +keystorefile=etc/managementportal/config/keystore.jks +export KEYTOOL_OPTS="-keystore ${keystorefile} -storepass radarbase -keypass radarbase $KEYTOOL_OPTS" -REST_API_CONF=etc/rest-api/radar-is.yml -echo "--> Updating Rest-API signature keys" -echo "resourceName: res_RestApi" > $REST_API_CONF -echo "publicKeys:" >> $REST_API_CONF - -ALIASES=($(keytool -list $KEYTOOL_OPTS | grep PrivateKeyEntry | sed -e 's/^\([^,]*\),.*$/\1/')) -for (( i=0; i < ${#ALIASES[@]}; i++)); do - ALIAS=${ALIASES[$i]} - ensure_variable "MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_$i=" $ALIAS .env - echo " - |-" >> $REST_API_CONF - if keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -text | grep -q ecdsa-with-SHA256; then - REPLACE_PUBKEY="EC PUBLIC KEY" - else - REPLACE_PUBKEY="PUBLIC KEY" - fi - - cert="$(keytool -export $KEYTOOL_OPTS -alias $ALIAS | openssl x509 -inform der -pubkey -noout)" - while IFS='' read -r line && [ -n "$line" ]; do - line=$(sed "s/PUBLIC KEY/$REPLACE_PUBKEY/" <<< $line) - echo " $line" >> $REST_API_CONF - done <<< "$cert" -done +createKeyStore "$keystorefile" +writeKeys etc/gateway/radar-is.yml res_gateway +writeKeys etc/rest-api/radar-is.yml res_RestApi diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index b5fcd2da4..f8810a4cb 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -13,6 +13,6 @@ /managementportal/config/oauth_client_details.csv /schema/commons/ /schema/specifications/ -/gateway/radar-is.yml +radar-is.yml /fitbit/docker/users/*.yml /fitbit/docker/source-fitbit.properties From bf6dabcc184f5bc5fd13b54097dff38fb388c0a9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 23 Oct 2018 10:04:39 +0200 Subject: [PATCH 137/194] Update gateway --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index bd92ad207..76b43b125 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -776,7 +776,7 @@ services: # RADAR Gateway # #---------------------------------------------------------------------------# gateway: - image: radarbase/radar-gateway:0.3.0 + image: radarbase/radar-gateway:0.3.1 networks: - api - kafka From 5d741db0f97ce224f192e6a869f555ede4312879 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 23 Oct 2018 10:14:37 +0200 Subject: [PATCH 138/194] Small fixes --- .../radar-cp-hadoop-stack/docker-compose.yml | 19 ------------------- .../etc/radar-backend/radar.yml.template | 8 +++----- .../optional-services.yml | 2 +- 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 76b43b125..8327c18fa 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -323,25 +323,6 @@ services: timeout: 5s retries: 3 - #---------------------------------------------------------------------------# - # RADAR REDCap Integration # - #---------------------------------------------------------------------------# - radar-integration: - image: radarbase/radar-redcapintegration:0.1.0 - networks: - - api - - default - depends_on: - - managementportal-app - restart: always - volumes: - - "./etc/redcap-integration:/usr/local/tomcat/conf/radar" - healthcheck: - test: ["CMD", "wget", "-IX", "POST", "http://localhost:8080/redcap/trigger"] - interval: 1m - timeout: 5s - retries: 3 - #---------------------------------------------------------------------------# # RADAR Dashboard # #---------------------------------------------------------------------------# diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template index 484e1efd2..08f12473e 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/radar-backend/radar.yml.template @@ -30,7 +30,7 @@ rest_proxy: #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority stream: - stream_priority: + threads_per_priority: low: 1 normal: 2 high: 4 @@ -78,8 +78,6 @@ stream: - android_phone_relative_location output_topic: source_statistics_android_phone - - #======================== Battery level monitor ========================# battery_monitor: level: LOW @@ -90,7 +88,7 @@ battery_monitor: - project_id: s2 email_address: - notifier2@email - email_host: localhost + email_host: smtp email_port: 25 email_user: user@example.com topics: @@ -105,7 +103,7 @@ disconnect_monitor: - project_id: s2 email_address: - notifier2@email - email_host: localhost + email_host: smtp email_port: 25 email_user: user@example.com topics: diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index a59271d6f..a7106b81a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -20,7 +20,7 @@ services: volumes: - "./etc/redcap-integration:/usr/local/etc/radar-redcap-int" healthcheck: - test: ["CMD-SHELL", "wget --spider http://localhost:8080/redcap/trigger 2>&1 | grep -q 500 || exit 1"] + test: ["CMD", "wget", "-IX", "POST", "http://localhost:8080/redcap/trigger"] interval: 1m timeout: 5s retries: 3 From 86466035222e72deece3d49a3a58e888f10412f6 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 23 Oct 2018 10:16:35 +0200 Subject: [PATCH 139/194] Fixed gitignore --- dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index f8810a4cb..80af66d9a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -3,6 +3,7 @@ /webserver/nginx.conf /webserver/kafka-manager.htpasswd /webserver/ip-access-control.conf +/webserver/optional-services.conf /radar-backend/radar.yml /smtp.env /rest-api/device-catalog.yml From 61d986f43bec626926e99fe3658ab5101be6c6b7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 23 Oct 2018 10:23:02 +0200 Subject: [PATCH 140/194] Update HDFS restructure version to fix time keeping --- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 5cfff74a1..fcd9b2d33 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -1,7 +1,7 @@ #!/bin/bash # HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.2 +DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.3 # HDFS restructure script flags HDFS_RESTRUCTURE_OPTS=( --compression gzip From 636617677b53301a0da035c574888cef3f9320da Mon Sep 17 00:00:00 2001 From: nivethika Date: Wed, 24 Oct 2018 18:48:46 +0200 Subject: [PATCH 141/194] added device-auth components to optional stack --- .../bin/postgres-upgrade | 2 +- .../radar-cp-hadoop-stack/etc/.gitignore | 1 + .../device_clients_configs.yml.template | 7 +++++ .../webserver/optional-services.conf.template | 18 ++++++++++--- .../optional-services.yml | 26 +++++++++++++++++++ 5 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade index d5c76e69f..562e27e81 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade @@ -11,7 +11,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." . ./.env . lib/util.sh -echo "Are you sure you want to to upgrade HDFS from version ${POSTGRES_VERSION} to ${NEW_VERSION}?" +echo "Are you sure you want to to upgrade POSTGRESQL from version ${POSTGRES_VERSION} to ${NEW_VERSION}?" select yn in "Yes" "No"; do case $yn in diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index 80af66d9a..133f5a216 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -17,3 +17,4 @@ radar-is.yml /fitbit/docker/users/*.yml /fitbit/docker/source-fitbit.properties +/device-authorizer/device_clients_configs.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template new file mode 100644 index 000000000..5279157e8 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template @@ -0,0 +1,7 @@ +device_clients: + - device_type: FitBit + authorization_endpoint: https://www.fitbit.com/oauth2/authorize + token_endpoint: https://api.fitbit.com/oauth2/token + client_id: + client_secret: + scope: activity heartrate location nutrition sleep social weight \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template index ce1032d97..b43f0a8ce 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template @@ -1,5 +1,15 @@ # uncomment and add according to which optional services need to use the webserver. -#location /redcapint/ { -# proxy_pass http://radar-integration:8080/redcap/; -# proxy_set_header Host $host; -#} +location /redcapint/ { + proxy_pass http://radar-integration:8080/redcap/; + proxy_set_header Host $host; +} + +location /device-authorizer/ { + proxy_pass http://radar-device-auth-frontend:80/; + proxy_set_header Host $host; +} + +location /device-auth/backend/ { + proxy_pass http://radar-device-auth-backend:8080/; + proxy_set_header Host $host; +} \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index a7106b81a..47e233d38 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -72,3 +72,29 @@ services: interval: 1m timeout: 5s retries: 3 + + radar-device-auth-backend: + image: radarbase/radar-device-auth-backend:dev + depends_on: + - radar-device-auth-postgresql + environment: + - SPRING_DATASOURCE_URL=jdbc:postgresql://radar-device-auth-postgresql:5432/deviceauthorizer + - SPRING_DATASOURCE_USERNAME=radarcns + - SPRING_DATASOURCE_PASSWORD=radarcns + - DEVICE_AUTHORIZER_DEVICE_CLIENTS_FILE_PATH=app-includes/device_clients_configs.yml + - APP_SLEEP=10 # gives time for the database to boot before the application + volumes: + - ./etc/device-authorizer/:/app-includes/ + + radar-device-auth-postgresql: + image: postgres:${POSTGRES_VERSION} + environment: + - POSTGRES_USER=radarcns + - POSTGRES_PASSWORD=radarcns + - POSTGRES_DB=deviceauthorizer + + radar-device-auth-frontend: + image: radarbase/radar-device-authorizer:dev + depends_on: + - radar-device-auth-backend + - radar-device-auth-postgresql From 521c1ed8ab9e03439a8ebb827ad4eb7bc8bd96fe Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 25 Oct 2018 09:31:15 +0200 Subject: [PATCH 142/194] Add RSA keypair on MP initialization --- .../radar-cp-hadoop-stack/bin/keystore-init | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init index a33f8862f..ff36400c8 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init @@ -9,7 +9,7 @@ function createKeyStore() { file=$1 if ! keytool -list $KEYTOOL_OPTS -alias radarbase-managementportal-ec >/dev/null 2>/dev/null; then - KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYTOOL_CREATE_OPTS" + KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYSTORE_CREATE_OPTS" if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" fi @@ -19,6 +19,17 @@ function createKeyStore() { echo "--> ECDSA keypair for signing JWTs already exists. Not creating a new one." fi + if ! keytool -list $KEYTOOL_OPTS -alias selfsigned >/dev/null 2>/dev/null; then + KEYTOOL_CREATE_OPTS="-genkeypair -alias selfsigned -keyalg RSA -keysize 4096 -storetype JKS $KEYSTORE_CREATE_OPTS" + if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then + KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" + fi + echo "--> Generating keystore to hold RSA keypair for JWT signing" + keytool $KEYTOOL_CREATE_OPTS $KEYTOOL_OPTS + else + echo "--> RSA keypair for signing JWTs already exists. Not creating a new one." + fi + chmod 400 "$file" } From 3619013ed7ed9dd70cc9b4db077bda4ed2c17118 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 25 Oct 2018 13:50:09 +0200 Subject: [PATCH 143/194] Add default MP key aliases --- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 0a4d00884..961b2e463 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -30,11 +30,11 @@ POSTGRES_VERSION=10.4-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD= PORTAINER_PASSWORD_HASH= -MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0= -MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1= +MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec +MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned RESTRUCTURE_OUTPUT_DIR=output MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD= MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy ENABLE_OPTIONAL_SERVICES=false FITBIT_API_CLIENT_ID=fitbit-client -FITBIT_API_CLIENT_SECRET=fitbit-secret \ No newline at end of file +FITBIT_API_CLIENT_SECRET=fitbit-secret From b957f5f5cdcea0bc50927c7393e84425c71aa95d Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 25 Oct 2018 17:24:50 +0200 Subject: [PATCH 144/194] update configs for to talk to device-authorizer --- .../etc/device-authorizer/device_clients_configs.yml.template | 2 +- .../etc/fitbit/docker/source-fitbit.properties.template | 2 ++ .../etc/fitbit/docker/users/fitbit-user.yml.template | 2 +- dcompose-stack/radar-cp-hadoop-stack/optional-services.yml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template index 5279157e8..f6676837b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template @@ -4,4 +4,4 @@ device_clients: token_endpoint: https://api.fitbit.com/oauth2/token client_id: client_secret: - scope: activity heartrate location nutrition sleep social weight \ No newline at end of file + scope: activity heartrate location nutrition sleep social weight profile \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template index 631321bb2..c41778626 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template @@ -6,3 +6,5 @@ rest.source.poll.interval.ms=5000 rest.source.request.generator.class=org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator fitbit.api.client= fitbit.api.secret= +fitbit.user.repository.class=org.radarbase.connect.rest.fitbit.user.ServiceUserRepository +fitbit.user.repository.url=http://radar-device-auth-backend:8080/ \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template index 56f4eeea6..c125be7d8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/users/fitbit-user.yml.template @@ -12,7 +12,7 @@ startDate: 2018-08-06T00:00:00Z # Date until when to collect data. endDate: 2019-01-01T00:00:00Z # Fitbit user ID as returned by the Fitbit authentication procedure -fitbitUserId: ? +externalUserId: ? oauth2: # Fitbit OAuth 2.0 access token as returned by the Fitbit authentication procedure accessToken: ? diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index 47e233d38..d271e20e7 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -29,7 +29,7 @@ services: # RADAR Fitbit connector # #---------------------------------------------------------------------------# radar-fitbit-connector: - image: radarbase/kafka-connect-rest-fitbit-source:0.1.0 + image: radarbase/kafka-connect-rest-fitbit-source:dev restart: on-failure volumes: - ./etc/fitbit/docker/source-fitbit.properties:/etc/kafka-connect/source-fitbit.properties From 8daaab5b33f04453b5d21f5a61f748f59411130c Mon Sep 17 00:00:00 2001 From: nivethika Date: Mon, 29 Oct 2018 15:02:02 +0100 Subject: [PATCH 145/194] add custom postgres with support to create multiple databases and modify configurations --- .../radar-cp-hadoop-stack/docker-compose.yml | 14 ++++++--- .../images/postgres/Dockerfile | 5 +++ .../images/postgres/entrypoint.sh | 31 +++++++++++++++++++ .../lib/perform-install.sh | 3 ++ .../optional-services.yml | 23 +++++++------- 5 files changed, 59 insertions(+), 17 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile create mode 100755 dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 8327c18fa..a882a4e5f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -683,12 +683,12 @@ services: - management - mail depends_on: - - managementportal-postgresql + - radarbase-postgresql - smtp - catalog-server environment: SPRING_PROFILES_ACTIVE: prod,swagger - SPRING_DATASOURCE_URL: jdbc:postgresql://managementportal-postgresql:5432/managementportal + SPRING_DATASOURCE_URL: jdbc:postgresql://radarbase-postgresql:5432/managementportal SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER} SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD} MANAGEMENTPORTAL_MAIL_FROM: ${FROM_EMAIL} @@ -712,8 +712,12 @@ services: retries: 3 - managementportal-postgresql: - image: postgres:${POSTGRES_VERSION} + radarbase-postgresql: + build: + context: ./images/postgres + args: + POSTGRES_VERSION: ${POSTGRES_VERSION} + image: postgres:radarbase-${POSTGRES_VERSION} volumes: - "${MP_POSTGRES_DIR}/data/:/var/lib/postgresql/data/" - "./postgres-backup/backups/postgresql:/backups/database/postgresql/" @@ -721,7 +725,7 @@ services: environment: POSTGRES_USER : ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: managementportal + POSTGRES_MULTIPLE_DATABASES: managementportal,deviceauthorizer networks: - management healthcheck: diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile new file mode 100644 index 000000000..f9b291bd1 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile @@ -0,0 +1,5 @@ +ARG POSTGRES_VERSION=10.4-alpine +FROM postgres:${POSTGRES_VERSION} + +COPY ./entrypoint.sh /docker-entrypoint-initdb.d/multi-db-init.sh + diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh new file mode 100755 index 000000000..cea4793df --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +set -u + + +function create_user_and_database() { + local database=$1 + echo "Processing database '$database'" + local query_databases="select datname from pg_database;" + local database_exist=$(psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'") + if [[ "$database_exist" == 1 ]]; + then + echo "Database already exists" + else + echo "Database does not exist" + echo " Creating database '$database' for user '$POSTGRES_USER'" + psql -v ON_ERROR_STOP=1 <<-EOSQL + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $POSTGRES_USER; +EOSQL + fi + +} + +if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" + for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do + create_user_and_database $db + done + echo "Multiple databases created" +fi \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index b69362f25..12c4a399e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -111,6 +111,9 @@ fi ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties echo "==> Configuring Management Portal" +sudo-linux bin/radar-docker up -d --build radarbase-postgresql +sleep 10 +sudo-linux bin/radar-docker exec --user postgres radarbase-postgresql /docker-entrypoint-initdb.d/multi-db-init.sh ensure_env_password MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET "ManagementPortal front-end client secret is not set in .env" ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index d271e20e7..36ae8d647 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -76,25 +76,24 @@ services: radar-device-auth-backend: image: radarbase/radar-device-auth-backend:dev depends_on: - - radar-device-auth-postgresql + - radarbase-postgresql + networks: + - default + - api + - management environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://radar-device-auth-postgresql:5432/deviceauthorizer - - SPRING_DATASOURCE_USERNAME=radarcns - - SPRING_DATASOURCE_PASSWORD=radarcns + - SPRING_DATASOURCE_URL=jdbc:postgresql://radarbase-postgresql:5432/deviceauthorizer + - SPRING_DATASOURCE_USERNAME=${POSTGRES_USER} + - SPRING_DATASOURCE_PASSWORD=${POSTGRES_PASSWORD} - DEVICE_AUTHORIZER_DEVICE_CLIENTS_FILE_PATH=app-includes/device_clients_configs.yml - APP_SLEEP=10 # gives time for the database to boot before the application volumes: - ./etc/device-authorizer/:/app-includes/ - radar-device-auth-postgresql: - image: postgres:${POSTGRES_VERSION} - environment: - - POSTGRES_USER=radarcns - - POSTGRES_PASSWORD=radarcns - - POSTGRES_DB=deviceauthorizer - radar-device-auth-frontend: image: radarbase/radar-device-authorizer:dev + networks: + - api depends_on: - radar-device-auth-backend - - radar-device-auth-postgresql + - radarbase-postgresql From 5504bcc036c08d1341bb0a35ee80c6004e6a47cf Mon Sep 17 00:00:00 2001 From: nivethika Date: Mon, 29 Oct 2018 15:17:07 +0100 Subject: [PATCH 146/194] change required scopes for fitbit --- .../etc/device-authorizer/device_clients_configs.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template index f6676837b..f9e712b44 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template @@ -4,4 +4,4 @@ device_clients: token_endpoint: https://api.fitbit.com/oauth2/token client_id: client_secret: - scope: activity heartrate location nutrition sleep social weight profile \ No newline at end of file + scope: activity heartrate sleep profile \ No newline at end of file From e358a3fe0bd3c03852a4d5610cf8d4b5ff87b3e1 Mon Sep 17 00:00:00 2001 From: nivethika Date: Mon, 29 Oct 2018 15:22:04 +0100 Subject: [PATCH 147/194] add default keystore alias to travis template --- dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index fac9f1f83..40e17e690 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -31,6 +31,8 @@ KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD=kafkamanager-pwd PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD=notadmin +MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec +MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy ENABLE_OPTIONAL_SERVICES=true FITBIT_API_CLIENT_ID=fitbit-client From 622fc82e609486f4edbe3e45d789d68b2f8e1aec Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 30 Oct 2018 15:00:40 +0100 Subject: [PATCH 148/194] Update radar-schemas --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- .../radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 8327c18fa..3f9d9ecc4 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -260,7 +260,7 @@ services: #---------------------------------------------------------------------------# kafka-init: build: images/radar-kafka-init - image: radarbase/kafka-init:0.3.6 + image: radarbase/kafka-init:0.4.1 networks: - kafka - zookeeper @@ -780,7 +780,7 @@ services: #---------------------------------------------------------------------------# catalog-server: build: images/radar-kafka-init - image: radarbase/kafka-init:0.3.6 + image: radarbase/kafka-init:0.4.1 networks: - management command: radar-schemas-tools serve /schema/merged diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile index 8a8bb9d6b..600da13b1 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile @@ -26,7 +26,7 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.3.6 +ENV RADAR_SCHEMAS_VERSION=0.4.1 RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 From d5e8bcbe1c6459574d63532da96b9fb4581344f9 Mon Sep 17 00:00:00 2001 From: nivethika Date: Tue, 30 Oct 2018 16:23:51 +0100 Subject: [PATCH 149/194] add new route changes and new images alter install script for some validations --- .../docker/source-fitbit.properties.template | 2 +- .../source_clients_configs.yml.template} | 4 ++-- .../etc/webserver/optional-services.conf.template | 8 ++++---- .../radar-cp-hadoop-stack/lib/perform-install.sh | 5 +++++ .../radar-cp-hadoop-stack/optional-services.yml | 14 +++++++------- 5 files changed, 19 insertions(+), 14 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/etc/{device-authorizer/device_clients_configs.yml.template => rest-source-authorizer/source_clients_configs.yml.template} (75%) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template index c41778626..e3a27e707 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/fitbit/docker/source-fitbit.properties.template @@ -7,4 +7,4 @@ rest.source.request.generator.class=org.radarbase.connect.rest.fitbit.request.Fi fitbit.api.client= fitbit.api.secret= fitbit.user.repository.class=org.radarbase.connect.rest.fitbit.user.ServiceUserRepository -fitbit.user.repository.url=http://radar-device-auth-backend:8080/ \ No newline at end of file +fitbit.user.repository.url=http://radar-rest-sources-backend:8080/ \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/source_clients_configs.yml.template similarity index 75% rename from dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template rename to dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/source_clients_configs.yml.template index f9e712b44..dc5fccb4c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/device-authorizer/device_clients_configs.yml.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/source_clients_configs.yml.template @@ -2,6 +2,6 @@ device_clients: - device_type: FitBit authorization_endpoint: https://www.fitbit.com/oauth2/authorize token_endpoint: https://api.fitbit.com/oauth2/token - client_id: - client_secret: + client_id: MYFITBITCLIENTID + client_secret: MYFITBITSECRET scope: activity heartrate sleep profile \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template index b43f0a8ce..e1059277c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/optional-services.conf.template @@ -4,12 +4,12 @@ location /redcapint/ { proxy_set_header Host $host; } -location /device-authorizer/ { - proxy_pass http://radar-device-auth-frontend:80/; +location /rest-sources/authorizer/ { + proxy_pass http://radar-rest-sources-authorizer:80/; proxy_set_header Host $host; } -location /device-auth/backend/ { - proxy_pass http://radar-device-auth-backend:8080/; +location /rest-sources/backend/ { + proxy_pass http://radar-rest-sources-backend:8080/; proxy_set_header Host $host; } \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 12c4a399e..15f937435 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -20,6 +20,7 @@ copy_template_if_absent etc/webserver/nginx.conf copy_template_if_absent etc/webserver/ip-access-control.conf copy_template_if_absent etc/webserver/optional-services.conf copy_template_if_absent etc/fitbit/docker/source-fitbit.properties +copy_template_if_absent etc/rest-source-authorizer/source_clients_configs.yml # Set permissions sudo-linux chmod og-rw ./.env @@ -140,6 +141,10 @@ if [[ "${ENABLE_OPTIONAL_SERVICES}" = "true" ]]; then ensure_variable 'fitbit.api.client=' $FITBIT_API_CLIENT_ID etc/fitbit/docker/source-fitbit.properties ensure_variable 'fitbit.api.secret=' $FITBIT_API_CLIENT_SECRET etc/fitbit/docker/source-fitbit.properties + echo "==> Configuring Rest Source Authorizer" + inline_variable 'client_id:[[:space:]]' "$FITBIT_API_CLIENT_ID" etc/rest-source-authorizer/source_clients_configs.yml + inline_variable 'client_secret:[[:space:]]' "$FITBIT_API_CLIENT_SECRET" etc/rest-source-authorizer/source_clients_configs.yml + check_config_present etc/redcap-integration/radar.yml fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index 36ae8d647..bf7ad296b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -73,8 +73,8 @@ services: timeout: 5s retries: 3 - radar-device-auth-backend: - image: radarbase/radar-device-auth-backend:dev + radar-rest-sources-backend: + image: radarbase/radar-rest-source-auth-backend:dev depends_on: - radarbase-postgresql networks: @@ -85,15 +85,15 @@ services: - SPRING_DATASOURCE_URL=jdbc:postgresql://radarbase-postgresql:5432/deviceauthorizer - SPRING_DATASOURCE_USERNAME=${POSTGRES_USER} - SPRING_DATASOURCE_PASSWORD=${POSTGRES_PASSWORD} - - DEVICE_AUTHORIZER_DEVICE_CLIENTS_FILE_PATH=app-includes/device_clients_configs.yml + - DEVICE_AUTHORIZER_DEVICE_CLIENTS_FILE_PATH=app-includes/source_clients_configs.yml - APP_SLEEP=10 # gives time for the database to boot before the application volumes: - - ./etc/device-authorizer/:/app-includes/ + - ./etc/rest-source-authorizer/:/app-includes/ - radar-device-auth-frontend: - image: radarbase/radar-device-authorizer:dev + radar-rest-sources-authorizer: + image: radarbase/radar-rest-source-authorizer:dev networks: - api depends_on: - - radar-device-auth-backend + - radar-rest-sources-backend - radarbase-postgresql From b2c45a8d0a5ea5b2294f52279f48c2c4ba454ba9 Mon Sep 17 00:00:00 2001 From: nivethika Date: Wed, 31 Oct 2018 17:46:47 +0100 Subject: [PATCH 150/194] integrate rest-source-auth 1.0.0 --- .../radar-cp-hadoop-stack/docker-compose.yml | 2 +- dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore | 2 +- .../rest_source_clients_configs.yml.template | 7 +++++++ .../radar-cp-hadoop-stack/lib/perform-install.sh | 6 +++--- .../radar-cp-hadoop-stack/optional-services.yml | 13 +++++++++---- 5 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/rest_source_clients_configs.yml.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index a882a4e5f..1f32b5726 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -725,7 +725,7 @@ services: environment: POSTGRES_USER : ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_MULTIPLE_DATABASES: managementportal,deviceauthorizer + POSTGRES_MULTIPLE_DATABASES: managementportal,restsourceauthorizer networks: - management healthcheck: diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index 133f5a216..ac0e72cac 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -17,4 +17,4 @@ radar-is.yml /fitbit/docker/users/*.yml /fitbit/docker/source-fitbit.properties -/device-authorizer/device_clients_configs.yml +/rest-source-authorizer/rest_source_clients_configs.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/rest_source_clients_configs.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/rest_source_clients_configs.yml.template new file mode 100644 index 000000000..0d482abfa --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/rest_source_clients_configs.yml.template @@ -0,0 +1,7 @@ +rest_source_clients: + - source_type: FitBit + authorization_endpoint: https://www.fitbit.com/oauth2/authorize + token_endpoint: https://api.fitbit.com/oauth2/token + client_id: FITBITIT + client_secret: FITBITSECRET + scope: activity heartrate sleep profile \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 15f937435..917787bea 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -20,7 +20,7 @@ copy_template_if_absent etc/webserver/nginx.conf copy_template_if_absent etc/webserver/ip-access-control.conf copy_template_if_absent etc/webserver/optional-services.conf copy_template_if_absent etc/fitbit/docker/source-fitbit.properties -copy_template_if_absent etc/rest-source-authorizer/source_clients_configs.yml +copy_template_if_absent etc/rest-source-authorizer/rest_source_clients_configs.yml # Set permissions sudo-linux chmod og-rw ./.env @@ -142,8 +142,8 @@ if [[ "${ENABLE_OPTIONAL_SERVICES}" = "true" ]]; then ensure_variable 'fitbit.api.secret=' $FITBIT_API_CLIENT_SECRET etc/fitbit/docker/source-fitbit.properties echo "==> Configuring Rest Source Authorizer" - inline_variable 'client_id:[[:space:]]' "$FITBIT_API_CLIENT_ID" etc/rest-source-authorizer/source_clients_configs.yml - inline_variable 'client_secret:[[:space:]]' "$FITBIT_API_CLIENT_SECRET" etc/rest-source-authorizer/source_clients_configs.yml + inline_variable 'client_id:[[:space:]]' "$FITBIT_API_CLIENT_ID" etc/rest-source-authorizer/rest_source_clients_configs.yml + inline_variable 'client_secret:[[:space:]]' "$FITBIT_API_CLIENT_SECRET" etc/rest-source-authorizer/rest_source_clients_configs.yml check_config_present etc/redcap-integration/radar.yml fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index bf7ad296b..085d3e133 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -74,7 +74,7 @@ services: retries: 3 radar-rest-sources-backend: - image: radarbase/radar-rest-source-auth-backend:dev + image: radarbase/radar-rest-source-auth-backend:1.0.0 depends_on: - radarbase-postgresql networks: @@ -82,16 +82,21 @@ services: - api - management environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://radarbase-postgresql:5432/deviceauthorizer + - SPRING_DATASOURCE_URL=jdbc:postgresql://radarbase-postgresql:5432/restsourceauthorizer - SPRING_DATASOURCE_USERNAME=${POSTGRES_USER} - SPRING_DATASOURCE_PASSWORD=${POSTGRES_PASSWORD} - - DEVICE_AUTHORIZER_DEVICE_CLIENTS_FILE_PATH=app-includes/source_clients_configs.yml + - REST_SOURCE_AUTHORIZER_SOURCE_CLIENTS_FILE_PATH=app-includes/rest_source_clients_configs.yml - APP_SLEEP=10 # gives time for the database to boot before the application volumes: - ./etc/rest-source-authorizer/:/app-includes/ + healthcheck: + test: ["CMD", "curl", "-f", "localhost:8080/users"] + interval: 1m30s + timeout: 5s + retries: 3 radar-rest-sources-authorizer: - image: radarbase/radar-rest-source-authorizer:dev + image: radarbase/radar-rest-source-authorizer:1.0.0 networks: - api depends_on: From 3a42857abeae9ccef5f403dc468efb6882103745 Mon Sep 17 00:00:00 2001 From: Nivethika Mahasivam Date: Wed, 31 Oct 2018 17:49:15 +0100 Subject: [PATCH 151/194] Delete source_clients_configs.yml.template --- .../source_clients_configs.yml.template | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/source_clients_configs.yml.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/source_clients_configs.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/source_clients_configs.yml.template deleted file mode 100644 index dc5fccb4c..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/rest-source-authorizer/source_clients_configs.yml.template +++ /dev/null @@ -1,7 +0,0 @@ -device_clients: - - device_type: FitBit - authorization_endpoint: https://www.fitbit.com/oauth2/authorize - token_endpoint: https://api.fitbit.com/oauth2/token - client_id: MYFITBITCLIENTID - client_secret: MYFITBITSECRET - scope: activity heartrate sleep profile \ No newline at end of file From d698a2327f92ccc6aecb879ee786364d87906f52 Mon Sep 17 00:00:00 2001 From: nivethika Date: Wed, 31 Oct 2018 17:56:15 +0100 Subject: [PATCH 152/194] increase sleep time before creating postgres --- dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 917787bea..a87d2d148 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -113,7 +113,7 @@ ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-h echo "==> Configuring Management Portal" sudo-linux bin/radar-docker up -d --build radarbase-postgresql -sleep 10 +sleep 20 sudo-linux bin/radar-docker exec --user postgres radarbase-postgresql /docker-entrypoint-initdb.d/multi-db-init.sh ensure_env_password MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET "ManagementPortal front-end client secret is not set in .env" ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." From 898fbd135e09306d9cf6d4e025b317762e112439 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 1 Nov 2018 10:44:15 +0100 Subject: [PATCH 153/194] change postgres image tag --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 1f32b5726..26dfffc6b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -717,7 +717,7 @@ services: context: ./images/postgres args: POSTGRES_VERSION: ${POSTGRES_VERSION} - image: postgres:radarbase-${POSTGRES_VERSION} + image: radarbase/postgres:${POSTGRES_VERSION}-1 volumes: - "${MP_POSTGRES_DIR}/data/:/var/lib/postgresql/data/" - "./postgres-backup/backups/postgresql:/backups/database/postgresql/" From ce32a3bd07ba0bd10e9858dfbf02c09389ef30fd Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 1 Nov 2018 11:52:22 +0100 Subject: [PATCH 154/194] changes based on PR comments --- .../images/postgres/entrypoint.sh | 14 ++++++++++++++ .../radar-cp-hadoop-stack/lib/perform-install.sh | 1 - 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh index cea4793df..77d395593 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh @@ -2,7 +2,19 @@ set -e set -u +DB_HOST="localhost" +DB_PORT=5432 +function wait_for_db() { + for count in {1..30}; do + echo "Pinging postgres database attempt "${count} + if $(nc -z ${DB_HOST} ${DB_PORT}) ; then + echo "Can connect into database" + break + fi + sleep 1 + done +} function create_user_and_database() { local database=$1 echo "Processing database '$database'" @@ -24,6 +36,8 @@ EOSQL if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" + #waiting for postgres + wait_for_db for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do create_user_and_database $db done diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index a87d2d148..30974c175 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -113,7 +113,6 @@ ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-h echo "==> Configuring Management Portal" sudo-linux bin/radar-docker up -d --build radarbase-postgresql -sleep 20 sudo-linux bin/radar-docker exec --user postgres radarbase-postgresql /docker-entrypoint-initdb.d/multi-db-init.sh ensure_env_password MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET "ManagementPortal front-end client secret is not set in .env" ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." From acdec231f2c6250b3d48db362986e67481157bcf Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 1 Nov 2018 12:38:24 +0100 Subject: [PATCH 155/194] update fitbit-connector tag --- dcompose-stack/radar-cp-hadoop-stack/optional-services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index 085d3e133..b5f0f1b94 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -29,7 +29,7 @@ services: # RADAR Fitbit connector # #---------------------------------------------------------------------------# radar-fitbit-connector: - image: radarbase/kafka-connect-rest-fitbit-source:dev + image: radarbase/kafka-connect-rest-fitbit-source:0.2.0 restart: on-failure volumes: - ./etc/fitbit/docker/source-fitbit.properties:/etc/kafka-connect/source-fitbit.properties From 10328e3cd792a1b21cda94f5e692996b0f0d2084 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 1 Nov 2018 12:40:41 +0100 Subject: [PATCH 156/194] change travis config --- .../radar-cp-hadoop-stack/travis/travis-env.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 40e17e690..2807428a2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -10,8 +10,8 @@ HOTSTORAGE_PASSWORD=mongo-pwd HOTSTORAGE_NAME=mongodb-database MONGODB_DIR=/home/travis/data/mongodb MP_POSTGRES_DIR=/home/travis/data/postgres -POSTGRES_USER=postgresdb-user -POSTGRES_PASSWORD=postgresdb-pwd +POSTGRES_USER=postgresdbuser +POSTGRES_PASSWORD=postgresdbpwd FROM_EMAIL=from-email MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT=false HEALTHCHECK_HIPCHAT_NOTIFY=no From adfe2236a685cbd53a7e1139bb3e877d075ec335 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 11:52:11 +0100 Subject: [PATCH 157/194] Logging to alternative directory --- README.md | 2 +- .../radar-cp-hadoop-stack/README.md | 9 +-- .../radar-cp-hadoop-stack/bin/radar-log | 66 +++++++++++++++++++ .../radar-check-health.service.template | 4 ++ .../lib/systemd/radar-docker.service.template | 4 ++ .../lib/systemd/radar-output.service.template | 3 + .../radar-renew-certificate.service.template | 4 ++ 7 files changed, 84 insertions(+), 8 deletions(-) create mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/radar-log diff --git a/README.md b/README.md index 962c7f7cb..c140deb9f 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ To run RADAR-base stack in a single node setup: ### Logging -Set up logging by going to the `dcompose-stack/logging` directory and follow the README there. +Set up a logging service by going to the `dcompose-stack/logging` directory and follow the README there. ## Work in progress diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index 927f4e131..ad8a76e44 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -102,6 +102,8 @@ sudo systemctl start radar-docker To rebuild an image and restart them, run `bin/radar-docker rebuild IMAGE`. To stop and remove an container, run `bin/radar-docker quit CONTAINER`. To start the HDFS cluster, run `bin/radar-docker hdfs`. For a health check, run `bin/radar-docker health`. +To log to a separate directory, run `sudo bin/radar-log /my/LOG_DIR`. This can be useful to separate the logs of RADAR from the generic `syslog` file and limiting the total log size. + ### Monitoring a topic To see current data coming out of a Kafka topic, run @@ -143,13 +145,6 @@ sudo systemctl start radar-renew-certificate.service Otherwise, the following manual commands can be invoked. If `SELF_SIGNED_CERT=no` in `./.env`, be sure to run `bin/radar-cert-renew` daily to ensure that your certificate does not expire. - -### cAdvisor - -cAdvisor (Container Advisor) provides container users an understanding of the resource usage and performance characteristics of their running containers. - -To view current resource performance,if running locally, try . This will bring up the built-in Web UI. Clicking on `/docker` in `Subcontainers` takes you to a new window with all of the Docker containers listed individually. - ### Portainer Portainer provides simple interactive UI-based docker management. If running locally, try for portainer's UI. To set-up portainer follow this [link](https://www.ostechnix.com/portainer-an-easiest-way-to-manage-docker/). diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log new file mode 100755 index 000000000..a0e3c3cd6 --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -0,0 +1,66 @@ +#!/bin/bash + +trap catch_errors ERR; + +function catch_errors() { + exit_code=$? + echo "### FAILURE ###"; + exit $exit_code; +} + +if [ $# != 1 ]; then + echo "Set logging output directory" + echo "Usage: $0 DIRECTORY" + echo " Env variables:" + echo " LOG_SIZE maximum number of bytes for a log (default 1000000000, is one GB)" + exit 1 +fi + +LOG_DIR="$1" + +mkdir -p "$LOG_DIR" + +echo "Writing log directives to /etc/rsyslog.d/radar.conf" + +cat < /etc/rsyslog.d/radar.conf +if \$programname == 'radar-docker' then $LOG_DIR/radar-docker.log +& stop +if \$programname == 'radar-output' then $LOG_DIR/radar-output.log +& stop +if \$programname == 'radar-check-health' then $LOG_DIR/radar-check-health.log +& stop +if \$programname == 'radar-renew-certificate' then $LOG_DIR/radar-renew-certificate.log +& stop +EOF + +LOG_SIZE=${LOG_SIZE:-1000000000} + +echo "Writing log rotation directives to /etc/logrotate.d/radar.conf (with maximum size LOG_SIZE)" + +cat < /etc/logrotate.d/radar.conf +$LOG_DIR/radar-*.log { + daily + maxsize $LOG_SIZE + rotate 10 + delaycompress + compress + notifempty + missingok + postrotate + /usr/bin/killall -HUP syslogd + endscript +} +EOF + +echo "Running log rotation every hour" + +if [ ! -e /etc/cron.hourly.logrotate ]; then + ln -s /etc/cron.daily/logrotate /etc/cron.hourly.logrotate + systemctl restart cron +fi + +echo "Restarting rsyslog" + +systemctl restart rsyslog + +echo "Done." \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template index 5150878fa..17f7db5e3 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-check-health.service.template @@ -9,3 +9,7 @@ WorkingDirectory= ExecStart= NotifyAccess=all + +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=radar-check-health diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template index fcb1f83f2..13399e51b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-docker.service.template @@ -7,6 +7,10 @@ TimeoutStopSec=90 Restart=always RestartSec=10 +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=radar-docker + WorkingDirectory= ExecStart=./lib/systemd/start-radar-stack.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template index 547384bfe..b1f6627e9 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template @@ -10,3 +10,6 @@ ExecStart= NotifyAccess=all +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=radar-output diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template index 5150878fa..ffc3ef76c 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-renew-certificate.service.template @@ -9,3 +9,7 @@ WorkingDirectory= ExecStart= NotifyAccess=all + +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=radar-renew-certificate From 239ff6e6fb8e3df257d9563a04fa43fd081bfe25 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 11:55:43 +0100 Subject: [PATCH 158/194] Fix kafka consumer and systemd installation by root --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer | 2 +- .../radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer index 51b0a6f17..0dd74df9e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-kafka-consumer @@ -43,7 +43,7 @@ if ! array_contains topics ${args[0]}; then fi # consumer command to run -KACC_CMD="kafka-avro-console-consumer --broker-list kafka-1:9092,kafka-2:9092,kafka-3:9092 --property schema.registry.url=http://schema-registry-1:8081 --property print.key=true --topic ${args[0]} ${args[@]:1}" +KACC_CMD="kafka-avro-console-consumer --bootstrap-server kafka-1:9092,kafka-2:9092,kafka-3:9092 --property schema.registry.url=http://schema-registry-1:8081 --property print.key=true --topic ${args[0]} ${args[@]:1}" DOCKER_CMD="docker-compose exec schema-registry-1" # run consumer diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index 96db3d45a..812c02703 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -4,7 +4,7 @@ echo $(pwd) . lib/util.sh . .env -if id -Gn | grep -qe '\'; then +if [ "$(id -un)" == "root" ] || id -Gn | grep -qe '\'; then BASE=/etc/systemd/system SYSTEMCTL_OPTS=() else From 22b68a39255623bc951bda4b8918e6ce65a401d8 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 12:07:31 +0100 Subject: [PATCH 159/194] Usability and util fixes --- .../radar-cp-hadoop-stack/bin/hdfs-upgrade | 6 +----- .../radar-cp-hadoop-stack/bin/postgres-upgrade | 9 ++------- .../lib/install-systemd-wrappers.sh | 14 +++++++------- .../radar-cp-hadoop-stack/lib/perform-install.sh | 2 +- dcompose-stack/radar-cp-hadoop-stack/lib/util.sh | 13 +++++++++++++ 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade index 105f10fcc..c9388ad17 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-upgrade @@ -28,11 +28,7 @@ echo "==> Stopping HDFS name nodes" $stack quit hdfs-namenode-1 radar-hdfs-connector echo "==> Updating HDFS version to ${VERSION}" -if grep -q HDFS_BASE_VERSION ./.env; then - sed_i "s/HDFS_BASE_VERSION=.*/HDFS_BASE_VERSION=${VERSION}/" ./.env -else - echo "HDFS_BASE_VERSION=${VERSION}" >> ./.env -fi +ensure_variable "HDFS_BASE_VERSION=" "${VERSION}" .env $stack build diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade index d5c76e69f..82ab65eb5 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade @@ -49,13 +49,8 @@ mv "${MP_POSTGRES_DIR}/data/" "${MP_POSTGRES_DIR}/data-backup-$(date +%FT%TZ)/" mv "${POSTGRES_NEW_DIR}" "${MP_POSTGRES_DIR}/data/" # change postgres version -sed_i 's| image: postgres:.*| image: postgres:${POSTGRES_VERSION}|' docker-compose.yml - -if grep -q "POSTGRES_VERSION=" .env; then - sed_i "s|POSTGRES_VERSION=.*|POSTGRES_VERSION=${NEW_VERSION}|" .env -else - echo "POSTGRES_VERSION=${NEW_VERSION}" >> .env -fi +inline_variable " image: postgres:" "${POSTGRES_VERSION}" docker-compose.yml +ensure_variable "POSTGRES_VERSION=" "${NEW_VERSION}" .env echo "==> Starting postgres..." diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index 812c02703..a7c8631d9 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -15,13 +15,13 @@ else fi echo "==> Copying templates" -copy_template_if_absent $BASE/radar-docker.service lib/systemd/radar-docker.service.template -copy_template_if_absent $BASE/radar-output.service lib/systemd/radar-output.service.template -copy_template_if_absent $BASE/radar-output.timer lib/systemd/radar-output.timer.template -copy_template_if_absent $BASE/radar-check-health.service lib/systemd/radar-check-health.service.template -copy_template_if_absent $BASE/radar-check-health.timer lib/systemd/radar-check-health.timer.template -copy_template_if_absent $BASE/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template -copy_template_if_absent $BASE/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template +copy_template $BASE/radar-docker.service lib/systemd/radar-docker.service.template +copy_template $BASE/radar-output.service lib/systemd/radar-output.service.template +copy_template $BASE/radar-output.timer lib/systemd/radar-output.timer.template +copy_template $BASE/radar-check-health.service lib/systemd/radar-check-health.service.template +copy_template $BASE/radar-check-health.timer lib/systemd/radar-check-health.timer.template +copy_template $BASE/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template +copy_template $BASE/radar-renew-certificate.timer lib/systemd/radar-renew-certificate.timer.template echo "==> Inlining variables" inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-docker.service diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index b69362f25..b7508666b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -12,7 +12,7 @@ check_command_exists docker-compose check_config_present .env etc/env.template check_config_present etc/smtp.env check_config_present etc/radar-backend/radar.yml -check_config_present etc/managementportal/config/oauth_client_details.csv +copy_template_if_absent etc/managementportal/config/oauth_client_details.csv copy_template_if_absent etc/mongodb-connector/sink-mongo.properties copy_template_if_absent etc/hdfs-connector/sink-hdfs.properties copy_template_if_absent etc/rest-api/radar.yml diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh index 87d4db50c..4451d77c7 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh @@ -97,6 +97,19 @@ copy_template_if_absent() { fi } +# Copies the template (defined by the given config file with suffix +# ".template") to intended configuration file. +copy_template() { + template=${2:-${1}.template} + if [ ! -f "$1" ] && [ -e "$1" ]; then + echo "Configuration file ${1} is not a regular file." + exit 1 + else + echo "Copying configuration file ${1} from template ${template}" + sudo-linux cp -p "${template}" "$1" + fi +} + check_config_present() { template=${2:-${1}.template} if [ ! -f "$1" ]; then From 0f8ee6290916d6580ee3df8878f452b607c0a3aa Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 12:12:56 +0100 Subject: [PATCH 160/194] Use standard logrotate terminology --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-log | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log index a0e3c3cd6..d06605d56 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -35,9 +35,9 @@ EOF LOG_SIZE=${LOG_SIZE:-1000000000} -echo "Writing log rotation directives to /etc/logrotate.d/radar.conf (with maximum size LOG_SIZE)" +echo "Writing log rotation directives to /etc/logrotate.d/radar (with maximum size $LOG_SIZE)" -cat < /etc/logrotate.d/radar.conf +cat < /etc/logrotate.d/radar $LOG_DIR/radar-*.log { daily maxsize $LOG_SIZE @@ -47,7 +47,7 @@ $LOG_DIR/radar-*.log { notifempty missingok postrotate - /usr/bin/killall -HUP syslogd + invoke-rc.d rsyslog rotate > /dev/null endscript } EOF From 30f9ceaa8af6780a71236fc4ad56e0ae5159d2bf Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 12:14:42 +0100 Subject: [PATCH 161/194] Fix typo --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log index d06605d56..00af3f25e 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -55,7 +55,7 @@ EOF echo "Running log rotation every hour" if [ ! -e /etc/cron.hourly.logrotate ]; then - ln -s /etc/cron.daily/logrotate /etc/cron.hourly.logrotate + ln -s /etc/cron.daily/logrotate /etc/cron.hourly/logrotate systemctl restart cron fi From 738e20174d4b8ca70ee198e7e879104e7a478342 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 13:42:41 +0100 Subject: [PATCH 162/194] Fix syslog config --- .../radar-cp-hadoop-stack/bin/radar-log | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log index 00af3f25e..4c0613ddb 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -19,18 +19,32 @@ fi LOG_DIR="$1" mkdir -p "$LOG_DIR" +chown root:syslog "$LOG_DIR" +chmod g+w "$LOG_DIR" -echo "Writing log directives to /etc/rsyslog.d/radar.conf" +echo "Writing log directives to /etc/rsyslog.d/99-radar.conf" -cat < /etc/rsyslog.d/radar.conf -if \$programname == 'radar-docker' then $LOG_DIR/radar-docker.log -& stop -if \$programname == 'radar-output' then $LOG_DIR/radar-output.log -& stop -if \$programname == 'radar-check-health' then $LOG_DIR/radar-check-health.log -& stop -if \$programname == 'radar-renew-certificate' then $LOG_DIR/radar-renew-certificate.log -& stop +cat < /etc/rsyslog.d/99-radar.conf +if ( \$programname == "radar-docker" ) then { + action(type="omfile" file="$LOG_DIR/radar-docker.log") + stop +} +if ( \$programname == "radar-output" ) then { + action(type="omfile" file="$LOG_DIR/radar-output.log") + stop +} +if ( \$programname == "radar-renew-certificate" ) then { + action(type="omfile" file="$LOG_DIR/radar-renew-certificate.log") + stop +} +if ( \$programname == "radar-check-health" ) then { + action(type="omfile" file="$LOG_DIR/radar-check-health.log") + stop +} +if ( $programname startswith "docker" ) then { + action(type="omfile" file="$LOG_DIR/docker.log") + stop +} EOF LOG_SIZE=${LOG_SIZE:-1000000000} From 6e869a79dc41ffd61b2fd83b430f5450f92d06a4 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 14:00:45 +0100 Subject: [PATCH 163/194] Typo fix --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log index 4c0613ddb..2b3771efb 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -68,7 +68,7 @@ EOF echo "Running log rotation every hour" -if [ ! -e /etc/cron.hourly.logrotate ]; then +if [ ! -e /etc/cron.hourly/logrotate ]; then ln -s /etc/cron.daily/logrotate /etc/cron.hourly/logrotate systemctl restart cron fi From 9213da48119049f83d9002288f70463afafff92a Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 15:16:21 +0100 Subject: [PATCH 164/194] Do not run in interactive mode for output --- .../lib/systemd/radar-output.service.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template index b1f6627e9..94b83fd03 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template @@ -10,6 +10,8 @@ ExecStart= NotifyAccess=all +Environment=DOCKER_OPTS= + StandardOutput=syslog StandardError=syslog SyslogIdentifier=radar-output From 3bdeefc68d93d58a26a47263f0ba1fef526102be Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 15:17:03 +0100 Subject: [PATCH 165/194] Capture docker-related output before other syslog tasks --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log index 2b3771efb..037cb4705 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -22,7 +22,7 @@ mkdir -p "$LOG_DIR" chown root:syslog "$LOG_DIR" chmod g+w "$LOG_DIR" -echo "Writing log directives to /etc/rsyslog.d/99-radar.conf" +echo "Writing log directives to /etc/rsyslog.d/00-radar.conf" cat < /etc/rsyslog.d/99-radar.conf if ( \$programname == "radar-docker" ) then { From 97a93cb6e98bd46622e8169b9156dcc1f8028728 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 15:18:00 +0100 Subject: [PATCH 166/194] Fixed typo --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log index 037cb4705..e21c0a377 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -41,7 +41,7 @@ if ( \$programname == "radar-check-health" ) then { action(type="omfile" file="$LOG_DIR/radar-check-health.log") stop } -if ( $programname startswith "docker" ) then { +if ( \$programname startswith "docker" ) then { action(type="omfile" file="$LOG_DIR/docker.log") stop } From c04c20fe51d9362712bfcddbf24c2dfb3d885745 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 16:40:35 +0100 Subject: [PATCH 167/194] Fix already existing typo --- dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade index 82ab65eb5..2b1708a84 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade @@ -49,7 +49,7 @@ mv "${MP_POSTGRES_DIR}/data/" "${MP_POSTGRES_DIR}/data-backup-$(date +%FT%TZ)/" mv "${POSTGRES_NEW_DIR}" "${MP_POSTGRES_DIR}/data/" # change postgres version -inline_variable " image: postgres:" "${POSTGRES_VERSION}" docker-compose.yml +inline_variable " image: postgres:" "${NEW_VERSION}" docker-compose.yml ensure_variable "POSTGRES_VERSION=" "${NEW_VERSION}" .env echo "==> Starting postgres..." From 4d9cb79dbdac208c559e66ebcc97538d96a79e63 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 16:47:14 +0100 Subject: [PATCH 168/194] Fix hadoop network --- .../radar-cp-hadoop-stack/lib/perform-install.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index b7508666b..272f5cbec 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -60,7 +60,12 @@ fi # Create networks and volumes if ! sudo-linux docker network ls --format '{{.Name}}' | grep -q "^hadoop$"; then echo "==> Creating docker network - hadoop" - sudo-linux docker network create hadoop > /dev/null + sudo-linux docker network create --internal hadoop > /dev/null +elif [ $(docker network inspect hadoop --format "{{.Internal}}") != "true" ]; then + echo "==> Re-creating docker network - hadoop" + sudo-linux bin/radar-docker quit radar-hdfs-connector hdfs-namenode-1 hdfs-datanode-1 hdfs-datanode-2 hdfs-datanode-3 > /dev/null + sudo-linux docker network rm hadoop > /dev/null + sudo-linux docker network create --internal hadoop > /dev/null else echo "==> Creating docker network - hadoop ALREADY EXISTS" fi From 962f0ed9effc1a346f89e2b72f5305c2c2dda1c9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 16:50:24 +0100 Subject: [PATCH 169/194] Correct radar-output systemd template --- .../lib/systemd/radar-output.service.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template index 547384bfe..d1c7d1c88 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template @@ -10,3 +10,5 @@ ExecStart= NotifyAccess=all +Environment=DOCKER_OPTS= + From 8062f349c500517f64e8f5f1f051ba61c70b8661 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 5 Nov 2018 16:51:45 +0100 Subject: [PATCH 170/194] Fix logrotate conf filename --- dcompose-stack/radar-cp-hadoop-stack/bin/radar-log | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log index e21c0a377..1672a0ab1 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/radar-log @@ -24,7 +24,7 @@ chmod g+w "$LOG_DIR" echo "Writing log directives to /etc/rsyslog.d/00-radar.conf" -cat < /etc/rsyslog.d/99-radar.conf +cat < /etc/rsyslog.d/00-radar.conf if ( \$programname == "radar-docker" ) then { action(type="omfile" file="$LOG_DIR/radar-docker.log") stop @@ -77,4 +77,4 @@ echo "Restarting rsyslog" systemctl restart rsyslog -echo "Done." \ No newline at end of file +echo "Done." From b29566e4cb5ae5e0ce5c488d1b10be218ac2d313 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 7 Nov 2018 10:00:13 +0100 Subject: [PATCH 171/194] Add README of how to revert logging to old situation --- dcompose-stack/radar-cp-hadoop-stack/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/README.md b/dcompose-stack/radar-cp-hadoop-stack/README.md index ad8a76e44..beae51fc8 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/README.md +++ b/dcompose-stack/radar-cp-hadoop-stack/README.md @@ -102,7 +102,17 @@ sudo systemctl start radar-docker To rebuild an image and restart them, run `bin/radar-docker rebuild IMAGE`. To stop and remove an container, run `bin/radar-docker quit CONTAINER`. To start the HDFS cluster, run `bin/radar-docker hdfs`. For a health check, run `bin/radar-docker health`. -To log to a separate directory, run `sudo bin/radar-log /my/LOG_DIR`. This can be useful to separate the logs of RADAR from the generic `syslog` file and limiting the total log size. +To log to a separate directory, run +```shell +sudo bin/radar-log /my/LOG_DIR +``` + +This can be useful to separate the logs of RADAR from the generic `/var/log/syslog` file and limiting the total log size. To revert logging to `/var/log/syslog`, run + +```shell +sudo rm /etc/rsyslog.d/00-radar.conf /etc/logrotate.d/radar /etc/cron.hourly/logrotate +sudo systemctl restart rsyslog +``` ### Monitoring a topic From f708ddf2cdc5a6dd1b43b94b4bdd16f604253ddf Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 15 Nov 2018 15:12:38 +0100 Subject: [PATCH 172/194] add health check fixes --- dcompose-stack/radar-cp-hadoop-stack/optional-services.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml index b5f0f1b94..2c2414209 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/optional-services.yml @@ -90,7 +90,7 @@ services: volumes: - ./etc/rest-source-authorizer/:/app-includes/ healthcheck: - test: ["CMD", "curl", "-f", "localhost:8080/users"] + test: ["CMD", "wget", "--spider", "http://localhost:8080/users"] interval: 1m30s timeout: 5s retries: 3 @@ -102,3 +102,8 @@ services: depends_on: - radar-rest-sources-backend - radarbase-postgresql + healthcheck: + test: ["CMD", "wget", "--spider", "http://localhost:80"] + interval: 1m30s + timeout: 5s + retries: 3 From e17d3db8a374238fc7f85cf876b09feb11bb40ad Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 15 Nov 2018 16:21:46 +0100 Subject: [PATCH 173/194] More dynamic postgres timeout --- .../images/postgres/entrypoint.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh index 77d395593..7a6b7ddfa 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh @@ -6,14 +6,16 @@ DB_HOST="localhost" DB_PORT=5432 function wait_for_db() { - for count in {1..30}; do - echo "Pinging postgres database attempt "${count} + echo "Waiting for postgres database..." + for count in {1..120}; do if $(nc -z ${DB_HOST} ${DB_PORT}) ; then - echo "Can connect into database" - break + echo "Database ready." + sleep 5 + return 0 fi sleep 1 done + return 1 } function create_user_and_database() { local database=$1 @@ -37,9 +39,12 @@ EOSQL if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" #waiting for postgres - wait_for_db + if ! wait_for_db; then + echo "Postgres database timeout" + exit 1 + fi for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do create_user_and_database $db done echo "Multiple databases created" -fi \ No newline at end of file +fi From b6e93f815b64f3d8b7921aa693901b529bdffb9b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 15 Nov 2018 16:37:36 +0100 Subject: [PATCH 174/194] Fix indents and bash nc syntax --- .../images/postgres/entrypoint.sh | 69 +++++++++---------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh index 7a6b7ddfa..1d1d23f0c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh @@ -6,45 +6,44 @@ DB_HOST="localhost" DB_PORT=5432 function wait_for_db() { - echo "Waiting for postgres database..." - for count in {1..120}; do - if $(nc -z ${DB_HOST} ${DB_PORT}) ; then - echo "Database ready." - sleep 5 - return 0 - fi - sleep 1 - done - return 1 + echo "Waiting for postgres database..." + for count in {1..120}; do + if nc -z ${DB_HOST} ${DB_PORT}; then + echo "Database ready." + sleep 5 + return 0 + fi + sleep 1 + done + return 1 } + function create_user_and_database() { - local database=$1 - echo "Processing database '$database'" - local query_databases="select datname from pg_database;" - local database_exist=$(psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'") - if [[ "$database_exist" == 1 ]]; - then - echo "Database already exists" - else - echo "Database does not exist" - echo " Creating database '$database' for user '$POSTGRES_USER'" - psql -v ON_ERROR_STOP=1 <<-EOSQL - CREATE DATABASE $database; - GRANT ALL PRIVILEGES ON DATABASE $database TO $POSTGRES_USER; + local database=$1 + echo "Processing database '$database'" + local query_databases="select datname from pg_database;" + local database_exist=$(psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'") + if [[ "$database_exist" == 1 ]]; then + echo "Database already exists" + else + echo "Database does not exist" + echo " Creating database '$database' for user '$POSTGRES_USER'" + psql -v ON_ERROR_STOP=1 <<-EOSQL + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $POSTGRES_USER; EOSQL - fi - + fi } if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then - echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" - #waiting for postgres - if ! wait_for_db; then - echo "Postgres database timeout" - exit 1 - fi - for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do - create_user_and_database $db - done - echo "Multiple databases created" + echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" + #waiting for postgres + if ! wait_for_db; then + echo "Postgres database timeout" + exit 1 + fi + for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do + create_user_and_database $db + done + echo "Multiple databases created" fi From 983a8441aa146355ba7daf47d926f6d8f8e75260 Mon Sep 17 00:00:00 2001 From: nivethika Date: Thu, 15 Nov 2018 17:01:25 +0100 Subject: [PATCH 175/194] fix error in container-name --- dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade index 93441cc09..61f43ccfe 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/postgres-upgrade @@ -35,14 +35,14 @@ POSTGRES_NEW=$(docker run -d -v "${POSTGRES_NEW_DIR}/:/var/lib/postgresql/data" sleep 5 echo "==> Migrating ManagementPortal database to ${NEW_VERSION}" -docker-compose exec managementportal-postgresql pg_dumpall -U "${POSTGRES_USER}" \ +docker-compose exec radarbase-postgresql pg_dumpall -U "${POSTGRES_USER}" \ | docker exec -i ${POSTGRES_NEW} psql -U "${POSTGRES_USER}" docker rm -vf "${POSTGRES_NEW}" echo "==> Stopping postgres..." -docker-compose stop managementportal-postgresql -docker-compose rm -vf managementportal-postgresql +docker-compose stop radarbase-postgresql +docker-compose rm -vf radarbase-postgresql echo "==> Moving old data to backup ${MP_POSTGRES_DIR}/data-backup-$(date +%FT%TZ)/" mv "${MP_POSTGRES_DIR}/data/" "${MP_POSTGRES_DIR}/data-backup-$(date +%FT%TZ)/" @@ -54,4 +54,4 @@ ensure_variable "POSTGRES_VERSION=" "${NEW_VERSION}" .env echo "==> Starting postgres..." -docker-compose up -d managementportal-postgresql +docker-compose up -d radarbase-postgresql From 4bb7f0d9b2f0e214eab99261cdab55a95cbe7e75 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 19 Nov 2018 09:39:52 +0100 Subject: [PATCH 176/194] Fixed multi-db-init --- .../images/postgres/Dockerfile | 4 +-- .../{entrypoint.sh => multi-db-init.sh} | 27 +++---------------- .../images/postgres/on-db-ready | 17 ++++++++++++ .../lib/perform-install.sh | 2 +- 4 files changed, 23 insertions(+), 27 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/images/postgres/{entrypoint.sh => multi-db-init.sh} (58%) create mode 100755 dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile index f9b291bd1..87673629a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile @@ -1,5 +1,5 @@ ARG POSTGRES_VERSION=10.4-alpine FROM postgres:${POSTGRES_VERSION} -COPY ./entrypoint.sh /docker-entrypoint-initdb.d/multi-db-init.sh - +COPY ./multi-db-init.sh /docker-entrypoint-initdb.d/multi-db-init.sh +COPY ./on-db-ready /usr/bin/on-db-ready diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh similarity index 58% rename from dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh rename to dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh index 1d1d23f0c..4135e7d59 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/entrypoint.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh @@ -2,31 +2,13 @@ set -e set -u -DB_HOST="localhost" -DB_PORT=5432 - -function wait_for_db() { - echo "Waiting for postgres database..." - for count in {1..120}; do - if nc -z ${DB_HOST} ${DB_PORT}; then - echo "Database ready." - sleep 5 - return 0 - fi - sleep 1 - done - return 1 -} - function create_user_and_database() { local database=$1 - echo "Processing database '$database'" - local query_databases="select datname from pg_database;" local database_exist=$(psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'") if [[ "$database_exist" == 1 ]]; then - echo "Database already exists" + echo "Database $database already exists" else - echo "Database does not exist" + echo "Database $database does not exist" echo " Creating database '$database' for user '$POSTGRES_USER'" psql -v ON_ERROR_STOP=1 <<-EOSQL CREATE DATABASE $database; @@ -38,12 +20,9 @@ EOSQL if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" #waiting for postgres - if ! wait_for_db; then - echo "Postgres database timeout" - exit 1 - fi for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do create_user_and_database $db done + echo -n "$POSTGRES_MULTIPLE_DATABASES" > /var/run/postgresql/multiple_databases_ready echo "Multiple databases created" fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready new file mode 100755 index 000000000..56f4503bc --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready @@ -0,0 +1,17 @@ +#!/bin/bash +set -e +set -u + +if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + echo "Waiting for postgres database..." + for count in {1..120}; do + if pg_isready -q && [ "$(cat /var/run/postgresql/multiple_databases_ready 2> /dev/null)" = "$POSTGRES_MULTIPLE_DATABASES" ]; then + echo "Database ready." + exec "$@" + fi + sleep 1 + done + + echo "Postgres database timeout" + exit 1 +fi \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index ef910cfe2..64f5ef2d6 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -118,7 +118,7 @@ ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-h echo "==> Configuring Management Portal" sudo-linux bin/radar-docker up -d --build radarbase-postgresql -sudo-linux bin/radar-docker exec --user postgres radarbase-postgresql /docker-entrypoint-initdb.d/multi-db-init.sh +sudo-linux bin/radar-docker exec --user postgres radarbase-postgresql on-db-ready /docker-entrypoint-initdb.d/multi-db-init.sh ensure_env_password MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET "ManagementPortal front-end client secret is not set in .env" ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." From 9418005a3f870cc33adbfb13273587579517239b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 19 Nov 2018 09:45:38 +0100 Subject: [PATCH 177/194] Updated default postgres version to 10.6 --- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 2 +- dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile | 2 +- .../radar-cp-hadoop-stack/images/postgres/multi-db-init.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 961b2e463..e45b8a4c0 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -26,7 +26,7 @@ HDFS_NAME_DIR_2=/usr/local/var/lib/docker/hdfs-name-2 HDFS_JOURNAL_DIR_1=/usr/local/var/lib/docker/hdfs-journal-1 HDFS_JOURNAL_DIR_2=/usr/local/var/lib/docker/hdfs-journal-2 HDFS_JOURNAL_DIR_3=/usr/local/var/lib/docker/hdfs-journal-3 -POSTGRES_VERSION=10.4-alpine +POSTGRES_VERSION=10.6-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD= PORTAINER_PASSWORD_HASH= diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile index 87673629a..b08fb6db5 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/Dockerfile @@ -1,4 +1,4 @@ -ARG POSTGRES_VERSION=10.4-alpine +ARG POSTGRES_VERSION=10.6-alpine FROM postgres:${POSTGRES_VERSION} COPY ./multi-db-init.sh /docker-entrypoint-initdb.d/multi-db-init.sh diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh index 4135e7d59..c413b13af 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh @@ -24,5 +24,5 @@ if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then create_user_and_database $db done echo -n "$POSTGRES_MULTIPLE_DATABASES" > /var/run/postgresql/multiple_databases_ready - echo "Multiple databases created" + echo "Databases created" fi From a9b39c03f2b5e8dc85f241d5488983690c5bbc96 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 19 Nov 2018 10:16:10 +0100 Subject: [PATCH 178/194] Forward-compatible with 10.6-alpine image --- .../images/postgres/multi-db-init.sh | 9 ++++----- .../radar-cp-hadoop-stack/images/postgres/on-db-ready | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh index c413b13af..a5e9229ca 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/multi-db-init.sh @@ -4,15 +4,15 @@ set -u function create_user_and_database() { local database=$1 - local database_exist=$(psql -tAc "SELECT 1 FROM pg_database WHERE datname='$database'") + local database_exist=$(psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$database';") if [[ "$database_exist" == 1 ]]; then echo "Database $database already exists" else echo "Database $database does not exist" echo " Creating database '$database' for user '$POSTGRES_USER'" - psql -v ON_ERROR_STOP=1 <<-EOSQL - CREATE DATABASE $database; - GRANT ALL PRIVILEGES ON DATABASE $database TO $POSTGRES_USER; + psql -U "$POSTGRES_USER" -v ON_ERROR_STOP=1 <<-EOSQL + CREATE DATABASE "$database"; + GRANT ALL PRIVILEGES ON DATABASE $database TO "$POSTGRES_USER"; EOSQL fi } @@ -23,6 +23,5 @@ if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do create_user_and_database $db done - echo -n "$POSTGRES_MULTIPLE_DATABASES" > /var/run/postgresql/multiple_databases_ready echo "Databases created" fi diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready index 56f4503bc..f1a9869cb 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready @@ -4,8 +4,9 @@ set -u if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then echo "Waiting for postgres database..." + HOSTNAME=$(hostname) for count in {1..120}; do - if pg_isready -q && [ "$(cat /var/run/postgresql/multiple_databases_ready 2> /dev/null)" = "$POSTGRES_MULTIPLE_DATABASES" ]; then + if pg_isready -h $HOSTNAME; then echo "Database ready." exec "$@" fi From 0649632abc09959d58f901a4bcefd8228ed7267f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 19 Nov 2018 10:28:52 +0100 Subject: [PATCH 179/194] Small pg_isready command-line fixes --- .../radar-cp-hadoop-stack/images/postgres/on-db-ready | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready index f1a9869cb..b3b2eddad 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready +++ b/dcompose-stack/radar-cp-hadoop-stack/images/postgres/on-db-ready @@ -6,7 +6,7 @@ if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then echo "Waiting for postgres database..." HOSTNAME=$(hostname) for count in {1..120}; do - if pg_isready -h $HOSTNAME; then + if pg_isready -U "$POSTGRES_USER" -q -h $HOSTNAME; then echo "Database ready." exec "$@" fi From b353f16f99b1e825d41f32032ea1036078e1bef5 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 19 Nov 2018 11:46:46 +0100 Subject: [PATCH 180/194] Fix postgres recreate issue --- dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 64f5ef2d6..3b8fad73d 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -117,7 +117,8 @@ fi ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-hdfs.properties echo "==> Configuring Management Portal" -sudo-linux bin/radar-docker up -d --build radarbase-postgresql +sudo-linux bin/radar-docker build --no-cache radarbase-postgresql +sudo-linux bin/radar-docker up -d --force-recreate radarbase-postgresql sudo-linux bin/radar-docker exec --user postgres radarbase-postgresql on-db-ready /docker-entrypoint-initdb.d/multi-db-init.sh ensure_env_password MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET "ManagementPortal front-end client secret is not set in .env" ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." From 6be5557a028d2e9ce889ddcb92525242302cb346 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 20 Nov 2018 13:18:19 +0100 Subject: [PATCH 181/194] Use PKCS12 instead of JKS keystore --- .../radar-cp-hadoop-stack/bin/keystore-init | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init index ff36400c8..87bf9e0a9 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/keystore-init @@ -9,7 +9,7 @@ function createKeyStore() { file=$1 if ! keytool -list $KEYTOOL_OPTS -alias radarbase-managementportal-ec >/dev/null 2>/dev/null; then - KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype JKS $KEYSTORE_CREATE_OPTS" + KEYTOOL_CREATE_OPTS="-genkeypair -alias radarbase-managementportal-ec -keyalg EC -keysize 256 -sigalg SHA256withECDSA -storetype PKCS12 $KEYSTORE_CREATE_OPTS" if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" fi @@ -20,7 +20,7 @@ function createKeyStore() { fi if ! keytool -list $KEYTOOL_OPTS -alias selfsigned >/dev/null 2>/dev/null; then - KEYTOOL_CREATE_OPTS="-genkeypair -alias selfsigned -keyalg RSA -keysize 4096 -storetype JKS $KEYSTORE_CREATE_OPTS" + KEYTOOL_CREATE_OPTS="-genkeypair -alias selfsigned -keyalg RSA -keysize 4096 -storetype PKCS12 $KEYSTORE_CREATE_OPTS" if [ -n "${MANAGEMENTPORTAL_KEY_DNAME}" ]; then KEYTOOL_CREATE_OPTS="$KEYTOOL_CREATE_OPTS -dname ${MANAGEMENTPORTAL_KEY_DNAME}" fi @@ -33,6 +33,17 @@ function createKeyStore() { chmod 400 "$file" } +function convertJksToPkcs12() { + src=$1 + dest=$2 + + if [ ! -e $dest ] && [ -e $src ]; then + echo "--> Importing PKCS12 key store from existing JKS key store" + keytool -importkeystore -srckeystore $src -destkeystore $dest -srcstoretype JKS -deststoretype PKCS12 -deststorepass radarbase -srcstorepass radarbase + chmod 400 $dest + fi +} + function writeKeys() { FILE=$1 RES=$2 @@ -60,8 +71,12 @@ function writeKeys() { done } -keystorefile=etc/managementportal/config/keystore.jks -export KEYTOOL_OPTS="-keystore ${keystorefile} -storepass radarbase -keypass radarbase $KEYTOOL_OPTS" +keystorefile=etc/managementportal/config/keystore.p12 +oldkeystorefile=etc/managementportal/config/keystore.jks + +convertJksToPkcs12 $oldkeystorefile $keystorefile + +export KEYTOOL_OPTS="-keystore ${keystorefile} -storepass radarbase -keypass radarbase $KEYSTORE_INIT_OPTS" createKeyStore "$keystorefile" From d255221fb5e5c7206cb610d543333f2ffd1a5894 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 20 Nov 2018 13:28:21 +0100 Subject: [PATCH 182/194] Update dependencies --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index b53de23b1..d5fbc58cf 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -643,7 +643,7 @@ services: # Webserver # #---------------------------------------------------------------------------# webserver: - image: nginx:1.13.1-alpine + image: nginx:1.14.0-alpine restart: always networks: - api @@ -676,7 +676,7 @@ services: # Management Portal # #---------------------------------------------------------------------------# managementportal-app: - image: radarbase/management-portal:0.5.1 + image: radarbase/management-portal:0.5.2 networks: - default - api From d22ce84ab3bea18e160184642289e7c382aef039 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 21 Nov 2018 11:48:01 +0100 Subject: [PATCH 183/194] Enable monthly snapshots --- .../bin/hdfs-restructure | 70 +++++++++++++++---- .../radar-cp-hadoop-stack/etc/env.template | 1 + 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index fcd9b2d33..6427ac0f5 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -1,12 +1,29 @@ #!/bin/bash +if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then + printf "Usage:\n$0 []\nThe destination directory defaults to ./output\n" + exit 1 +fi + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# Absolute directory to write output to +OUTPUT_DIR=${2:-$DIR/output} +OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" + +cd $DIR + +. .env + # HDFS restructure version DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.3 + +NUM_THREADS=${RESTRUCTURE_NUM_THREADS:-3} # HDFS restructure script flags HDFS_RESTRUCTURE_OPTS=( --compression gzip --deduplicate - --num-threads 3 + --num-threads $NUM_THREADS ) OUTPUT_USER=${OUTPUT_USER:-$(id -un)} OUTPUT_GROUP=${OUTPUT_GROUP:-$(id -gn)} @@ -35,19 +52,6 @@ if [ ! -z $PROFILE_PORT ]; then DOCKER_OPTS="$DOCKER_OPTS -p ${PROFILE_PORT}:${PROFILE_PORT}" fi -if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then - printf "Usage:\n$0 []\nThe destination directory defaults to ./output\n" - exit 1 -fi - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -# Absolute directory to write output to -OUTPUT_DIR=${2:-$DIR/output} -OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" - -cd $DIR - . lib/util.sh # Start HDFS if not started already @@ -73,3 +77,41 @@ sudo-linux docker run ${DOCKER_OPTS} \ -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" \ -e RADAR_HDFS_RESTRUCTURE_OPTS \ $DOCKER_IMAGE "${HDFS_COMMAND[@]}" + +if ! ${RESTRUCTURE_ENABLE_SNAPSHOTS:-true}; then + echo "Snapshots disabled. No snapshots will be made." + exit 0 +fi + +if ! command -v zip > /dev/null 2>&1; then + echo "WARN: zip not installed. No snapshots will be made." + exit 0 +fi + +function create_snapshots() { + PROJECT_DIR=$1 + project=basename "$1" + cd "$PROJECT_DIR" + SNAPSHOT_DIR=../snapshots/$project + SNAPSHOT_TMP_DIR=../snapshots/+tmp + mkdir -p $SNAPSHOT_DIR $SNAPSHOT_TMP_DIR + + months=$(find . -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9].*" -printf "%f\n" | cut -c-6 | sort -u) + for month in $months; do + SNAPSHOT=$SNAPSHOT_DIR/$month.zip + SNAPSHOT_TMP=$SNAPSHOT_TMP_DIR/$month.zip + if [ -e $SNAPSHOT ]; then + find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -@u $SNAPSHOT -O $SNAPSHOT_TMP + if [ -e $SNAPSHOT_TMP ]; then + mv $SNAPSHOT_TMP $SNAPSHOT + fi + else + find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -@ $SNAPSHOT_TMP + mv $SNAPSHOT_TMP $SNAPSHOT + fi + done + cd "$OLDPWD" +} + +export -f create_snapshots +find "$OUTPUT_DIR" -mindepth 1 -maxdepth 1 -type d ! -name snapshots ! -name +tmp -print0 | xargs -0 -n 1 -P $NUM_THREADS -I {} bash -c 'create_snapshots "$@"' _ {} diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index e45b8a4c0..2a7ccc269 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -38,3 +38,4 @@ MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-priv ENABLE_OPTIONAL_SERVICES=false FITBIT_API_CLIENT_ID=fitbit-client FITBIT_API_CLIENT_SECRET=fitbit-secret +ENABLE_SNAPSHOTS=true From 20daaa5157a44e37021a9abe6e8d1cf22cab38c4 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 21 Nov 2018 11:51:52 +0100 Subject: [PATCH 184/194] Fix env template --- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 2a7ccc269..753f22668 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -23,9 +23,6 @@ HDFS_DATA_DIR_2=/usr/local/var/lib/docker/hdfs-data-2 HDFS_DATA_DIR_3=/usr/local/var/lib/docker/hdfs-data-3 HDFS_NAME_DIR_1=/usr/local/var/lib/docker/hdfs-name-1 HDFS_NAME_DIR_2=/usr/local/var/lib/docker/hdfs-name-2 -HDFS_JOURNAL_DIR_1=/usr/local/var/lib/docker/hdfs-journal-1 -HDFS_JOURNAL_DIR_2=/usr/local/var/lib/docker/hdfs-journal-2 -HDFS_JOURNAL_DIR_3=/usr/local/var/lib/docker/hdfs-journal-3 POSTGRES_VERSION=10.6-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD= @@ -33,9 +30,10 @@ PORTAINER_PASSWORD_HASH= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned RESTRUCTURE_OUTPUT_DIR=output +RESTRUCTURE_NUM_THREADS=3 +RESTRUCTURE_ENABLE_SNAPSHOTS=true MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD= MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy ENABLE_OPTIONAL_SERVICES=false FITBIT_API_CLIENT_ID=fitbit-client FITBIT_API_CLIENT_SECRET=fitbit-secret -ENABLE_SNAPSHOTS=true From 2597001c47d3bd6932a46782b0bf4a68173159cf Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 21 Nov 2018 11:54:37 +0100 Subject: [PATCH 185/194] Do not compress already compressed files --- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 6427ac0f5..bcb562601 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -101,12 +101,12 @@ function create_snapshots() { SNAPSHOT=$SNAPSHOT_DIR/$month.zip SNAPSHOT_TMP=$SNAPSHOT_TMP_DIR/$month.zip if [ -e $SNAPSHOT ]; then - find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -@u $SNAPSHOT -O $SNAPSHOT_TMP + find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -0@u $SNAPSHOT -O $SNAPSHOT_TMP if [ -e $SNAPSHOT_TMP ]; then mv $SNAPSHOT_TMP $SNAPSHOT fi else - find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -@ $SNAPSHOT_TMP + find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -0@ $SNAPSHOT_TMP mv $SNAPSHOT_TMP $SNAPSHOT fi done From c72e8f6f4296d1f5399ecf20c08e35b17ae1bc2f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 26 Nov 2018 15:20:00 +0100 Subject: [PATCH 186/194] Update radar-schemas --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 4 ++-- .../radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile | 2 +- dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index b53de23b1..f0867c40f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -260,7 +260,7 @@ services: #---------------------------------------------------------------------------# kafka-init: build: images/radar-kafka-init - image: radarbase/kafka-init:0.4.1 + image: radarbase/kafka-init:0.4.2 networks: - kafka - zookeeper @@ -784,7 +784,7 @@ services: #---------------------------------------------------------------------------# catalog-server: build: images/radar-kafka-init - image: radarbase/kafka-init:0.4.1 + image: radarbase/kafka-init:0.4.2 networks: - management command: radar-schemas-tools serve /schema/merged diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile index 600da13b1..bfd3e590b 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile @@ -26,7 +26,7 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" -ENV RADAR_SCHEMAS_VERSION=0.4.1 +ENV RADAR_SCHEMAS_VERSION=0.4.2 RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 3b8fad73d..07d2d74f4 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -95,7 +95,7 @@ ensure_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mo KAFKA_INIT_OPTS=( --rm -v "$PWD/etc/schema:/schema/conf" - radarbase/kafka-init:0.3.6 + radarbase/kafka-init:0.4.2 ) # Set topics From 5357f8ed01abc7d7cb16d10b0641a888344e71bd Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 26 Nov 2018 15:24:00 +0100 Subject: [PATCH 187/194] Update RADAR-Gateway --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 30f7b621a..5fc22b68f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -761,7 +761,7 @@ services: # RADAR Gateway # #---------------------------------------------------------------------------# gateway: - image: radarbase/radar-gateway:0.3.1 + image: radarbase/radar-gateway:0.3.2 networks: - api - kafka From 020b98d0935d156ce9331b06a70d4f039c7a0a0f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 26 Nov 2018 16:24:56 +0100 Subject: [PATCH 188/194] Fix postgres in background --- dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 07d2d74f4..a70b47dff 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -119,7 +119,7 @@ ensure_variable 'topics=' "${COMBINED_RAW_TOPIC_LIST}" etc/hdfs-connector/sink-h echo "==> Configuring Management Portal" sudo-linux bin/radar-docker build --no-cache radarbase-postgresql sudo-linux bin/radar-docker up -d --force-recreate radarbase-postgresql -sudo-linux bin/radar-docker exec --user postgres radarbase-postgresql on-db-ready /docker-entrypoint-initdb.d/multi-db-init.sh +sudo-linux bin/radar-docker exec --user postgres -T radarbase-postgresql on-db-ready /docker-entrypoint-initdb.d/multi-db-init.sh ensure_env_password MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET "ManagementPortal front-end client secret is not set in .env" ensure_env_password MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD "Admin password for ManagementPortal is not set in .env." From bc4a66db581b1891ef0372843143c7cc57c0a8f9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 26 Nov 2018 16:27:34 +0100 Subject: [PATCH 189/194] Ignore keystore.p12 --- dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore index ac0e72cac..f026b7255 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/.gitignore @@ -18,3 +18,4 @@ radar-is.yml /fitbit/docker/users/*.yml /fitbit/docker/source-fitbit.properties /rest-source-authorizer/rest_source_clients_configs.yml +keystore.p12 From eebe8783d89738188683208b8bb845908519833f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 27 Nov 2018 11:49:05 +0100 Subject: [PATCH 190/194] Update project output dir --- dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index bcb562601..42aa4a3e6 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -89,8 +89,8 @@ if ! command -v zip > /dev/null 2>&1; then fi function create_snapshots() { - PROJECT_DIR=$1 - project=basename "$1" + PROJECT_DIR="$1" + project=$(basename "${PROJECT_DIR}") cd "$PROJECT_DIR" SNAPSHOT_DIR=../snapshots/$project SNAPSHOT_TMP_DIR=../snapshots/+tmp From 701d0a867bac2ac9502da3812e118e2e030bfb08 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 28 Nov 2018 09:19:20 +0100 Subject: [PATCH 191/194] Fix gateway http code propagation --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 5fc22b68f..6f0e1ab0a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -761,7 +761,7 @@ services: # RADAR Gateway # #---------------------------------------------------------------------------# gateway: - image: radarbase/radar-gateway:0.3.2 + image: radarbase/radar-gateway:0.3.3 networks: - api - kafka From 65ea719ab4f8030eba367705cc21cf519a075b46 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 28 Nov 2018 13:42:53 +0100 Subject: [PATCH 192/194] Make radar-schemas version variable --- .../radar-cp-hadoop-stack/docker-compose.yml | 14 ++++++++++---- .../radar-cp-hadoop-stack/etc/env.template | 1 + .../images/radar-kafka-init/Dockerfile | 3 ++- .../radar-cp-hadoop-stack/lib/perform-install.sh | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 6f0e1ab0a..afd7a2487 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -259,8 +259,11 @@ services: # Kafka Init # #---------------------------------------------------------------------------# kafka-init: - build: images/radar-kafka-init - image: radarbase/kafka-init:0.4.2 + build: + context: images/radar-kafka-init + args: + SCHEMAS_VERSION: ${RADAR_SCHEMAS_VERSION} + image: radarbase/kafka-init:${RADAR_SCHEMAS_VERSION} networks: - kafka - zookeeper @@ -783,8 +786,11 @@ services: # Catalog server from radar-schemas # #---------------------------------------------------------------------------# catalog-server: - build: images/radar-kafka-init - image: radarbase/kafka-init:0.4.2 + build: + context: images/radar-kafka-init + args: + SCHEMAS_VERSION: ${RADAR_SCHEMAS_VERSION} + image: radarbase/kafka-init:${RADAR_SCHEMAS_VERSION} networks: - management command: radar-schemas-tools serve /schema/merged diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 753f22668..7d3025c81 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -37,3 +37,4 @@ MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-priv ENABLE_OPTIONAL_SERVICES=false FITBIT_API_CLIENT_ID=fitbit-client FITBIT_API_CLIENT_SECRET=fitbit-secret +RADAR_SCHEMAS_VERSION=0.4.2 diff --git a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile index bfd3e590b..9e92feb03 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile +++ b/dcompose-stack/radar-cp-hadoop-stack/images/radar-kafka-init/Dockerfile @@ -25,8 +25,9 @@ RUN curl -#o /usr/share/java/avro-tools.jar \ "$(curl -s http://www.apache.org/dyn/closer.cgi/avro/\?as_json \ | jq --raw-output ".preferred")avro/avro-1.8.2/java/avro-tools-1.8.2.jar" +ARG SCHEMAS_VERSION=0.4.2 -ENV RADAR_SCHEMAS_VERSION=0.4.2 +ENV RADAR_SCHEMAS_VERSION=${SCHEMAS_VERSION} RUN curl -#L https://github.com/RADAR-base/RADAR-Schemas/releases/download/v${RADAR_SCHEMAS_VERSION}/radar-schemas-tools-${RADAR_SCHEMAS_VERSION}.tar.gz \ | tar xz -C /usr --strip-components 1 diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index a70b47dff..1866ddc7b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -95,7 +95,7 @@ ensure_variable 'mongo.database=' $HOTSTORAGE_NAME etc/mongodb-connector/sink-mo KAFKA_INIT_OPTS=( --rm -v "$PWD/etc/schema:/schema/conf" - radarbase/kafka-init:0.4.2 + radarbase/kafka-init:${RADAR_SCHEMAS_VERSION} ) # Set topics From 0ad15e286ad1a580c5764fcef4b373d7eeb28b3b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 28 Nov 2018 13:47:09 +0100 Subject: [PATCH 193/194] Update travis environment --- .../radar-cp-hadoop-stack/travis/travis-env.template | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 2807428a2..17679c8ee 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -23,17 +23,18 @@ HDFS_DATA_DIR_2=/home/travis/data/hdfs-data-2 HDFS_DATA_DIR_3=/home/travis/data/hdfs-data-3 HDFS_NAME_DIR_1=/home/travis/data/hdfs-name-1 HDFS_NAME_DIR_2=/home/travis/data/hdfs-name-2 -HDFS_JOURNAL_DIR_1=/home/travis/data/hdfs-journal-1 -HDFS_JOURNAL_DIR_2=/home/travis/data/hdfs-journal-2 -HDFS_JOURNAL_DIR_3=/home/travis/data/hdfs-journal-3 -POSTGRES_VERSION=10.4-alpine +POSTGRES_VERSION=10.6-alpine KAFKA_MANAGER_USERNAME=kafkamanager-user KAFKA_MANAGER_PASSWORD=kafkamanager-pwd PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 -MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD=notadmin MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned +RESTRUCTURE_OUTPUT_DIR=output +RESTRUCTURE_NUM_THREADS=3 +RESTRUCTURE_ENABLE_SNAPSHOTS=true +MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD=notadmin MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy ENABLE_OPTIONAL_SERVICES=true FITBIT_API_CLIENT_ID=fitbit-client FITBIT_API_CLIENT_SECRET=fitbit-secret +RADAR_SCHEMAS_VERSION=0.4.2 From 826a0d90e31c9592ad51e1634e72686729e10c78 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 20 Dec 2018 10:57:09 +0100 Subject: [PATCH 194/194] Only snapshot updated files and check zip integrity --- .../bin/hdfs-restructure | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure index 42aa4a3e6..f285f814c 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure @@ -101,16 +101,27 @@ function create_snapshots() { SNAPSHOT=$SNAPSHOT_DIR/$month.zip SNAPSHOT_TMP=$SNAPSHOT_TMP_DIR/$month.zip if [ -e $SNAPSHOT ]; then - find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -0@u $SNAPSHOT -O $SNAPSHOT_TMP - if [ -e $SNAPSHOT_TMP ]; then - mv $SNAPSHOT_TMP $SNAPSHOT + UPDATED_FILES_TMP=$SNAPSHOT_TMP.updated + find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" -newer $SNAPSHOT > $UPDATED_FILES_TMP + if [ ! -s $UPDATED_FILES_TMP ]; then + # No new files for this snapshot + continue + fi + if zip -T $SNAPSHOT >/dev/null; then + zip -0@u $SNAPSHOT -O $SNAPSHOT_TMP < $UPDATED_FILES_TMP + rm $UPDATED_FILES_TMP + if [ -e $SNAPSHOT_TMP ]; then + mv $SNAPSHOT_TMP $SNAPSHOT + fi + continue + else + echo "Snapshot for ${project}/${month}.zip was corrupted. Recreating." + rm $SNAPSHOT fi - else - find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -0@ $SNAPSHOT_TMP - mv $SNAPSHOT_TMP $SNAPSHOT fi + find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -0@ $SNAPSHOT_TMP + mv $SNAPSHOT_TMP $SNAPSHOT done - cd "$OLDPWD" } export -f create_snapshots