From bafb4e44fd3114f8f0deb85461b46e7b4573cfa0 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 14 Nov 2019 13:56:16 +0100 Subject: [PATCH 1/9] Use latest configuration settings --- .../bin/hdfs-restructure | 128 ------------------ .../radar-cp-hadoop-stack/docker-compose.yml | 18 +++ .../etc/restructure.yml.template | 92 +++++++++++++ .../lib/install-systemd-wrappers.sh | 7 - .../lib/perform-install.sh | 1 + .../lib/systemd/radar-output.service.template | 17 --- .../lib/systemd/radar-output.timer.template | 11 -- 7 files changed, 111 insertions(+), 163 deletions(-) delete mode 100755 dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure create mode 100644 dcompose-stack/radar-cp-hadoop-stack/etc/restructure.yml.template delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template delete mode 100644 dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure b/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure deleted file mode 100755 index 96522efa2..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/bin/hdfs-restructure +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash - -if [[ $# -lt 1 || $1 = "-h" || $1 = "--help" ]]; then - printf "Usage:\n$0 []\nThe destination directory defaults to ./output\n" - exit 1 -fi - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -# Absolute directory to write output to -OUTPUT_DIR=${2:-$DIR/output} -OUTPUT_DIR="$(cd "$(dirname "$OUTPUT_DIR")"; pwd)/$(basename "$OUTPUT_DIR")" - -cd $DIR - -. .env - -# HDFS restructure version -DOCKER_IMAGE=radarbase/radar-hdfs-restructure:0.5.7 - -NUM_THREADS=${RESTRUCTURE_NUM_THREADS:-3} -# HDFS restructure script flags -HDFS_RESTRUCTURE_OPTS=( - --compression gzip - --deduplicate - --num-threads $NUM_THREADS - ) -OUTPUT_USER=${OUTPUT_USER:-$(id -un)} -OUTPUT_GROUP=${OUTPUT_GROUP:-$(id -gn)} - -# HDFS restructure JVM flags -export RADAR_HDFS_RESTRUCTURE_OPTS="$JAVA_OPTS -Xmx4g" -# Without DOCKER_OPTS, run in interactive mode. -# From systemd or cron, override DOCKER_OPTS to remove -# interactive mode, e.g., -# DOCKER_OPTS="" bin/hdfs-restructure /mydir -DOCKER_OPTS=${DOCKER_OPTS--i} - -# For profiling, run e.g. jvisualvm and connect to localhost:$PROFILE_PORT -# after running: -# PROFILE_PORT=9101 bin/hdfs-restructure /mydir -# Note that profiling the application makes it slower. -if [ ! -z $PROFILE_PORT ]; then - export RADAR_HDFS_RESTRUCTURE_OPTS="$RADAR_HDFS_RESTRUCTURE_OPTS - -Djava.rmi.server.hostname=${PROFILE_HOST:-localhost} - -Dcom.sun.management.jmxremote - -Dcom.sun.management.jmxremote.port=${PROFILE_PORT} - -Dcom.sun.management.jmxremote.rmi.port=${PROFILE_PORT} - -Dcom.sun.management.jmxremote.local.only=false - -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false" - DOCKER_OPTS="$DOCKER_OPTS -p ${PROFILE_PORT}:${PROFILE_PORT}" -fi - -. lib/util.sh - -# Start HDFS if not started already -sudo-linux bin/radar-docker hdfs - -# HDFS filename to get -HDFS_FILE=$1 -# Internal docker directory to write output to -HDFS_OUTPUT_DIR=/output -# HDFS command to run -HDFS_COMMAND=( - "${HDFS_RESTRUCTURE_OPTS[@]}" - -p local-uid=$(id -u ${OUTPUT_USER}) - -p local-gid=$(grep "^${OUTPUT_GROUP}:" /etc/group | cut -d: -f3) - -n hdfs-namenode-1 - -o "$HDFS_OUTPUT_DIR" - --tmp-dir "$HDFS_OUTPUT_DIR/+tmp" - "$HDFS_FILE" - ) - -sudo-linux docker run ${DOCKER_OPTS} \ - -t --rm --network hadoop \ - -v "$OUTPUT_DIR:$HDFS_OUTPUT_DIR" \ - -e RADAR_HDFS_RESTRUCTURE_OPTS \ - $DOCKER_IMAGE "${HDFS_COMMAND[@]}" - -if ! ${RESTRUCTURE_ENABLE_SNAPSHOTS:-true}; then - echo "Snapshots disabled. No snapshots will be made." - exit 0 -fi - -if ! command -v zip > /dev/null 2>&1; then - echo "WARN: zip not installed. No snapshots will be made." - exit 0 -fi - -function create_snapshots() { - PROJECT_DIR="$1" - project=$(basename "${PROJECT_DIR}") - cd "$PROJECT_DIR" - SNAPSHOT_DIR=../snapshots/$project - SNAPSHOT_TMP_DIR=../snapshots/+tmp - mkdir -p $SNAPSHOT_DIR $SNAPSHOT_TMP_DIR - - months=$(find . -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9].*" -printf "%f\n" | cut -c-6 | sort -u) - for month in $months; do - SNAPSHOT=$SNAPSHOT_DIR/$month.zip - SNAPSHOT_TMP=$SNAPSHOT_TMP_DIR/$month.zip - if [ -e $SNAPSHOT ]; then - UPDATED_FILES_TMP=$SNAPSHOT_TMP.updated - find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" -newer $SNAPSHOT > $UPDATED_FILES_TMP - if [ ! -s $UPDATED_FILES_TMP ]; then - # No new files for this snapshot - continue - fi - if zip -T $SNAPSHOT >/dev/null; then - zip -0@u $SNAPSHOT -O $SNAPSHOT_TMP < $UPDATED_FILES_TMP - rm $UPDATED_FILES_TMP - if [ -e $SNAPSHOT_TMP ]; then - mv $SNAPSHOT_TMP $SNAPSHOT - fi - continue - else - echo "Snapshot for ${project}/${month}.zip was corrupted. Recreating." - rm $SNAPSHOT - fi - fi - find . -name "$month[0-9][0-9]_[0-9][0-9][0-9][0-9].*" | zip -0@ $SNAPSHOT_TMP - mv $SNAPSHOT_TMP $SNAPSHOT - done -} - -export -f create_snapshots -find "$OUTPUT_DIR" -mindepth 1 -maxdepth 1 -type d ! -name snapshots ! -name +tmp -print0 | xargs -0 -n 1 -P $NUM_THREADS -I {} bash -c 'create_snapshots "$@"' _ {} diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index d58f5b26f..8e4fd4cb2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -808,3 +808,21 @@ services: interval: 1m30s timeout: 5s retries: 3 + + radar-output: + image: radarbase/radar-hdfs-restructure:0.6.0 + restart: always + stop_signal: SIGINT + networks: + - hadoop + depends_on: + - hdfs-namenode-1 + - hdfs-datanode-1 + - hdfs-datanode-2 + - hdfs-datanode-3 + volumes: + - ${RESTRUCTURE_OUTPUT_DIR}:/output + - ./etc/restructure.yml:/etc/restructure.yml + environment: + RADAR_HDFS_RESTRUCTURE_OPTS: -Xms250m -Xmx4g + command: -F /etc/restructure.yml -S diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/restructure.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/restructure.yml.template new file mode 100644 index 000000000..6bd20a35d --- /dev/null +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/restructure.yml.template @@ -0,0 +1,92 @@ +service: + # Whether to run the application as a polling service. + enable: true + # Polling interval in seconds. + interval: 300 + +# Compression characteristics +compression: + # Compression type: none, zip or gzip + type: gzip + # Compression Factory class + # factory: org.radarbase.hdfs.data.CompressionFactory + # Additional compression properties + # properties: {} + +# File format +format: + # Format type: CSV or JSON + type: csv + # Whether to deduplicate the files in each topic by default + deduplication: + enable: true + # Deduplicate considering only distinct fields sourceId and time. + # This may incur data loss if multiple measurements are recorded + # at exactly the same time. By default, all values are considered + # when looking at distinct lines. + # distinctFields: [key.sourceId, value.time] + # Format factory class + # factory: org.radarbase.hdfs.data.FormatFactory + # Additional format properties + # properties: {} + +# Worker settings +worker: + # Maximum number of files and converters to keep open while processing + cacheSize: 300 + # Number of threads to do processing with + numThreads: 2 + # Maximum number of files to process in any given topic. + # maxFilesPerTopic: null + +storage: + factory: org.radarbase.hdfs.storage.LocalStorageDriver + properties: + localUid: 0 + localGid: 0 + +# Path settings +paths: + # Input directories in HDFS + inputs: + - /topicAndroidNew + # Root temporary directory for local file processing. + temp: /output/+tmp + # Output directory + output: /output + # Output path construction factory + #factory: org.radarbase.hdfs.MonthlyObservationKeyPathFactory + # Additional properties + # properties: {} + +# Individual topic configuration +topics: {} +# # topic name +# connect_fitbit_source: +# # deduplicate this topic, regardless of the format settings +# deduplication: +# # deduplicate this topic only using given fields. +# distinctFields: [value.time] +# connect_fitbit_bad: +# # Do not process this topic +# exclude: true +# biovotion_acceleration: +# # Disable deduplication +# deduplication: +# enable: false + +# HDFS settings +hdfs: + # HDFS name node in case of a single name node, or HDFS cluster ID in case of high availability. + name: hdfs-namenode-1 + # High availability settings: + # nameNodes: + # - name: hdfs1 + # hostname: hdfs-namenode-1 + # - name: hdfs2 + # hostname: hdfs-namenode-2 + # Where files will be locked. This value should be the same for all restructure processes. + lockPath: /logs/org.radarbase.hdfs/lock + # Additional raw HDFS configuration properties + # properties: {} + diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh index a7c8631d9..8b4b387cf 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh @@ -16,8 +16,6 @@ fi echo "==> Copying templates" copy_template $BASE/radar-docker.service lib/systemd/radar-docker.service.template -copy_template $BASE/radar-output.service lib/systemd/radar-output.service.template -copy_template $BASE/radar-output.timer lib/systemd/radar-output.timer.template copy_template $BASE/radar-check-health.service lib/systemd/radar-check-health.service.template copy_template $BASE/radar-check-health.timer lib/systemd/radar-check-health.timer.template copy_template $BASE/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template @@ -27,9 +25,6 @@ echo "==> Inlining variables" inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-docker.service inline_variable 'ExecStart=' "$PWD/bin/radar-docker foreground" $BASE/radar-docker.service -inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-output.service -inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure /topicAndroidNew ${RESTRUCTURE_OUTPUT_DIR:-output}" $BASE/radar-output.service - inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-check-health.service inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" $BASE/radar-check-health.service @@ -39,10 +34,8 @@ inline_variable 'ExecStart=' "$PWD/bin/radar-docker cert-renew" $BASE/radar-rene echo "==> Reloading systemd" systemctl "${SYSTEMCTL_OPTS[@]}" daemon-reload systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-docker -systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-output.timer systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-check-health.timer systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-renew-certificate.timer systemctl "${SYSTEMCTL_OPTS[@]}" start radar-docker -systemctl "${SYSTEMCTL_OPTS[@]}" start radar-output.timer systemctl "${SYSTEMCTL_OPTS[@]}" start radar-check-health.timer systemctl "${SYSTEMCTL_OPTS[@]}" start radar-renew-certificate.timer diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index 23db28938..c1c7510ac 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -20,6 +20,7 @@ copy_template_if_absent etc/webserver/ip-access-control.conf copy_template_if_absent etc/webserver/optional-services.conf copy_template_if_absent etc/fitbit/docker/source-fitbit.properties copy_template_if_absent etc/rest-source-authorizer/rest_source_clients_configs.yml +copy_template_if_absent etc/restructure.yml # Set permissions sudo-linux chmod og-rw ./.env diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template deleted file mode 100644 index aa3447356..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.service.template +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=RADAR-Docker service -After=radar-docker.service - -[Service] - -WorkingDirectory= - -ExecStart= - -NotifyAccess=all - -Environment=DOCKER_OPTS= - -StandardOutput=syslog -StandardError=syslog -SyslogIdentifier=radar-output \ No newline at end of file diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template b/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template deleted file mode 100644 index abc8edb30..000000000 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/systemd/radar-output.timer.template +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=RADAR-Docker hourly data restructuring -BindsTo=radar-docker.service - -[Timer] -OnBootSec=15min -OnUnitActiveSec=1h -Unit=radar-output.service - -[Install] -WantedBy=default.target From 489fd55ef74f4d141c2431bc33f4cfaf2435c238 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 14 Nov 2019 13:57:10 +0100 Subject: [PATCH 2/9] Updated env variables --- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 3 --- 1 file changed, 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index 99ab1b885..a30113761 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -30,9 +30,6 @@ KAFKA_MANAGER_PASSWORD= PORTAINER_PASSWORD_HASH= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned -RESTRUCTURE_OUTPUT_DIR=output -RESTRUCTURE_NUM_THREADS=3 -RESTRUCTURE_ENABLE_SNAPSHOTS=true MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD= MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy ENABLE_OPTIONAL_SERVICES=false From d0e2059f6e72f665f135f9b2d2c3877bbcdf866b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 14 Nov 2019 14:09:22 +0100 Subject: [PATCH 3/9] PR fixes --- dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml | 2 +- dcompose-stack/radar-cp-hadoop-stack/etc/env.template | 1 + .../etc/{ => hdfs-restructure}/restructure.yml.template | 0 dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) rename dcompose-stack/radar-cp-hadoop-stack/etc/{ => hdfs-restructure}/restructure.yml.template (100%) diff --git a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml index 8e4fd4cb2..2576f7af2 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml +++ b/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml @@ -822,7 +822,7 @@ services: - hdfs-datanode-3 volumes: - ${RESTRUCTURE_OUTPUT_DIR}:/output - - ./etc/restructure.yml:/etc/restructure.yml + - ./etc/hdfs-restructure/restructure.yml:/etc/restructure.yml environment: RADAR_HDFS_RESTRUCTURE_OPTS: -Xms250m -Xmx4g command: -F /etc/restructure.yml -S diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template index a30113761..31fa395b9 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/env.template @@ -30,6 +30,7 @@ KAFKA_MANAGER_PASSWORD= PORTAINER_PASSWORD_HASH= MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned +RESTRUCTURE_OUTPUT_DIR=./output MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD= MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy ENABLE_OPTIONAL_SERVICES=false diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/restructure.yml.template b/dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-restructure/restructure.yml.template similarity index 100% rename from dcompose-stack/radar-cp-hadoop-stack/etc/restructure.yml.template rename to dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-restructure/restructure.yml.template diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh index c1c7510ac..fbad888ae 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh @@ -20,7 +20,7 @@ copy_template_if_absent etc/webserver/ip-access-control.conf copy_template_if_absent etc/webserver/optional-services.conf copy_template_if_absent etc/fitbit/docker/source-fitbit.properties copy_template_if_absent etc/rest-source-authorizer/rest_source_clients_configs.yml -copy_template_if_absent etc/restructure.yml +copy_template_if_absent etc/hdfs-restructure/restructure.yml # Set permissions sudo-linux chmod og-rw ./.env From 225df97c9ee73170b331562fafc8d66a332396e3 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 14 Nov 2019 14:15:57 +0100 Subject: [PATCH 4/9] Fix travis build --- dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh | 1 + .../radar-cp-hadoop-stack/travis/travis-env.template | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh index c9c238336..72887f69b 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/setup-env.sh @@ -13,6 +13,7 @@ cp ../etc/hdfs-connector/sink-hdfs.properties.template ../etc/hdfs-connector/sin cp ../etc/mongodb-connector/sink-mongo.properties.template ../etc/mongodb-connector/sink-mongo.properties cp ../etc/managementportal/config/oauth_client_details.csv.template ../etc/managementportal/config/oauth_client_details.csv cp ../etc/redcap-integration/radar.yml.template ../etc/redcap-integration/radar.yml +cp ../etc/hdfs-restructure/restructure.yml.template ../etc/hdfs-restructure/restructure.yml ../bin/keystore-init diff --git a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template index 17679c8ee..2f3e7336f 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template +++ b/dcompose-stack/radar-cp-hadoop-stack/travis/travis-env.template @@ -29,9 +29,7 @@ KAFKA_MANAGER_PASSWORD=kafkamanager-pwd PORTAINER_PASSWORD_HASH=$2y$05$POrvWxSUsnkeGNZ7LzHUceL6rQM3U79lYofaM2NtyQJ8iWJ4ve542 MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned -RESTRUCTURE_OUTPUT_DIR=output -RESTRUCTURE_NUM_THREADS=3 -RESTRUCTURE_ENABLE_SNAPSHOTS=true +RESTRUCTURE_OUTPUT_DIR=./output MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD=notadmin MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy ENABLE_OPTIONAL_SERVICES=true From dbc3b023f88d676249f1327399fe37653044578a Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Fri, 10 Jan 2020 14:07:26 +0000 Subject: [PATCH 5/9] Add cors support to schema registry --- .../etc/webserver/nginx.conf.template | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template index 43bf46f39..586d7a22a 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template @@ -77,7 +77,26 @@ http { deny all; } location /schema/ { - limit_except GET { + if ($request_method = 'OPTIONS') { + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + } + + limit_except GET OPTIONS { deny all; } proxy_pass http://schema-registry-1:8081/; From 2eb683079b2b2839ba67fc870a93beed5f7bac4d Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Fri, 10 Jan 2020 14:09:02 +0000 Subject: [PATCH 6/9] Add cors support in no-ssl mode --- .../etc/webserver/nginx.nossl.conf.template | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template index dbb3cdbd7..a1d607054 100644 --- a/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template +++ b/dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template @@ -48,7 +48,26 @@ http { deny all; } location /schema/ { - limit_except GET { + if ($request_method = 'OPTIONS') { + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always; + } + + limit_except GET OPTIONS { deny all; } proxy_pass http://schema-registry-1:8081/; From 2517dedb9c336f73d49c46a6f7fa92ba9fab19d0 Mon Sep 17 00:00:00 2001 From: nivethika Date: Mon, 3 Feb 2020 16:55:33 +0100 Subject: [PATCH 7/9] update release --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8dc910b4d..b1dc3e570 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RADAR-Docker 2.0.1 +# RADAR-Docker 2.2.0 The dockerized RADAR stack for deploying the RADAR-base platform. Component repositories can be found at [RADAR-base DockerHub org](https://hub.docker.com/u/radarbase/dashboard/) From e72e03fe2a5dca214e28fcb20c54d9464ba725f7 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Tue, 4 Feb 2020 18:13:46 +0000 Subject: [PATCH 8/9] Use ACME protocol v2 for lets encrypt certificate Version 1 is deprecated and will stop working after 1 June, 2020 --- dcompose-stack/radar-cp-hadoop-stack/lib/util.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh index 4451d77c7..8d99694c3 100755 --- a/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh +++ b/dcompose-stack/radar-cp-hadoop-stack/lib/util.sh @@ -143,7 +143,7 @@ letsencrypt_certonly() { sudo-linux docker run --rm -v certs:/etc/openssl alpine:3.7 /bin/sh -c "find /etc/openssl -name '${SERVER_NAME}*' -prune -exec rm -rf '{}' +" CERTBOT_DOCKER_OPTS=(--rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) - CERTBOT_OPTS=(--webroot --webroot-path=/data/letsencrypt --agree-tos -m "${MAINTAINER_EMAIL}" -d "${SERVER_NAME}" --non-interactive) + CERTBOT_OPTS=(--server "https://acme-v02.api.letsencrypt.org/directory" --webroot --webroot-path=/data/letsencrypt --agree-tos -m "${MAINTAINER_EMAIL}" -d "${SERVER_NAME}" --non-interactive) sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" # mark the directory as letsencrypt dir @@ -154,7 +154,7 @@ letsencrypt_renew() { SERVER_NAME=$1 echo "==> Renewing Let's Encrypt SSL certificate for ${SERVER_NAME}" CERTBOT_DOCKER_OPTS=(--rm -v certs:/etc/letsencrypt -v certs-data:/data/letsencrypt deliverous/certbot) - CERTBOT_OPTS=(-n --webroot --webroot-path=/data/letsencrypt -d "${SERVER_NAME}" --non-interactive) + CERTBOT_OPTS=(-n --server "https://acme-v02.api.letsencrypt.org/directory" --webroot --webroot-path=/data/letsencrypt -d "${SERVER_NAME}" --non-interactive) sudo-linux docker run "${CERTBOT_DOCKER_OPTS[@]}" certonly "${CERTBOT_OPTS[@]}" } From f3c9fc46533ba4ee7fa3ae046566a65ee5b8fb2e Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Tue, 4 Feb 2020 18:42:42 +0000 Subject: [PATCH 9/9] Fix central maven repo in hsqldb docker image --- images/hsqldb/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/hsqldb/Dockerfile b/images/hsqldb/Dockerfile index 557cf7a12..ff6cf9db1 100644 --- a/images/hsqldb/Dockerfile +++ b/images/hsqldb/Dockerfile @@ -8,7 +8,7 @@ RUN mkdir -p /opt/hsqldb/lib /etc/opt/hsqldb/conf /var/opt/hsqldb/data && \ useradd --system -g hsqldb -u 9999 hsqldb && \ chown hsqldb:hsqldb -R /var/opt/hsqldb -ENV MVN_CENTRAL_URL http://central.maven.org/maven2 +ENV MVN_CENTRAL_URL https://repo1.maven.org/maven2/ ENV HSQLDB_MVN_GRP org/hsqldb ENV HSQLDB_VERSION 2.5.0 ENV LOG4J_VERSION 1.2.17