diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41f547aa88..f66ec47c9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,6 @@ jobs: - name: pull run: make dev.pull.${{matrix.services}} - - name: set exit option - run: set -e - - name: provision run: make dev.provision.${{matrix.services}} diff --git a/check.sh b/check.sh index 9c69dea01c..d6caa24331 100755 --- a/check.sh +++ b/check.sh @@ -16,9 +16,7 @@ # Note that passing in a non-existent service will not fail if there are # other successful checks. -set -e -set -o pipefail -set -u +set -eu -o pipefail # Grab all arguments into one string, replacing plus signs with spaces. # Pad on either side with spaces so that the regex in `should_check` works correctly. diff --git a/destroy.sh b/destroy.sh index da47f02cb6..1c92e23784 100755 --- a/destroy.sh +++ b/destroy.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash - -set -e +set -eu -o pipefail read -p "This will delete all data in your devstack. Would you like to proceed? [y/n] " -r if [[ $REPLY =~ ^[Yy]$ ]] diff --git a/enterprise/provision.sh b/enterprise/provision.sh index 559e02d671..8d77ce2229 100755 --- a/enterprise/provision.sh +++ b/enterprise/provision.sh @@ -1,2 +1,6 @@ -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' -cat enterprise/worker_permissions.py | docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +#!/usr/bin/env bash +set -eu -o pipefail +set -x + +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' +cat enterprise/worker_permissions.py | docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' diff --git a/load-db.sh b/load-db.sh index 0fd4124ade..99be594989 100755 --- a/load-db.sh +++ b/load-db.sh @@ -7,8 +7,7 @@ # # This will load the edxapp database from a file named exapp.sql. -set -e -set -o pipefail +set -eu -o pipefail if [ -z "$1" ] then diff --git a/programs/provision.sh b/programs/provision.sh index 967ae39774..32697100d7 100755 --- a/programs/provision.sh +++ b/programs/provision.sh @@ -1,6 +1,7 @@ -#!/bin/sh +#!/usr/bin/env bash +set -eu -o pipefail +set -x -set -e # # To add programs support, we need to tweak/add certain rows in the database. # We want to go through Django for this (rather than direct db modification), since we have a lot of Python @@ -44,19 +45,20 @@ docker_exec() { /edx/app/$app/$repo/manage.py $cmd " - docker-compose exec -T "$service" bash -c "$CMDS" + docker-compose exec -T "$service" bash -e -c "$CMDS" } provision_ida() { - service=${1} - cmd=${2:-"shell"} + service=$1 + cmd=$2 + shift 2 # Escape double quotes and backticks from the Python PROGRAM_SCRIPT="$(sed -E 's/("|`)/\\\1/g' < "$BASEDIR/$service.py")" cmd="$cmd -c \"$PROGRAM_SCRIPT\"" - docker_exec "$service" "$cmd" "$3" "$4" + docker_exec "$service" "$cmd" "$@" } trap reset_color 1 2 3 6 15 @@ -68,7 +70,10 @@ fi if [ "$1" = "discovery" -o -z "$1" ]; then notice Adding demo program to Discovery... - provision_ida discovery + set +e + # FIXME[bash-e]: Bash scripts should use -e -- but this command fails + provision_ida discovery "shell" + set -e fi if [ "$1" = "cache" -o -z "$1" ]; then diff --git a/provision-credentials.sh b/provision-credentials.sh index 34c7a6b2dd..ef03037b20 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +set -eu -o pipefail + +. scripts/colors.sh +set -x # NOTE (CCB): We do NOT call provision-ida because it expects a virtualenv. # The new images for Credentials do not use virtualenv. @@ -9,20 +13,28 @@ port=18150 docker-compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements' -- "$name" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +set +e +# FIXME[bash-e]: Bash scripts should use -e -- but this script fails +# with missing manage.py (need another "credentials" in that cd path?) echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +set -e +set +e +# FIXME[bash-e]: Bash scripts should use -e -- but this script fails +# with missing manage.py (need another "credentials" in that cd path?) echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker-compose exec -T ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +set -e ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec -T ${name} bash -e -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-discovery.sh b/provision-discovery.sh index 251bb5166d..676e78de22 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -1,10 +1,20 @@ +#!/usr/bin/env bash # Provisioning script for the discovery service +set -eu -o pipefail +set -x + ./provision-ida.sh discovery discovery 18381 -docker-compose exec -T discovery bash -c 'rm -rf /edx/var/discovery/*' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --lms-coursemode-api-url "http://edx.devstack.lms:18000/api/course_modes/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/" --studio-url "http://edx.devstack.studio:18010/" --publisher-url "http://edx.devstack.frontend-app-publisher:18400/"' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' +docker-compose exec -T discovery bash -e -c 'rm -rf /edx/var/discovery/*' +docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --lms-coursemode-api-url "http://edx.devstack.lms:18000/api/course_modes/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/" --studio-url "http://edx.devstack.studio:18010/" --publisher-url "http://edx.devstack.frontend-app-publisher:18400/"' + +set +e +# FIXME[bash-e]: Bash scripts should use -e -- but this script fails +# (after many retries) when trying to talk to ecommerce +docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' +set -e + +docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' # Add demo program ./programs/provision.sh discovery diff --git a/provision-e2e.sh b/provision-e2e.sh index 0949174904..078a6a26da 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -1,5 +1,6 @@ -set -e -set -o pipefail +#!/usr/bin/env bash + +set -eu -o pipefail set -x if [ -z "$DEVSTACK_WORKSPACE" ]; then @@ -13,11 +14,11 @@ fi docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent --no-print-directory dev.print-container.studio)":/tmp/ # Extract the test course tarball -docker-compose exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' +docker-compose exec -T studio bash -e -c 'cd /tmp && tar xzf course.tar.gz' # Import the course content -docker-compose exec -T studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' +docker-compose exec -T studio bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' # Clean up the temp files -docker-compose exec -T studio bash -c 'rm /tmp/course.tar.gz' -docker-compose exec -T studio bash -c 'rm -r /tmp/course' +docker-compose exec -T studio bash -e -c 'rm /tmp/course.tar.gz' +docker-compose exec -T studio bash -e -c 'rm -r /tmp/course' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index a94b54ea9d..85cda52306 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -1,9 +1,13 @@ +#!/usr/bin/env bash +set -eu -o pipefail +set -x + # Load database dumps for the largest databases to save time ./load-db.sh ecommerce ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose exec -T ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' -docker-compose exec -T ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker-compose exec -T ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' +docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index afc51e2827..d943e5e1f7 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -1,6 +1,6 @@ -set -e -set -o pipefail +#!/usr/bin/env bash +set -eu -o pipefail set -x docker-compose up -d forum -docker-compose exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' +docker-compose exec -T forum bash -e -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index 14e245ba06..658191d3a5 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -1,5 +1,17 @@ +#!/usr/bin/env bash +set -eu -o pipefail +set -x + +# FIXME[bash-e]: Bash scripts should use -e -- but this script fails +# with the following errors for ecommerce & edx_notes_api: +# - RuntimeError: Model class openedx.core.djangoapps.content_libraries.models.ContentLibrary doesn't declare an explicit app_label and isn't in an application in INSTALLED_APPS. +# - django.db.utils.ProgrammingError: (1146, "Table 'edxapp.auth_user' doesn't exist") +set +e + #This script depends on the LMS being up! +. scripts/colors.sh + app_name=$1 client_name=$2 client_port=$3 @@ -7,8 +19,15 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" + + +# FIXME[bash-e]: Suppress last error so that calling script can set -e +# for itself. Remove this once *this* script is run entirely with `-e` +# in effect (or at least the last command is no longer erroring for +# any services). +true diff --git a/provision-ida.sh b/provision-ida.sh index 07850b780a..4fb0db27be 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -1,4 +1,9 @@ -#!/bin/sh -x +#!/usr/bin/env bash +set -eu -o pipefail + +. scripts/colors.sh +set -x + app_name=$1 # The name of the IDA application, i.e. /edx/app/ client_name=$2 # The name of the Oauth client stored in the edxapp DB. client_port=$3 # The port corresponding to this IDA service in devstack. @@ -7,17 +12,17 @@ container_name=${4:-$1} # (Optional) The name of the container. If missing, wil docker-compose up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-lms.sh b/provision-lms.sh index 6ec0221867..3494ff6b91 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -1,5 +1,5 @@ -set -e -set -o pipefail +#!/usr/bin/env bash +set -eu -o pipefail set -x apps=( lms studio ) @@ -13,33 +13,33 @@ for app in "${apps[@]}"; do docker-compose up -d $app done -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' # Create a superuser for edxapp -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' # Create an enterprise service user for edxapp and give them appropriate permissions ./enterprise/provision.sh # Enable the LMS-E-Commerce integration -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -docker-compose exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +docker-compose exec -T lms bash -e -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' # Fix missing vendor file by clearing the cache -docker-compose exec -T lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' +docker-compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' # Create static assets for both LMS and Studio for app in "${apps[@]}"; do - docker-compose exec -T $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' + docker-compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' done # Provision a retirement service account user diff --git a/provision-notes.sh b/provision-notes.sh index ae7cac58c9..9202cb50d4 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -1,8 +1,13 @@ +#!/usr/bin/env bash # Provisioning script for the notes service +set -eu -o pipefail + +. scripts/colors.sh +set -x # Common provisioning tasks for IDAs, including requirements, migrations, oauth client creation, etc. ./provision-ida.sh edx_notes_api edx-notes 18120 edx_notes_api # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker-compose exec -T edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py search_index --rebuild -f' -- edx_notes_api +docker-compose exec -T edx_notes_api bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py search_index --rebuild -f' -- edx_notes_api diff --git a/provision-registrar.sh b/provision-registrar.sh index 7f3e0441c3..31bfe68a3d 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +set -eu -o pipefail + +. scripts/colors.sh +set -x name=registrar port=18734 @@ -6,17 +10,17 @@ port=18734 docker-compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec -T ${name} bash -e -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index d76ed19d1e..2df6d814ee 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -1,8 +1,13 @@ +#!/usr/bin/env bash #This script depends on the LMS being up! +set -eu -o pipefail + +. scripts/colors.sh +set -x app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision-xqueue.sh b/provision-xqueue.sh index 3cc9c9b4f3..8fde5dafce 100755 --- a/provision-xqueue.sh +++ b/provision-xqueue.sh @@ -1,13 +1,13 @@ -set -e -set -o pipefail +#!/usr/bin/env bash +set -eu -o pipefail set -x # Bring up XQueue, we don't need the consumer for provisioning docker-compose up -d xqueue # Update dependencies -docker-compose exec -T xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' +docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' # Run migrations -docker-compose exec -T xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' +docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' # Add users that graders use to fetch data, there's one default user in Ansible which is part of our settings -docker-compose exec -T xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' +docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' diff --git a/provision.sh b/provision.sh index 69f86e7779..9927b971c8 100755 --- a/provision.sh +++ b/provision.sh @@ -26,15 +26,10 @@ # ./provision.sh lms ecommerce discovery # Provision these three services. # ./provision.sh lms+ecommerce+discovery # Same as previous command. -set -e -set -o pipefail -set -u +set -eu -o pipefail set -x -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -NC='\033[0m' # No Color +. scripts/colors.sh # All provisionable services. # (Leading and trailing space are necessary for if-checks.) @@ -133,7 +128,7 @@ fi # Temporary until MySQL 5.6 is removed echo "${GREEN}Waiting for MySQL 5.6.${NC}" -until docker-compose exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker-compose exec -T mysql bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -141,7 +136,7 @@ done # Ensure the MySQL server is online and usable echo "${GREEN}Waiting for MySQL 5.7.${NC}" -until docker-compose exec -T mysql57 bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -154,12 +149,12 @@ echo -e "${GREEN}MySQL ready.${NC}" # Temporary until MySQL 5.6 is removed echo -e "${GREEN}Ensuring MySQL 5.6 databases and users exist...${NC}" -docker-compose exec -T mysql bash -c "mysql -uroot mysql" < provision.sql +docker-compose exec -T mysql bash -e -c "mysql -uroot mysql" < provision.sql # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL 5.7 databases and users exist...${NC}" -docker-compose exec -T mysql57 bash -c "mysql -uroot mysql" < provision.sql +docker-compose exec -T mysql57 bash -e -c "mysql -uroot mysql" < provision.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. @@ -173,7 +168,7 @@ if needs_mongo "$to_provision_ordered"; then done echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker-compose exec -T mongo bash -c "mongo" < mongo-provision.js + docker-compose exec -T mongo bash -e -c "mongo" < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/scripts/colors.sh b/scripts/colors.sh new file mode 100644 index 0000000000..1f1f24e757 --- /dev/null +++ b/scripts/colors.sh @@ -0,0 +1,6 @@ +# Source this file to get color variables + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' # No Color diff --git a/setup_native_nfs_docker_osx.sh b/setup_native_nfs_docker_osx.sh index 68e394b31d..645eb254f5 100755 --- a/setup_native_nfs_docker_osx.sh +++ b/setup_native_nfs_docker_osx.sh @@ -2,6 +2,7 @@ # # Coppied from https://github.com/ajeetraina/docker101/tree/master/os/macOS/nfs # +set -eu -o pipefail OS=`uname -s` MAC_VERSION=`sw_vers -productVersion | awk -F. '{ printf "%s.%s", $1, $2; }'`; diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 501b1074ac..91c7de3d84 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -3,12 +3,8 @@ # Make sure you have added your user to the docker group before running this script # or use sudo to run it - -set -x # echo on -set -e # exit when any command fails -set -u # exit when undefined variables are found -set -o pipefail # prevents errors in a pipeline from being masked - +set -eu -o pipefail +set -x # constants readonly EDXAPP_MYSQL_DB_USER="edxapp001" diff --git a/upgrade_mongo_4_0.sh b/upgrade_mongo_4_0.sh index 9af5a02482..0409d03698 100755 --- a/upgrade_mongo_4_0.sh +++ b/upgrade_mongo_4_0.sh @@ -1,11 +1,9 @@ #!/usr/bin/env bash +set -eu -o pipefail # This script will upgrade a devstack that was previosly running Mongo DB 3.2, 3.4 or 3.6 to MongoDB 4.0 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -NC='\033[0m' # No Color +. scripts/colors.sh export MONGO_VERSION=3.4.24 current_mongo_version="3.4"