diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aefe7c..7b9c32a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: branches: [master] jobs: - test: - name: Test docker compose + test-prod: + name: '[Prod] Test docker compose' runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -26,4 +26,18 @@ jobs: docker compose -f docker-compose.local.yml build - name: Test if Indico works - run: ./test.sh + run: ./test.sh prod + + test-dev: + name: '[Dev] Test docker compose' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Build containers + run: | + cd indico-dev + docker compose build + + - name: Test if Indico works + run: ./test.sh dev diff --git a/.gitignore b/.gitignore index f3a6d29..62772ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ data/* indico-prod/data/* +indico-dev/data/* # IDE files .idea/ diff --git a/README.md b/README.md index 58f68c7..dbaf954 100644 --- a/README.md +++ b/README.md @@ -70,3 +70,23 @@ $ docker run \ ``` In the above, we omit the network setup to make e.g. the DB accessible from the containers. + +## Development setup + +The development setup builds an image from the latest master. Essentially, it puts this [tutorial](https://docs.getindico.io/en/stable/installation/development/) into a Dockerfile. The main purpose of this setup is to allow anyone to relatively quickly and painlessly try out the latest translations from Transifex without having to manually setup an Indico instance. + +### Quickstart + +To start the containers, run: +```sh +$ cd indico-dev && docker compose up +``` +Indico should now be accessible at [localhost:8080](localhost:8080). + +The Indico container by default pulls the latest translations from a public mirror which does not need authentication. The mirror updates its translations once every hour. If you want to pull directly from Transifex, you can provide your Transifex API token in the [dev.env](dev.env) file (`TRANSIFEX_API_TOKEN=1/xxx`). + +The development setup creates a default admin user with username `admin` and password `indiko42`. + +Since emails also get translated, you can use maildump to check those as well. Maildump is available at [localhost:60000](localhost:60000). + +Check the [production setup](#production-like-setup) to see more configuration options. diff --git a/indico-dev/.env b/indico-dev/.env new file mode 100644 index 0000000..f0cda6a --- /dev/null +++ b/indico-dev/.env @@ -0,0 +1,26 @@ +## DB config +## =========================================== +PGHOST=indico-postgres +PGUSER=indico +PGPASSWORD=indicopass +PGDATABASE=indico + +## Celery +## =========================================== +C_FORCE_ROOT=true + +## Nginx +## =========================================== +NGINX_PORT=8080 +# Should match the 'BASE_URL' in indico.conf +NGINX_SERVER_NAME=localhost:8080 + +## Indico +## =========================================== +# Path to the indico.conf file +INDICO_CONFIG=indico.conf + +## Transifex +## =========================================== +# If not specified, the translations are pulled from a public mirror +TRANSIFEX_API_TOKEN= diff --git a/indico-dev/docker-compose.yml b/indico-dev/docker-compose.yml new file mode 100644 index 0000000..623ee7f --- /dev/null +++ b/indico-dev/docker-compose.yml @@ -0,0 +1,83 @@ +version: "3" +services: + # The main Indico container which runs flask + # The same image is also used to run celery + indico-web: &indico-web + build: worker + command: /opt/indico/run_indico.sh + depends_on: + - indico-redis + - indico-celery + environment: + - PGHOST=${PGHOST} + - PGUSER=${PGUSER} + - PGPASSWORD=${PGPASSWORD} + - PGDATABASE=${PGDATABASE} + - C_FORCE_ROOT=${C_FORCE_ROOT} + - TRANSIFEX_API_TOKEN=${TRANSIFEX_API_TOKEN} + networks: + - backend + - frontend + ports: + # Indico is accessible either via nginx (localhost:8080 by default), or + # directly via localhost:9090. In that case, static assets are served by flask + - "9090:59999" + # Maildump is accessible at localhost:60000 + - "60000:60000" + volumes: + - 'archive:/opt/indico/archive' # file storage + - 'customization:/opt/indico/custom' + - 'static-files:/opt/indico/static' + - './data/indico/log:/opt/indico/log' # logs + - type: bind + source: ${INDICO_CONFIG} + target: /opt/indico/etc/indico.conf + read_only: true + tmpfs: + - /opt/indico/tmp + # Indico celery + indico-celery: + <<: *indico-web + command: /opt/indico/run_celery.sh + depends_on: + - indico-redis + networks: + - backend + ports: [] + # Redis + indico-redis: + image: redis + networks: + - backend + volumes: + - './data/redis:/data' + # Postgres + indico-postgres: + image: centos/postgresql-13-centos7 + environment: + - POSTGRESQL_USER=${PGUSER} + - POSTGRESQL_PASSWORD=${PGPASSWORD} + - POSTGRESQL_DATABASE=${PGDATABASE} + - POSTGRESQL_ADMIN_PASSWORD=${PGPASSWORD} + networks: + - backend + # Nginx proxy + # Indico can be accessed via localhost:8080 + indico-nginx: + build: nginx + environment: + - NGINX_SERVER_NAME=${NGINX_SERVER_NAME} + networks: + - frontend + ports: + - "${NGINX_PORT:-8080}:8080" + volumes: + - 'static-files:/opt/indico/static:ro' + - './data/nginx/log:/var/log/nginx' # logs +volumes: + archive: + static-files: + customization: +networks: + backend: {} + frontend: {} diff --git a/indico-dev/indico.conf b/indico-dev/indico.conf new file mode 100644 index 0000000..42abd68 --- /dev/null +++ b/indico-dev/indico.conf @@ -0,0 +1,31 @@ +# General settings +SQLALCHEMY_DATABASE_URI = 'postgresql://indico:indicopass@indico-postgres:5432/indico' +SECRET_KEY = 'super_secret_random_key' +BASE_URL = 'http://localhost:8080' + +DEFAULT_TIMEZONE = 'Europe/Zurich' +DEFAULT_LOCALE = 'en_GB' + +REDIS_CACHE_URL = 'redis://indico-redis:6379/0' +CELERY_BROKER = 'redis://indico-redis:6379/1' + +ENABLE_ROOMBOOKING = True + +LOG_DIR = '/opt/indico/log' +TEMP_DIR = '/opt/indico/tmp' +CACHE_DIR = '/opt/indico/cache' +CUSTOMIZATION_DIR = '/opt/indico/custom' + +STORAGE_BACKENDS = {'default': 'fs:/opt/indico/archive'} +ATTACHMENT_STORAGE = 'default' + +PLUGINS = {'previewer_code', 'vc_zoom', 'payment_manual'} + +# Development settings +DB_LOG = True +DEBUG = True +SMTP_USE_CELERY = False +LOCAL_MODERATION = True + +NO_REPLY_EMAIL = 'noreply@example.com' +SUPPORT_EMAIL = 'support@example.com' diff --git a/indico-dev/nginx/Dockerfile b/indico-dev/nginx/Dockerfile new file mode 100644 index 0000000..dfa295f --- /dev/null +++ b/indico-dev/nginx/Dockerfile @@ -0,0 +1,20 @@ +FROM nginxinc/nginx-unprivileged:stable-alpine + +USER root + +RUN set -ex && \ + apk add --update py-pip unzip && \ + rm -rf /var/cache/apk/* + +RUN rm /etc/nginx/conf.d/default.conf + +EXPOSE 8080 + +# OpenShift runs containers using an arbitrarily assigned user ID for security reasons +# This user is always in the root group so it is needed to grant privileges to group 0. +RUN chgrp -R 0 /var/* /etc/nginx && chmod -R g+rwX /var/* /etc/nginx + +COPY run_nginx.sh indico.conf.template / +RUN chmod +x /run_nginx.sh + +ENTRYPOINT ["/run_nginx.sh"] diff --git a/indico-dev/nginx/indico.conf.template b/indico-dev/nginx/indico.conf.template new file mode 100644 index 0000000..1305ab8 --- /dev/null +++ b/indico-dev/nginx/indico.conf.template @@ -0,0 +1,32 @@ +server { + # localhost:8080 is the main entrypoint of this docker-compose setup + listen 8080; + listen [::]:8080; + server_name ${NGINX_SERVER_NAME}; + + access_log /var/log/nginx/access.log combined; + access_log /dev/stdout combined; + error_log /var/log/nginx/error.log info; + error_log stderr info; + + root /var/empty; + + sendfile on; + + # Serve static files + location ~ ^/(images|fonts)(.*)/(.+?)(__v[0-9a-f]+)?\.([^.]+)$ { + alias /opt/indico/static/$1$2/$3.$5; + } + + location ~ ^/(css|dist|images|fonts)/(.*)$ { + alias /opt/indico/static/$1/$2; + } + + location / { + # indico-web is the container running Indico + proxy_pass http://indico-web:59999; + proxy_set_header Host $server_name; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/indico-dev/nginx/run_nginx.sh b/indico-dev/nginx/run_nginx.sh new file mode 100644 index 0000000..97fa8c0 --- /dev/null +++ b/indico-dev/nginx/run_nginx.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +envsubst '\$NGINX_SERVER_NAME' < /indico.conf.template > /etc/nginx/conf.d/indico.conf +nginx -g "daemon off;" diff --git a/indico-dev/worker/.transifexrc b/indico-dev/worker/.transifexrc new file mode 100644 index 0000000..bea6cb2 --- /dev/null +++ b/indico-dev/worker/.transifexrc @@ -0,0 +1,5 @@ +[https://www.transifex.com] +api_hostname = https://api.transifex.com +hostname = https://www.transifex.com +username = api +rest_hostname = https://rest.api.transifex.com diff --git a/indico-dev/worker/Dockerfile b/indico-dev/worker/Dockerfile new file mode 100644 index 0000000..c6ab3aa --- /dev/null +++ b/indico-dev/worker/Dockerfile @@ -0,0 +1,61 @@ +FROM python:3.12-alpine + +ENV INDICO_VIRTUALENV="/opt/indico/.venv" INDICO_CONFIG="/opt/indico/etc/indico.conf" + +ARG pip="${INDICO_VIRTUALENV}/bin/pip" + +USER root + +# Install dependencies +RUN set -ex && apk update && \ + apk add --no-cache libpq-dev postgresql-client vim less bash curl gettext git nodejs npm zip wget \ + gcc g++ make musl-dev linux-headers mupdf-dev freetype-dev pango + # gcc, g++, make, musl-dev, linux-headers, mupdf-dev and freetype-dev are needed to build some python and/or node libraries + +# Install maildump +RUN pip install --no-cache-dir pipx && \ + pipx install maildump && \ + pip uninstall pipx -y + +# Create a virtual environment +RUN python -m venv ${INDICO_VIRTUALENV} +RUN ${pip} install --no-cache-dir --upgrade pip uwsgi + +# Install the Transifex client +RUN mkdir -p /opt/tx +RUN cd /opt/tx && curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash + +WORKDIR /opt/indico + +# Clone indico +RUN git clone https://github.com/indico/indico.git src --depth 1 + +# Clone indico-plugins +RUN git clone https://github.com/indico/indico-plugins.git indico-plugins --depth 1 + +# Install requirements +RUN cd /opt/indico/src && ${pip} install --no-cache-dir -e '.[dev]' +RUN for dir in /opt/indico/indico-plugins/*/; do (cd "${dir}" && ${pip} install --no-cache-dir -e '.[dev]'); done + +# Run webpack for indico and plugins +RUN cd /opt/indico/src && npm ci && \ + ${INDICO_VIRTUALENV}/bin/python ./bin/maintenance/build-assets.py indico --dev && \ + ${INDICO_VIRTUALENV}/bin/python ./bin/maintenance/build-assets.py all-plugins --dev /opt/indico/indico-plugins && \ + npm cache clean --force + +RUN ["/bin/bash", "-c", "mkdir -p --mode=775 /opt/indico/{etc,tmp,log,cache,archive}"] + +RUN ${INDICO_VIRTUALENV}/bin/indico setup create-symlinks /opt/indico +RUN ${INDICO_VIRTUALENV}/bin/indico setup create-logging-config /opt/indico/etc + +COPY .transifexrc /opt/indico/etc/ + +WORKDIR /opt/indico + +# uwsgi & maildump ports +EXPOSE 59999 60000 + +COPY uwsgi.ini /etc/uwsgi.ini + +COPY run_indico.sh run_celery.sh pull_translations.sh run_initial_setup.py /opt/indico/ +RUN chmod 755 /opt/indico/*.sh diff --git a/indico-dev/worker/pull_translations.sh b/indico-dev/worker/pull_translations.sh new file mode 100644 index 0000000..7b8bb0b --- /dev/null +++ b/indico-dev/worker/pull_translations.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [[ -z $TRANSIFEX_API_TOKEN ]]; then + echo "Transifex API token not provided" + echo "Pulling translations from a mirror..." + cd /opt/indico/src + wget --tries=5 https://test-indico-transifex-mirror.app.cern.ch/translations.zip + if [ $? -ne 0 ]; then + echo "Failed to pull from mirror" + exit 1 + fi + unzip translations.zip + # Delete existing translations + rm -R indico/translations/*/ + # Move the current translations + mv translations/* indico/translations/ + rm -r translations +else + echo "Transifex API token provided" + echo "Pulling translations from Transifex..." + cd /opt/indico/src && /opt/tx/tx --token=$TRANSIFEX_API_TOKEN --root-config=/opt/indico/etc/.transifexrc pull --all -f +fi diff --git a/indico-dev/worker/run_celery.sh b/indico-dev/worker/run_celery.sh new file mode 100644 index 0000000..c44dab1 --- /dev/null +++ b/indico-dev/worker/run_celery.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +. /opt/indico/.venv/bin/activate + +check_db_ready() { + psql -c 'SELECT COUNT(*) FROM events.events' +} + +# Wait until the DB becomes ready +check_db_ready +until [ $? -eq 0 ]; do + echo "Waiting for DB to be ready..." + sleep 10 + check_db_ready +done + +echo 'Starting Celery...' +indico celery worker -B diff --git a/indico-dev/worker/run_indico.sh b/indico-dev/worker/run_indico.sh new file mode 100644 index 0000000..0e1a468 --- /dev/null +++ b/indico-dev/worker/run_indico.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +. /opt/indico/.venv/bin/activate + +connect_to_db() { + psql -lqt | cut -d \| -f 1 | grep -qw $PGDATABASE +} + +# Wait until the DB becomes available +until connect_to_db; do + echo "Waiting for DB to become available..." + sleep 1 +done + +# Check whether the DB is already setup +psql -c 'SELECT COUNT(*) FROM events.events' + +if [ $? -eq 1 ]; then + echo 'Preparing DB...' + echo 'CREATE EXTENSION unaccent;' | psql + echo 'CREATE EXTENSION pg_trgm;' | psql + indico db prepare + echo 'Running initial setup...' + python /opt/indico/run_initial_setup.py +fi + +echo "Pulling translations..." +/opt/indico/pull_translations.sh + +echo 'Compiling translations...' +indico i18n compile-catalog +indico i18n compile-catalog-react + +echo 'Starting maildump...' +/root/.local/bin/maildump -n --http-ip 0.0.0.0 --http-port 60000 --db /tmp/maildump.sqlite --smtp-ip 127.0.0.1 --smtp-port 25 & + +echo 'Starting Indico...' +uwsgi /etc/uwsgi.ini diff --git a/indico-dev/worker/run_initial_setup.py b/indico-dev/worker/run_initial_setup.py new file mode 100644 index 0000000..29a0fb0 --- /dev/null +++ b/indico-dev/worker/run_initial_setup.py @@ -0,0 +1,44 @@ +from indico.web.flask.app import make_app + + +def _create_user(db): + """Create and admin user in order to skip the manual /bootstrap setup""" + from indico.core.config import config + from indico.modules.auth import Identity + from indico.modules.users import User + + user = User() + user.first_name = "John" + user.last_name = "Doe" + user.affiliation = "CERN" + user.email = "john.doe@example.com" + user.is_admin = True + + identity = Identity(provider='indico', identifier="admin", password="indiko42") + user.identities.add(identity) + + db.session.add(user) + db.session.flush() + + user.settings.set('timezone', config.DEFAULT_TIMEZONE) + user.settings.set('lang', config.DEFAULT_LOCALE) + db.session.commit() + + +def _create_announcement(db): + from indico.modules.announcement import announcement_settings + + message = ('THIS INSTANCE IS MEANT TO BE USED FOR DEVELOPMENT AND TESTING ONLY. ' + 'DO NOT USE IT IN PRODUCTION OR ON ANY PUBLICLY ACCESSIBLE SERVER!') + announcement_settings.set_multi({ 'enabled': True, 'message': message }) + db.session.flush() + db.session.commit() + + +with make_app().app_context(): + from indico.core.db import db + + print("Creating an admin user...") + _create_user(db) + print("Creating an announcement...") + _create_announcement(db) diff --git a/indico-dev/worker/uwsgi.ini b/indico-dev/worker/uwsgi.ini new file mode 100644 index 0000000..e369a9f --- /dev/null +++ b/indico-dev/worker/uwsgi.ini @@ -0,0 +1,37 @@ +[uwsgi] +;uid = indico +;gid = nginx +;umask = 027 +;pidfile = /run/uwsgi/uwsgi.pid +;stats = /opt/indico/web/uwsgi-stats.sock + +processes = 4 +enable-threads = true +http-socket = 0.0.0.0:59999 +protocol = http + +master = true +auto-procname = true +procname-prefix-spaced = indico +disable-logging = true + +;plugin = python +single-interpreter = true + +touch-reload = /opt/indico/indico.wsgi +wsgi-file = /opt/indico/indico.wsgi +virtualenv = /opt/indico/.venv + +ignore-sigpipe = true +ignore-write-errors = true +disable-write-exception = true + +vacuum = true +memory-report = true +max-requests = 2500 +harakiri = 900 +harakiri-verbose = true +reload-on-rss = 2048 +evil-reload-on-rss = 8192 + +offload-threads = 4