docker-compose.yml

# -------------------------------------------------------------
# This service configuration defines our main public website access stack.
# -------------------------------------------------------------

version: '3.2'

services:

  # -------------------------------------------------------------
  # Web proxy and static site setup
  # REQUIRED
  # -------------------------------------------------------------
  # All routing to sub-components goes through here.
  # All complex setup relating to the public website is here.
  # Static content hosted within the container, including robots.txt
  nginx:
    image:  "${UKWA_NGINX_IMAGE}"
    hostname: "{{.Service.Name}}-{{.Task.Slot}}"
    ports:
      - "80:80"
    volumes:
      - ./config/nginx.conf:/etc/nginx/conf.d/default.conf:ro
    depends_on:
      - shine
      - pywb
      - ukwa-ui
      - api
      - mementos

  # -------------------------------------------------------------
  # The main UKWA UI itself
  # REQUIRED
  # -------------------------------------------------------------
  ukwa-ui:
    # UI container image configured by deployment context (see deployment script):
    image:  "${UKWA_UI_IMAGE}"
    command: java -Djava.security.egd=file:/dev/./urandom -jar /ukwa-ui.war --spring.config.location=file:/application.properties
    #ports:
    #  - "9030:8080"
    environment:
     - "OFF_SITE_URL=/wayback/archive/"
     - "SOLR_COLLECTION_SEARCH_PATH=http://collections_solr:8983"
     - "SOLR_COLLECTION_SEARCH_REQUEST_HANDLER=/solr/collections/select"
     - "SOLR_FULL_TEXT_SEARCH_PATH=${SOLR_FULL_TEXT_SEARCH_PATH}"
     - "SOLR_FULL_TEXT_SEARCH_REQUEST_HANDLER=/solr/all/select"
     - "SOLR_READ_TIMEOUT=600000"
     - "SOLR_CONNECTION_TIMEOUT=60000"
     - "SOLR_USERNAME="
     - "SOLR_PASSWORD="
     - "SOLR_SHOW_STUB_DATA_SERVICE_NOT_AVAILABLE=true"
     - "BL_SMTP_SERVER_HOST=juno.bl.uk"
     - "BL_SMTP_USERNAME="
     - "BL_SMTP_PASSWORD="
     # CAPTCHA validation needs to talk to Google, but other web connections are internal, so this is set specifically:
     - "CAPTCHA_PROXY_HOST=explorer.bl.uk"
     - "CAPTCHA_PROXY_PORT=3127"
     - "RECAPTCHA_SITE_KEY=${RECAPTCHA_SITE_KEY}"
     - "RECAPTCHA_SECRET_KEY=${RECAPTCHA_SECRET_KEY}"
    volumes:
      - ./config/ukwa-ui/ukwa-ui.properties:/application.properties:ro
    networks:
      - default

  # -------------------------------------------------------------
  # OA Wayback configuration:
  # REQUIRED
  # -------------------------------------------------------------
  pywb:
    image: "${PYWB_IMAGE}"
    environment:
      - "WEBHDFS_USER=access"
      - "UKWA_INDEX=${UKWA_INDEX}"
      - "UKWA_ARCHIVE=webhdfs://warc-server:8000/by-filename/"
      - "LANG=en_GB.UTF-8"
    ports:
      - "7171:8080"
    volumes:
      - ./config/pywb/config.yaml:/webarchive/config.yaml
      - ./config/pywb/uwsgi.ini:/ukwa_pywb/uwsgi.ini
      # Used to store the access control lists used by the pywb services.
      - "${PWYB_ACL_PATH}:/ukwa_pywb/acl/"
    deploy:
      # If we don't place a limit on resources it just consumes all RAM rather than cleaning up:
      resources:
        limits:
          cpus: '4.0'
          memory: '2G'

  # -------------------------------------------------------------
  # An embedded, bannerless OA Wayback to render screenshots:
  # OPTIONAL: Used by the API but config. is here so it's easier to compare with the maiun PyWB instance above
  # -------------------------------------------------------------
  pywb-nobanner:
    image: "${PYWB_IMAGE}"
    environment:
      - "WEBHDFS_USER=access"
      - "UKWA_INDEX=${UKWA_INDEX}"
      - "UKWA_ARCHIVE=webhdfs://warc-server:8000/by-filename/"
    volumes:
      - ./config/pywb/config-nobanner.yaml:/webarchive/config.yaml
      - ./config/pywb/empty-banner.js:/ukwa_pywb/static/default_banner.js
      - "${PWYB_ACL_PATH}:/ukwa_pywb/acl/"
      - "${CONFIG_PATH}/pywb-ca.pem:/certs/pywb-ca.pem"
    deploy:
      # If we don't place a limit on resources it just consumes all RAM rather than cleaning up:
      resources:
        limits:
          cpus: '4.0'
          memory: '2G'

  # -------------------------------------------------------------
  # Internal WARC server that can pull from multiple clusters and local disk
  # REQUIRED
  # -------------------------------------------------------------
  warc-server:
    image: ukwa/warc-server:1.1.0
    # Alternative command for debugging:
    #command: "gunicorn --log-level debug --error-logfile - --access-logfile - --bind 0.0.0.0:8000 --workers 5 --worker-class gthread --threads 20 warc_server:app"
    ports:
      - 8002:8000
    environment:
      - "WARC_PATHS=/heritrix/output,/heritrix/wren"
      - "TRACKDB_URL=http://solr8.api.wa.bl.uk/solr/tracking"
      # Default WebHDFS service used if there's no access_url_s in TrackDB (user is 'access'):
      - "WEBHDFS_PREFIX=http://hdfs.api.wa.bl.uk/webhdfs/v1"
    volumes:
      - "/mnt/gluster/fc/heritrix/output:/heritrix/output"
      - "/mnt/gluster/fc/heritrix/wren:/heritrix/wren"

  # -------------------------------------------------------------
  # Collections index for Topics & Themes of the UKWA UI
  # REQUIRED
  # -------------------------------------------------------------
  collections_solr:
    image: ukwa/ukwa-ui-collections-solr:1.2.0
    user: "${CURRENT_UID}"
    volumes:
        - "${STORAGE_PATH_WEBSITE}/collections_solr_cores:/opt/solr/server/solr/mycores"
        - "${STORAGE_PATH_WEBSITE}/collections_solr_logs:/opt/solr/server/logs"
    ports:
        - "9021:8983" # Exposed port so external clients can run checks (TBC)

  # -------------------------------------------------------------
  # Mementos
  # OPTIONAL
  # -------------------------------------------------------------
  mementos:
    image: ukwa/mementoweb-webclient
    command: "/mementos/bin/mementos-webclient -Dhttp.proxyHost=${PROXYHOST} -Dhttp.proxyPort=${PROXYPORT} -Dconfig.file=/mementos/conf/application.conf -Dpidfile.path=/dev/null"
    environment:
      - "JAVA_OPTS=-Xmx1g -Dhttp.proxyHost=${PROXYHOST} -Dhttp.proxyPort=${PROXYPORT}"

  # -------------------------------------------------------------
  # Shine service and database
  # OPTIONAL
  # -------------------------------------------------------------
  # Shine
  shine:
   image: ukwa/shine
   environment:
     - "SOLR_URL=${SHINE_SOLR}"
     - "JAVA_OPTS=-Xmx2g -XX:+ExitOnOutOfMemoryError"
   depends_on:
     - shinedb

  # Shine state
  shinedb:
    image: postgres:9.6.2
    volumes:
      # Used to store the Shine PostgreSQL database (which hold user accounts).
      - "${STORAGE_PATH_WEBSITE}/shine-postgres-data:/var/lib/postgresql/data"
      
      
  # -------------------------------------------------------------
  # The set of services that provides a public API:
  # OPTIONAL
  # -------------------------------------------------------------
  api:
    image: "${API_IMAGE}"
    environment:
      - "CDX_SERVER=${CDX_SERVER}"
      - "WEBHDFS_PREFIX=http://warc-server:8000/by-filename/"
      - "API_LABEL=UK Web Archive API"
      - "KAFKA_LAUNCH_BROKER=${KAFKA_BROKER}"
      - "KAFKA_LAUNCH_TOPIC=fc.tocrawl.npld"
      #- "HTTPS_PROXY=http://${PROXYHOST}:${PROXYPORT}"
      - "SERVER_NAME=${SERVER_NAME}"
      - "ANALYSIS_SOURCE_FILE=/analysis/fc.crawled.json"
      - "SCRIPT_NAME=/api" # Deploy the API under this prefix
      - "HTTP_X_FORWARDED_PROTO=https"
      - "WEBRENDER_ARCHIVE_SERVER=http://webrender:8010/render"
      - "CACHE_FOLDER=/cache"
      # Used to check access permissions:
      - "WAYBACK_SERVER=http://pywb:8080/archive/"
      # Used to secure cookies/tokens/etc.
      - "APP_SECRET=${APP_SECRET}"
      # Nominations DB creation is not safe to run under multiple processes, so is off at present:
      - "WORKERS=10"
    volumes:
      - "${STORAGE_PATH_WEBSITE}/cache:/cache"
      - fc_analysis:/analysis
    depends_on:
      - webrender

  # Web page rendering post-crawl:
  # (uses pywb-nobanner)
  webrender:
    image: ukwa/webrender-puppeteer:2.3.3
    environment:
      - "HTTP_PROXY=http://pywb-nobanner:8080"
      - "RUN_BEHAVIOURS=false"
      - "LC_ALL=en_US.utf8"
      - "PORT=8010"
      - "NODE_ENV=production"
      - "PUPPETEER_CLUSTER_SIZE=8"
    depends_on:
      - pywb-nobanner
    deploy:
      replicas: 1

  # IIIF Server to cache and manipulate images
  iiif:
    image: lyrasis/cantaloupe
    environment:
      SOURCE_STATIC: "HttpSource"
      HTTPSOURCE_BASICLOOKUPSTRATEGY_URL_PREFIX: "http://api:8000/iiif/render_raw?pwid="
      HTTPSOURCE_BASICLOOKUPSTRATEGY_URL_SUFFIX: ""
      HTTPSOURCE_REQUEST_TIMEOUT: 3600
      PROCESSOR_STREAM_RETRIEVAL_STRATEGY: "CacheStrategy"
      # Seems to cope better with sharp edges:
      PROCESSOR_DOWNSCALE_FILTER: "Lanczos3"
      # Web pages suffer badly from JPEG artefacts. Can use PNG instead.
      PROCESSOR_JPG_QUALITY: 100
      CACHE_SERVER_RESOLVE_FIRST: "false"
      LOG_APPLICATION_LEVEL: "info"
      LOG_ACCESS_CONSOLEAPPENDER_ENABLED: "true"
      # Not needed in prod:
      PRINT_STACK_TRACE_ON_ERROR_PAGES: "false"
    #ports:
    #  - 8182:8182
    volumes:
      - ${STORAGE_PATH_WEBSITE}/iiif_cache:/var/cache/cantaloupe

  # Analyses recent crawl behaviour by processing the crawled data stream:
  analyse:
    image: ukwa/crawl-streams
    user: "${CURRENT_UID}"
    command: "analyse -k ${KAFKA_BROKER} -u 2 -o /analysis/fc.crawled.json"
    volumes:
      - "fc_analysis:/analysis"

# -----------------------------------------
# Volumes and networks supporting the above
# -----------------------------------------

networks:
  # This attachable network is needed so the screenshotter can spin up containers and talk to them over HTTP.
  # It also makes it simpler to connect to the PostgreSQL instance for maintenance.
  default:
    driver: overlay
    attachable: true

volumes:

  fc_analysis:
    driver: local
    driver_opts:
      type: none
      device: ${STORAGE_PATH_WEBSITE}/fc_analysis
      o: bind