docker-compose.yml

---
version: '3.8'
services:

  gateway:
    container_name: gateway
    image: nginx:1.19-alpine
    ports:
      - "80:80"
    depends_on:
      - assembler.collector-load-balancer
      - distributor.load-balancer
    volumes:
      - ./gateway/nginx.conf:/etc/nginx/nginx.conf
      - ./gateway/www:/var/www
      - ./gateway/log:/var/log/nginx


  zookeeper:
    image: wurstmeister/zookeeper:latest
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000


  trie-backend-applier:
    container_name: trie-backend-applier
    build: ./trie-backend-applier
    depends_on:
      - zookeeper
    volumes:
      - ./trie-backend-applier/applier.py:/app/trie-backend-applier/applier.py
    environment:
      - NUMBER_NODES_PER_PARTITION=2
      - ZOOKEEPER_HOST=zookeeper
      - LOG_LEVEL=INFO
    command: python /app/trie-backend-applier/applier.py


  # # # # # # # # # 
  # Distributor
  # # # # # # # # # 

  distributor.load-balancer:
    container_name: distributor.load-balancer
    image: nginx:1.19-alpine
    ports:
      - "5000:5000"
    depends_on:
      - distributor.frontend
    volumes:
      - ./distributor/load-balancer/nginx.conf:/etc/nginx/nginx.conf
      - ./distributor/load-balancer/log:/var/log/nginx


  distributor.frontend:
    build: ./distributor/frontend
    ports:
      - "8000"
    depends_on:
      - zookeeper
    volumes:
      - ./distributor/frontend/wsgi.py:/app/distributor/frontend/wsgi.py
      - ./distributor/frontend/main.py:/app/distributor/frontend/main.py
      - ./distributor/frontend/frontend.py:/app/distributor/frontend/frontend.py
      - ./distributor/frontend/gunicorn_config.py:/app/distributor/frontend/gunicorn_config.py
    environment:
      - ZOOKEEPER_HOST=zookeeper
      - DISTRIBUTED_CACHE_HOST=distributor.distributed-cache
      - DISTRIBUTED_CACHE_ENABLED=false
      - PORT=8000
      - RELOAD=true
      - NUM_WORKERS=1
      - LOG_LEVEL=DEBUG
    command: gunicorn --chdir /app/distributor/frontend -c /app/distributor/frontend/gunicorn_config.py wsgi:app


  distributor.backend:
    build: ./distributor/backend
    ports:
      - "8001"
    depends_on:
      - zookeeper
      - assembler.hadoop.namenode
    volumes:
      - ./distributor/backend/wsgi.py:/app/distributor/backend/wsgi.py
      - ./distributor/backend/main.py:/app/distributor/backend/main.py
      - ./distributor/backend/backend.py:/app/distributor/backend/backend.py
      - ./shared/trie.py:/app/distributor/backend/trie.py
      - ./distributor/backend/gunicorn_config.py:/app/distributor/backend/gunicorn_config.py
    environment:
      - NUMBER_NODES_PER_PARTITION=2
      - HADOOP_NAMENODE_HOST=assembler.hadoop.namenode
      - ZOOKEEPER_HOST=zookeeper
      - PORT=8001
      - RELOAD=true
      - NUM_WORKERS=1
      - LOG_LEVEL=INFO
    command: gunicorn --chdir /app/distributor/backend -c /app/distributor/backend/gunicorn_config.py wsgi:app


  distributor.distributed-cache:
    container_name: distributor.distributed-cache
    image: redis:6.0.6
    ports:
      - "6379:6379"


  # # # # # # # # # 
  # Assembler
  # # # # # # # # # 

  assembler.collector-load-balancer:
    container_name: assembler.collector-load-balancer
    image: nginx:1.19-alpine
    ports:
      - "6000:6000"
    depends_on:
      - assembler.collector
    volumes:
      - ./assembler/collector-load-balancer/nginx.conf:/etc/nginx/nginx.conf
      - ./assembler/collector-load-balancer/log:/var/log/nginx


  assembler.collector:
    build: ./assembler/collector
    ports:
      - "7000"
    depends_on:
      - assembler.broker
      - assembler.schema-registry
    volumes:
      - ./assembler/collector/wsgi.py:/app/assembler/collector/wsgi.py
      - ./assembler/collector/main.py:/app/assembler/collector/main.py
      - ./assembler/collector/collector.py:/app/assembler/collector/collector.py
      - ./assembler/collector/gunicorn_config.py:/app/assembler/collector/gunicorn_config.py
    environment:
      - BROKER_HOST=assembler.broker
      - SCHEMA_REGISTRY_HOST=assembler.schema-registry
      - PORT=7000
      - RELOAD=true
      - NUM_WORKERS=1
      - LOG_LEVEL=DEBUG
    command: gunicorn --chdir /app/assembler/collector -c /app/assembler/collector/gunicorn_config.py wsgi:app


  assembler.broker:
    image: wurstmeister/kafka:2.12-2.5.0
    hostname: assembler.broker
    container_name: assembler.broker
    depends_on:
      - zookeeper
    ports:
      - "19092:19092"
      - "9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://assembler.broker:9092,CONNECTIONS_FROM_HOST://localhost:19092
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_JMX_PORT: 9101


  assembler.schema-registry:
    image: confluentinc/cp-schema-registry:5.5.1
    container_name: assembler.schema-registry
    ports: 
      - 8081:8081
    depends_on:
      - zookeeper
      - assembler.broker
    environment:
      SCHEMA_REGISTRY_HOST_NAME: assembler.schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181


  assembler.kafka-connect:
    build: ./assembler/kafka-connect
    hostname: assembler.kafka-connect
    container_name: assembler.kafka-connect
    depends_on:
      - zookeeper
      - assembler.broker
      - assembler.schema-registry
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'assembler.broker:9092'
      CONNECT_REST_ADVERTISED_HOST_NAME: assembler.kafka-connect
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://assembler.schema-registry:8081'
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://assembler.schema-registry:8081'
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      CONNECT_PLUGIN_PATH: "/usr/share/confluent-hub-components"
      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
    volumes:
      - ./assembler/kafka-connect/scripts:/scripts
    command: 
      - bash 
      - -c 
      - |
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run & 
        #
        echo "Waiting for Kafka Connect to start listening on $$CONNECT_REST_ADVERTISED_HOST_NAME ⏳"

        echo "Waiting for Kafka Connect to start listening on $$CONNECT_REST_ADVERTISED_HOST_NAME ⏳"
        while [ $$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors) -ne 200 ] ; do 
          echo -e $$(date) " Kafka Connect listener HTTP state: " $$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors) " (waiting for 200)"
          sleep 2
        done

        nc -vz $$CONNECT_REST_ADVERTISED_HOST_NAME $$CONNECT_REST_PORT
        echo -e "\n--\n+> Deploying HDFS Connector"
        chmod +x /scripts/deploy-connect-hdfs.sh
        /scripts/deploy-connect-hdfs.sh 
        sleep infinity


  assembler.trie-builder:
    container_name: assembler.trie-builder
    build: ./assembler/trie-builder
    depends_on:
      - zookeeper
      - assembler.hadoop.namenode
    volumes:
      - ./assembler/trie-builder/triebuilder.py:/app/assembler/trie-builder/triebuilder.py
      - ./shared/trie.py:/app/assembler/trie-builder/trie.py
    environment:
      - HADOOP_NAMENODE_HOST=assembler.hadoop.namenode
      - HADOOP_DATANODE_HOST=assembler.hadoop.datanode
      - ZOOKEEPER_HOST=zookeeper
      - LOG_LEVEL=INFO
    command: python /app/assembler/trie-builder/triebuilder.py


  assembler.hadoop.namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: assembler.hadoop.namenode
    hostname: assembler.hadoop.namenode
    restart: always
    ports:
      - 9870:9870
      - 9000:9000
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./assembler/hadoop/hadoop.env

  assembler.hadoop.datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: assembler.hadoop.datanode
    hostname: assembler.hadoop.datanode
    restart: always
    ports:
      - 9864:9864
    environment:
      SERVICE_PRECONDITION: "assembler.hadoop.namenode:9870"
    env_file:
      - ./assembler/hadoop/hadoop.env
  
  assembler.hadoop.resourcemanager:
    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
    container_name: assembler.hadoop.resourcemanager
    restart: always
    environment:
      SERVICE_PRECONDITION: "assembler.hadoop.namenode:9000 assembler.hadoop.namenode:9870 assembler.hadoop.datanode:9864"
    env_file:
      - ./assembler/hadoop/hadoop.env

  assembler.hadoop.nodemanager:
    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
    container_name: assembler.hadoop.nodemanager
    restart: always
    environment:
      SERVICE_PRECONDITION: "assembler.hadoop.namenode:9000 assembler.hadoop.namenode:9870 assembler.hadoop.datanode:9864 assembler.hadoop.resourcemanager:8088"
    env_file:
      - ./assembler/hadoop/hadoop.env
  
  assembler.hadoop.historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
    container_name: assembler.hadoop.historyserver
    restart: always
    environment:
      SERVICE_PRECONDITION: "assembler.hadoop.namenode:9000 assembler.hadoop.namenode:9870 assembler.hadoop.datanode:9864 assembler.hadoop.resourcemanager:8088"
    env_file:
      - ./assembler/hadoop/hadoop.env