docker-compose.yml

version: '3.9'

services:
  # ESGF East Node FastAPI Server
  app-elasticsearch-east:
    container_name: stac-fastapi-es-east
    image: ghcr.io/djspstfc/stac-fastapi-es:1.0
    restart: always
    environment:
      - STAC_FASTAPI_TITLE=Simulated ESGF East Node Index STAC interface
      - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend
      - STAC_FASTAPI_VERSION=2.1
      - APP_HOST=0.0.0.0
      - APP_PORT=8080
      - RELOAD=true
      - ENVIRONMENT=local
      - WEB_CONCURRENCY=10
      - ES_HOST=elasticsearch-east
      - ES_PORT=9200
      - ES_USE_SSL=false
      - ES_VERIFY_CERTS=false
      - BACKEND=elasticsearch
    ports:
      - "9010:8080"
    volumes:
      - ./esdata-east:/usr/share/elasticsearch/data
    depends_on:
      - elasticsearch-east
    command:
      bash -c "./scripts/wait-for-it-es.sh es-container-east:9200 && python -m stac_fastapi.elasticsearch.app"

  # ESGF East Node ElasticSearch Server
  elasticsearch-east:
    container_name: es-container-east
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
    hostname: elasticsearch-east
    environment:
      ES_JAVA_OPTS: -Xms512m -Xmx1g
    volumes:
      - ./elasticsearch/config/elasticsearch-core.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - "9200:9200"

  # ESGF East Node STAC Browser
  stac-browser-east:
    container_name: stac-browser-east
    image: esgfdeploy/stac-browser:v1
    environment:
      SB_catalogUrl: http://localhost:9010/
    ports:
      - "9011:8080"
    depends_on:
      - app-elasticsearch-east

  # ESGF East Node KAFKA Consumer
  stac-consumer-east: # TODO build this
    build:
      context: esgf-consumer
      dockerfile: Dockerfile
    environment:
      CONSUMER_GROUP: "esgf-east"
      BOOTSTRAP_SERVERS: '["kafka1:19092"]'
      STAC_SERVER: "http://app-elasticsearch-east:8080"
    depends_on:
      kafka1:
        condition: service_healthy

  # ESGF West Node FastAPI Server
  app-elasticsearch-west:
    container_name: stac-fastapi-es-west
    image: ghcr.io/djspstfc/stac-fastapi-es:1.0
    restart: always
    environment:
      - STAC_FASTAPI_TITLE=Simulated ESGF West Node Index STAC interface
      - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend
      - STAC_FASTAPI_VERSION=2.1
      - APP_HOST=0.0.0.0
      - APP_PORT=8080
      - RELOAD=true
      - ENVIRONMENT=local
      - WEB_CONCURRENCY=10
      - ES_HOST=elasticsearch-west
      - ES_PORT=9200
      - ES_USE_SSL=false
      - ES_VERIFY_CERTS=false
      - BACKEND=elasticsearch
    ports:
      - "9014:8080"
    volumes:
      - ./esdata-west:/usr/share/elasticsearch/data
    depends_on:
      - elasticsearch-west
    command:
      bash -c "./scripts/wait-for-it-es.sh es-container-west:9200 && python -m stac_fastapi.elasticsearch.app"

  # ESGF West Node ElasticSearch Server
  elasticsearch-west:
    container_name: es-container-west
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
    hostname: elasticsearch-west
    environment:
      ES_JAVA_OPTS: -Xms512m -Xmx1g
    volumes:
      - ./elasticsearch/config/elasticsearch-core.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - "9202:9200"

  # ESGF West Node STAC Browser
  stac-browser-west:
    container_name: stac-browser-west
    image: esgfdeploy/stac-browser:v1
    environment:
      SB_catalogUrl: http://localhost:9014/
    ports:
      - "9015:8080"
    depends_on:
      - app-elasticsearch-west

  # ESGF East West KAFKA Consumer
  stac-consumer-west: # TODO build this
    build:
      context: esgf-consumer
      dockerfile: Dockerfile
    environment:
      CONSUMER_GROUP: "esgf-west"
      BOOTSTRAP_SERVERS: '["kafka1:19092"]'
      STAC_SERVER: "http://app-elasticsearch-west:8080"
    depends_on:
      kafka1:
        condition: service_healthy

  # Secondary ESGF Node FastAPI
  app-elasticsearch-secondary:
    container_name: stac-fastapi-es-secondary
    image: ghcr.io/djspstfc/stac-fastapi-es:1.0
    restart: always
    environment:
      - STAC_FASTAPI_TITLE=Simulated ESGF Secondary Index STAC interface
      - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend
      - STAC_FASTAPI_VERSION=2.1
      - APP_HOST=0.0.0.0
      - APP_PORT=8080
      - RELOAD=true
      - ENVIRONMENT=local
      - WEB_CONCURRENCY=10
      - ES_HOST=elasticsearch-secondary
      - ES_PORT=9200
      - ES_USE_SSL=false
      - ES_VERIFY_CERTS=false
      - BACKEND=elasticsearch
    ports:
      - "9012:8080"
    volumes:
      - ./esdata-secondary:/usr/share/elasticsearch/data
    depends_on:
      - elasticsearch-secondary
    command:
      bash -c "./scripts/wait-for-it-es.sh es-container-secondary:9200 && python -m stac_fastapi.elasticsearch.app"

  # Secondary ESGF Node ElasticSearch Server
  elasticsearch-secondary:
    container_name: es-container-secondary
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
    hostname: elasticsearch-secondary
    environment:
      ES_JAVA_OPTS: -Xms512m -Xmx1g
    volumes:
      - ./elasticsearch/config/elasticsearch-secondary.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - "9201:9200"

  # Secondary ESGF Node STAC Browser
  stac-browser-secondary:
    container_name: stac-browser
    image: esgfdeploy/stac-browser:v1
    environment:
      SB_catalogUrl: http://localhost:9012/
    ports:
    - "9013:8080"
    depends_on:
    - app-elasticsearch-secondary

  # Secondary ESGF Node KAFKA Consumer
  stac-consumer-secondary: # TODO build this
    build:
      context: esgf-consumer
      dockerfile: Dockerfile
    environment:
      CONSUMER_GROUP: "secondary-node"
      BOOTSTRAP_SERVERS : '["kafka1:19092"]'
      KAFKA_TOPICS: '.*\.historical\..*'
      STAC_SERVER: "http://app-elasticsearch-secondary:8080"
    depends_on:
      kafka1:
        condition: service_healthy

  # ESGF East Ingestion API Server
  transaction_api_east:
    build:
      context: esgf-transaction-api
      dockerfile: Dockerfile
    environment:
      BOOTSTRAP_SERVERS : '["kafka1:19092"]'
    ports:
      - "9050:8000"
    depends_on:
      kafka1:
        condition: service_healthy

  # ESGF West Ingestion API Server
  transaction_api_west:
    build:
      context: esgf-transaction-api
      dockerfile: Dockerfile
    environment:
      BOOTSTRAP_SERVERS : '["kafka1:19092"]'
    ports:
      - "9051:8000"
    depends_on:
      kafka1:
        condition: service_healthy

  # Central ESGF Zookeeper Server
  zoo1:
    image: confluentinc/cp-zookeeper:7.3.2
    hostname: zoo1
    container_name: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888

  # Central ESGF KAFKA Server
  kafka1:
    image: confluentinc/cp-kafka:7.3.2
    hostname: kafka1
    container_name: kafka1
    ports:
      - "9092:9092"
      - "29092:29092"
      - "9999:9999"
    environment:
      KAFKA_LISTENERS: INTERNAL://:19092,EXTERNAL://:9092,DOCKER://:29092
      # TODO: Fix external
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://localhost:9092,DOCKER://host.docker.internal:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_JMX_PORT: 9001
      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "9092"]
      interval: 15s
      timeout: 10s
      retries: 3
    depends_on:
      - zoo1

  # Central ESGF Schema Registry (Optional)
  kafka-schema-registry:
    image: confluentinc/cp-schema-registry:7.3.2
    hostname: kafka-schema-registry
    container_name: kafka-schema-registry
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    depends_on:
      - zoo1
      - kafka1

  # Central ESGF KAFKA REST API Server (Optional)
  kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:7.3.2
    hostname: kafka-rest-proxy
    container_name: kafka-rest-proxy
    ports:
      - "8082:8082"
    environment:
      # KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
    depends_on:
      - zoo1
      - kafka1
      - kafka-schema-registry

  # Central ESGF KAFKA Connect Server (Optional)
  kafka-connect:
    image: confluentinc/cp-kafka-connect:7.3.2
    hostname: kafka-connect
    container_name: kafka-connect
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars,/usr/share/confluent-hub-components'
    volumes:
      - ./connectors:/etc/kafka-connect/jars/
    depends_on:
      - zoo1
      - kafka1
      - kafka-schema-registry
      - kafka-rest-proxy
    command:
      - bash
      - -c
      - |
        confluent-hub install --no-prompt debezium/debezium-connector-mysql:latest
        confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.4.0
        /etc/confluent/docker/run


  # Central ESGF KSQLDB Server (Optional)
  ksqldb-server:
    image: confluentinc/cp-ksqldb-server:7.3.2
    hostname: ksqldb-server
    container_name: ksqldb-server
    ports:
      - "8088:8088"
    environment:
      KSQL_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
      KSQL_LISTENERS: http://0.0.0.0:8088/
      KSQL_KSQL_SERVICE_ID: ksqldb-server_
    depends_on:
      - zoo1
      - kafka1

  # Central ESGF PostgrSQL Server (Optional)
  postgresql:
    hostname: postgresql
    container_name: postgresql
    extends:
      service: postgresql
      file: conduktor.yml

  # Central ESGF Conduktor Console Server (Optional)
  conduktor-console:
    hostname: conduktor-console
    container_name: conduktor-console
    extends:
      service: conduktor-console
      file: conduktor.yml

volumes:
  pg_data: {}
  conduktor_data: {}