docker-compose.yaml

version: '3.7'

networks:
  BigData:
    name: BigData-network
    driver: bridge

services:
  spark:
    build:
      dockerfile: ./dockerfiles/spark/dockerfile
    container_name: spark
    volumes:
      - ./spark-container:/spark-container
      - ./config:/config
    env_file:
      - ./config/dwh.env
      - ./config/nessie.env
      - ./config/minio.env
    ports:
      - 8888:8888
      - 8082:8080
      - 10000:10000
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - AWS_DEFAULT_REGION=us-east-1
      - SPARK_MASTER_HOST=spark
      - SPARK_MASTER_PORT=7077 
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=0.5g
      - PYTHONPATH=spark-container/spark:/config:/spark-container$PYTHONPATH
    networks:
      - BigData
    
  nessie:
    image: projectnessie/nessie:0.67.0
    container_name: nessie
    ports:
      - 19120:19120
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
    networks:
      - BigData

  minio:
    image: minio/minio:RELEASE.2023-07-21T21-12-44Z
    container_name: minio
    hostname: minio-server
    ports:
      - 9001:9001
      - 9000:9000
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
    command: ["server", "/data", "--console-address", ":9001"]
    networks:
      - BigData

  mc:
    image: minio/mc
    container_name: mc
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c "
      tail -f /dev/null
      "      
    networks:
      - BigData
    volumes:
      - ./data/original_dataset/batchs:/data/original_dataset/batchs
    

  # dremio:
  #   platform: linux/x86_64
  #   container_name: dremio
  #   image: dremio/dremio-oss:latest
  #   ports:
  #     - 9047:9047
  #     - 31010:31010
  #     - 32010:32010
  #   networks:
  #     - BigData