diff --git a/infra/airflow/.env b/infra/airflow/.env index e69de29..a4fd63e 100644 --- a/infra/airflow/.env +++ b/infra/airflow/.env @@ -0,0 +1,14 @@ +POSTGRESQL_DATABASE=airflow +POSTGRESQL_USERNAME=airflow +POSTGRESQL_PASSWORD=airflow + + +AIRFLOW_FERNET_KEY== +AIRFLOW_EXECUTOR=CeleryExecutor +AIRFLOW_DATABASE_NAME=airflow +AIRFLOW_DATABASE_USERNAME=airflow +AIRFLOW_DATABASE_PASSWORD=airflow +AIRFLOW_LOAD_EXAMPLES=no +AIRFLOW_PASSWORD=bitnami123 +AIRFLOW_USERNAME=user +AIRFLOW_EMAIL=user@example.com diff --git a/infra/airflow/.gitignore b/infra/airflow/.gitignore new file mode 100644 index 0000000..1269488 --- /dev/null +++ b/infra/airflow/.gitignore @@ -0,0 +1 @@ +data diff --git a/infra/airflow/data/.gitignore b/infra/airflow/data/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/infra/airflow/docker-compose.yml b/infra/airflow/docker-compose.yml index e13572d..00db222 100644 --- a/infra/airflow/docker-compose.yml +++ b/infra/airflow/docker-compose.yml @@ -1,40 +1,72 @@ -# Based on https://github.com/puckel/docker-airflow/blob/master/docker-compose-LocalExecutor.yml +version: '2' -version: '3.7' services: - postgres: - image: postgres:9.6 - environment: - - POSTGRES_USER=airflow - - POSTGRES_PASSWORD=airflow - - POSTGRES_DB=airflow - logging: - options: - max-size: 10m - max-file: "3" - - webserver: - image: docker.pkg.github.com/instidea/controlciudadano/dags:latest -# build: ../../scripts/python/airflow/ - restart: always - depends_on: - - postgres + + postgresql: + image: bitnami/postgresql:12.5.0 + env_file: + - .env + volumes: + - ./data:/bitnami/postgresql + networks: + - airflow + + redis: + image: bitnami/redis:5.0.10 + env_file: + - .env environment: - - LOAD_EX=n - - EXECUTOR=Local - - AIRFLOW__WEBSERVER__AUTHENTICATE=True - - AIRFLOW__WEBSERVER__AUTH_BACKEND=airflow.contrib.auth.backends.password_auth + - ALLOW_EMPTY_PASSWORD=yes + volumes: + - ./data:/bitnami + networks: + - airflow + + airflow-worker: + image: bitnami/airflow-worker:2.0.0 + env_file: + - .env + volumes: + - ./data:/bitnami +# - "/opt/etl:/external/:rw" +# - "/home/cdssa/services/public_data/contraloria/declaraciones:/external/cgr/pdfs:rw" + networks: + - airflow + + airflow-scheduler: + image: bitnami/airflow-scheduler:2.0.0 + env_file: + - .env + volumes: + - ./data:/bitnami +# - "/opt/etl:/external/:rw" +# - "/home/cdssa/services/public_data/contraloria/declaraciones:/external/cgr/pdfs:rw" + networks: + - airflow + + airflow: + build: ../../scripts/python/airflow/ + #image: docker.pkg.github.com/instidea/controlciudadano/dags:latest env_file: - .env - logging: - options: - max-size: 10m - max-file: "3" ports: - - "48080:8080" - command: webserver - healthcheck: - test: [ "CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]" ] - interval: 30s - timeout: 30s - retries: 3 + - '8080:8080' + volumes: + - ./data:/bitnami +# - "/opt/etl:/external/:rw" +# - "/home/cdssa/services/public_data/contraloria/declaraciones:/external/cgr/pdfs:rw" + labels: + - "traefik.enable=true" + - "traefik.http.routers.airlfow.rule=Host(`airflow.controlciudadanopy.org`)" + - "traefik.http.routers.airlfow.entrypoints=websecure" + - "traefik.http.routers.airlfow.tls.certresolver=leresolver" + - "traefik.http.services.airlfow.loadbalancer.server.port=8080" + networks: + - airflow + #- proxy + + +networks: + airflow: + #proxy: + #external: true diff --git a/scripts/python/airflow/Dockerfile b/scripts/python/airflow/Dockerfile index 949c078..08a3673 100644 --- a/scripts/python/airflow/Dockerfile +++ b/scripts/python/airflow/Dockerfile @@ -1,7 +1,6 @@ -FROM puckel/docker-airflow:1.10.9 +FROM bitnami/airflow:2.0.0 MAINTAINER Arturo Volpe -COPY ./dags/ /usr/local/airflow/dags +COPY ./dags/ /opt/bitnami/airflow/dags COPY ./requirements.txt /requirements.txt -CMD ["webserver"]