From b9b614133f8254506844b214e594b6a4bf3a12f7 Mon Sep 17 00:00:00 2001 From: Arturo Volpe Date: Wed, 20 Jan 2021 18:36:25 -0300 Subject: [PATCH 1/3] feat: Update to airflow 2 Signed-off-by: Arturo Volpe --- infra/airflow/.env | 11 ++++ infra/airflow/.gitignore | 1 + infra/airflow/data/.gitignore | 0 infra/airflow/docker-compose.yml | 100 ++++++++++++++++++++----------- 4 files changed, 78 insertions(+), 34 deletions(-) create mode 100644 infra/airflow/.gitignore create mode 100644 infra/airflow/data/.gitignore diff --git a/infra/airflow/.env b/infra/airflow/.env index e69de29..50fd4b5 100644 --- a/infra/airflow/.env +++ b/infra/airflow/.env @@ -0,0 +1,11 @@ +POSTGRESQL_DATABASE=airflow +POSTGRESQL_USERNAME=airflow +POSTGRESQL_PASSWORD=airflow + + +AIRFLOW_FERNET_KEY== +AIRFLOW_EXECUTOR=CeleryExecutor +AIRFLOW_DATABASE_NAME=airflow +AIRFLOW_DATABASE_USERNAME=airflow +AIRFLOW_DATABASE_PASSWORD=airflow +AIRFLOW_LOAD_EXAMPLES=yes diff --git a/infra/airflow/.gitignore b/infra/airflow/.gitignore new file mode 100644 index 0000000..1269488 --- /dev/null +++ b/infra/airflow/.gitignore @@ -0,0 +1 @@ +data diff --git a/infra/airflow/data/.gitignore b/infra/airflow/data/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/infra/airflow/docker-compose.yml b/infra/airflow/docker-compose.yml index e13572d..00db222 100644 --- a/infra/airflow/docker-compose.yml +++ b/infra/airflow/docker-compose.yml @@ -1,40 +1,72 @@ -# Based on https://github.com/puckel/docker-airflow/blob/master/docker-compose-LocalExecutor.yml +version: '2' -version: '3.7' services: - postgres: - image: postgres:9.6 - environment: - - POSTGRES_USER=airflow - - POSTGRES_PASSWORD=airflow - - POSTGRES_DB=airflow - logging: - options: - max-size: 10m - max-file: "3" - - webserver: - image: docker.pkg.github.com/instidea/controlciudadano/dags:latest -# build: ../../scripts/python/airflow/ - restart: always - depends_on: - - postgres + + postgresql: + image: bitnami/postgresql:12.5.0 + env_file: + - .env + volumes: + - ./data:/bitnami/postgresql + networks: + - airflow + + redis: + image: bitnami/redis:5.0.10 + env_file: + - .env environment: - - LOAD_EX=n - - EXECUTOR=Local - - AIRFLOW__WEBSERVER__AUTHENTICATE=True - - AIRFLOW__WEBSERVER__AUTH_BACKEND=airflow.contrib.auth.backends.password_auth + - ALLOW_EMPTY_PASSWORD=yes + volumes: + - ./data:/bitnami + networks: + - airflow + + airflow-worker: + image: bitnami/airflow-worker:2.0.0 + env_file: + - .env + volumes: + - ./data:/bitnami +# - "/opt/etl:/external/:rw" +# - "/home/cdssa/services/public_data/contraloria/declaraciones:/external/cgr/pdfs:rw" + networks: + - airflow + + airflow-scheduler: + image: bitnami/airflow-scheduler:2.0.0 + env_file: + - .env + volumes: + - ./data:/bitnami +# - "/opt/etl:/external/:rw" +# - "/home/cdssa/services/public_data/contraloria/declaraciones:/external/cgr/pdfs:rw" + networks: + - airflow + + airflow: + build: ../../scripts/python/airflow/ + #image: docker.pkg.github.com/instidea/controlciudadano/dags:latest env_file: - .env - logging: - options: - max-size: 10m - max-file: "3" ports: - - "48080:8080" - command: webserver - healthcheck: - test: [ "CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]" ] - interval: 30s - timeout: 30s - retries: 3 + - '8080:8080' + volumes: + - ./data:/bitnami +# - "/opt/etl:/external/:rw" +# - "/home/cdssa/services/public_data/contraloria/declaraciones:/external/cgr/pdfs:rw" + labels: + - "traefik.enable=true" + - "traefik.http.routers.airlfow.rule=Host(`airflow.controlciudadanopy.org`)" + - "traefik.http.routers.airlfow.entrypoints=websecure" + - "traefik.http.routers.airlfow.tls.certresolver=leresolver" + - "traefik.http.services.airlfow.loadbalancer.server.port=8080" + networks: + - airflow + #- proxy + + +networks: + airflow: + #proxy: + #external: true From f0bc17db6e2350e24fedf34c17f2772432219907 Mon Sep 17 00:00:00 2001 From: Arturo Volpe Date: Wed, 20 Jan 2021 18:49:59 -0300 Subject: [PATCH 2/3] feat: Add default user Signed-off-by: Arturo Volpe --- infra/airflow/.env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra/airflow/.env b/infra/airflow/.env index 50fd4b5..a4fd63e 100644 --- a/infra/airflow/.env +++ b/infra/airflow/.env @@ -8,4 +8,7 @@ AIRFLOW_EXECUTOR=CeleryExecutor AIRFLOW_DATABASE_NAME=airflow AIRFLOW_DATABASE_USERNAME=airflow AIRFLOW_DATABASE_PASSWORD=airflow -AIRFLOW_LOAD_EXAMPLES=yes +AIRFLOW_LOAD_EXAMPLES=no +AIRFLOW_PASSWORD=bitnami123 +AIRFLOW_USERNAME=user +AIRFLOW_EMAIL=user@example.com From 731fe58b2e0281dc95201d4ac2ec29e267ef8627 Mon Sep 17 00:00:00 2001 From: Arturo Volpe Date: Wed, 20 Jan 2021 18:50:33 -0300 Subject: [PATCH 3/3] feat: Use bitnami airflow image for airflow Signed-off-by: Arturo Volpe --- scripts/python/airflow/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/python/airflow/Dockerfile b/scripts/python/airflow/Dockerfile index 949c078..08a3673 100644 --- a/scripts/python/airflow/Dockerfile +++ b/scripts/python/airflow/Dockerfile @@ -1,7 +1,6 @@ -FROM puckel/docker-airflow:1.10.9 +FROM bitnami/airflow:2.0.0 MAINTAINER Arturo Volpe -COPY ./dags/ /usr/local/airflow/dags +COPY ./dags/ /opt/bitnami/airflow/dags COPY ./requirements.txt /requirements.txt -CMD ["webserver"]