pipeline-spec.yaml

github_pull_covid19_israel:
  schedule:
    crontab: "* * * * *"
  pipeline:
    - flow: avid_covider_pipelines.github_pull_covid19_israel
      parameters:
        dump_to_path: data/github_pull_covid19_israel
        change-run-covid: true


covid19_israel_files_list:
  schedule:
    crontab: "* * * * *"
  pipeline:
    - flow: avid_covider_pipelines.covid19_israel_files_list
      parameters:
        dump_to_path: data/covid19_israel_files_list


covid19_israel_files_zip:
  schedule:
    crontab: "0 4 * * *"
  pipeline:
    - flow: avid_covider_pipelines.covid19_israel_files_zip
      parameters:
        dump_to_path: data/covid19_israel_files_zip


corona_data_collector:
  schedule:
    crontab: "0 2 * * *"
  pipeline:
    - flow: corona_data_collector.download_gdrive_data
      parameters:
#        limit_rows: 200
        files_dump_to_path: data/corona_data_collector/gdrive_data
        # COVID19 WS > Data > Analysis Raw Data
        google_drive_csv_folder_id: 1pzAyk-uXy__bt1tCX4rpTiPZNmrehTOz
        # the source id (google / hebrew_google..) should match the sources in load_from_db file_sources
        file_sources:
          COVID-19-English.csv: google
          COVID-19-Russian.csv: google
          COVID-19-Hebrew.csv: hebrew_google
    - flow: corona_data_collector.load_from_db
#      parameters:
#        limit_rows: 200
#        where: "id > 600000 and id < 622250"
    - flow: corona_data_collector.add_gps_coordinates
      parameters:
        source_fields: {
          "db": {
            "street": "street",
            "city_town": "city",
          },
          "google": {
            "Street": "street",
            "Город проживания": "street",
            "City": "city",
            "Улица": "city",
          },
          "hebrew_google": {
            "עיר / ישוב מגורים": "city",
            "עיר / יישוב מגורים": "city",
            "רחוב מגורים": "street",
          }
        }
        workplace_source_fields: {
          "db": {
            "routine_workplace_street": "street",
            "routine_workplace_city_town": "city"
          }
        }
        dump_to_path: data/corona_data_collector/with_gps_data
        # gps_data: data/corona_data_collector/gps_data.json
        gps_datapackage_path: data/corona_data_collector/gps_data_cache
    - flow: corona_data_collector.export_corona_bot_answers
      parameters:
        destination_output: data/corona_data_collector/corona_bot_answers
    - flow: corona_data_collector.export_corona_bot_answers
      parameters:
        unsupported: true
        destination_output: data/corona_data_collector/corona_bot_answers_unsupported