-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathcovid19israel.source-spec.yaml
137 lines (125 loc) · 5.19 KB
/
covid19israel.source-spec.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
#
#
# This file contains pipeline steps which run from the COVID19-ISRAEL private repository
#
# You can add additional steps using the same format
#
# Each step should have a unique ID
#
# Available attributes for each step:
#
# dependencies:
#
# list of other step ids which must run before this step
# you can refer to other steps in this file
# or to the special steps:
# corona_data_collector - runs every day at 02:00
# github_pull_covid19_israel - will cause your pipeline to run only if there was a change in the COVID19-ISRAEL repository
#
#
# module:
#
# module name in COVID19-ISRAEL repository
# example of how a module is run from COVID19-ISRAEL repository:
# python3 -m src.utils.get_raw_Data
#
#
# args:
#
# list of additional args to pass to the module invocation
# the args are appended to the python3 command
# example of how a module is run from COVID19-ISRAEL repository:
# python3 -m src.utils.lamas_features --if-not-exists
#
#
# external_sharing_packages:
#
# list of output packages to share to external sources (only if the step completes successfully).
# some fields can contain python format values from package metadata, this includes all fields in the external sharing package json
# as well as the following fields: COVID19_ISRAEL_GITHUB_SHA1
# each item in the external_sharing_packages list can have the following attributes:
# package_path: path to json file in COVID19-ISRAEL repository which was generated using external_sharing.save_package
# publish_targets: list of targets to publish the package to, each item has the following attributes:
# github_repo: repository to publish the data to
# deploy_key: id of deploy key with write access to the repository
# files: list of files:
# key: key from the external sharing package
# value: target path in the repository to save the file into.
# the value can contain values from the package metadata in standard python format e.g. {metadata_attribute_name}
# files_foreach: allows to get a dynamic list of files form the package:
# key: name of value from the package meteadata which is a list of strings
# value: object with the following key/values:
# key: key from package, may include {foreach_value} replaced by an item from the list as well as other package metadata
# value: target path, may include {foreach_value} as well as other package metadata
# file_contents: allows to create files with dynamic content based on package metadata
# key: target path in the repository to save the file content into
# value: the file content, can include package metadata in standard python format e.g. {metadata_attribute_name}
#
#
# skip-failures:
#
# if set and is true - if pipeline fails, the dashboard will show it as a success and dependant pipelines will run
# at the end of all pipeline steps, there will be an additional step which will show failure with details of which pipelines failed
#
#
# run-once-daily:
#
# if set and is true - before running the step it will check the last runs for this step, if there was a successful run for current day -
# it will not run again. In this case the step will immediately fail.
get_raw_data:
dependencies:
- corona_data_collector
- github_pull_covid19_israel
module: src.utils.get_raw_data
preprocess_raw_data:
dependencies:
- get_raw_data
module: src.utils.preprocess_raw_data
lamas_features:
dependencies:
- preprocess_raw_data
module: src.utils.lamas_features
args: ["--if-not-exists"]
idf_report_cities_by_week:
dependencies:
- lamas_features
module: src.idf_report.cities_by_week
external_sharing_packages:
- package_path: out/bayesian/idf/cities_and_neighborhoods_by_week_datapackage.json
publish_targets:
- github_repo: hrossman/Covid19-Survey
deploy_key: hrossman_covid19_survey
files:
cities_csv: "aggregated_data/cities_by_week.csv"
latest_ts: "aggregated_data/ts.txt"
neighborhoods_csv: "aggregated_data/neighborhoods_by_week.csv"
skip-failures: true
maps_generate_daily_summary:
dependencies:
- idf_report_cities_by_week
module: src.utils.maps.generate_daily_summary
external_sharing_packages:
- package_path: out/external_sharing/HASADNA/datapackage.json
publish_targets:
- github_repo: hasadna/avid-covider-raw-data
deploy_key: hasadna_avid_covider_raw_data
files:
daily_summary: "input/{POSTERIOR_DATE}.csv"
all_dates: "input/all_dates.csv"
latest_ts: "input/ts.txt"
file_contents:
"input/commit_id.txt": "{COVID19_ISRAEL_GITHUB_SHA1}"
skip-failures: true
idf_report:
dependencies:
- idf_report_cities_by_week
module: src.idf_report.idf_report
external_sharing_packages:
- package_path: out/bayesian/idf/datapackage.json
publish_targets:
- github_repo: hrossman/Covid19-Survey
deploy_key: hrossman_covid19_survey
files_foreach:
min_per_region_lst:
"min_per_region_{foreach_value}_html": "aggregated_data/min_per_region_{foreach_value}.html"
skip-failures: true