-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathMakefile
193 lines (151 loc) · 7.37 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#! /usr/bin/make -f
SHELL:=/bin/bash
include appKey.mk
# appKey.mk holds your et.water.ca.gov appKey
# appKey:=[Add Here or on command line, make api_key=foobar ]
targets:=esparto fair_oaks bryte davis winters dixon \
lodi_west twitchell_island concord brentwood \
manteca tracy pleasanton modesto hastings_east
esparto.id:=196
fair_oaks.id:=131
bryte.id:=155
davis.id:=6
winters.id:=139
dixon.id:=121
lodi_west.id:=166
twitchell_island.id:=140
concord.id:=170
brentwood.id:=47
manteca.id:=70
tracy.id:=167
pleasanton.id:=191
modesto.id:=71
hastings_east.id:=212
empty:=
sp:=${empty} ${empty}
comma:=,
target_ids := $(foreach t,${targets},${$t.id})
target_ids := $(subst ${sp},|,${target_ids})
water-years:=2015 2016
#start.2015:=2014-10-01
# 2016 was a leap year
start.2016:=$(shell date --date='2016-10-01 - 1 year' +%Y-%m-%d)
dates.2016:=$(shell for i in `seq 0 365`; do date --date="${start.2016} + $$i days" +%Y/%m/%d; done)
months.2016:=$(shell declare -A mo; for i in ${dates.2016}; do m=$${i%/??}; mo[$$m]=1; done; echo $${!mo[@]})
start.2015:=$(shell date --date='2015-10-01 - 1 year' +%Y-%m-%d)
dates.2015:=$(shell for i in `seq 0 364`; do date --date="${start.2015} + $$i days" +%Y/%m/%d; done)
months.2015:=$(shell declare -A mo; for i in ${dates.2015}; do m=$${i%/??}; mo[$$m]=1; done; echo $${!mo[@]})
start.2014:=2014-09-01
dates.2014:=$(shell for i in `seq 0 30`; do date --date="${start.2014} + $$i days" +%Y/%m/%d; done)
months.2014:=$(shell declare -A mo; for i in ${dates.2014}; do m=$${i%/??}; mo[$$m]=1; done; echo $${!mo[@]})
sed.json:=sed -e "s/^\(.\)/.\u\1/" -e "s/\-\(.\)/\u\1/g"
items.daily:=day-air-tmp-min day-air-tmp-max day-air-tmp-avg day-dew-pnt day-eto day-asce-eto\
day-precip day-sol-rad-avg day-sol-rad-net day-wind-spd-avg day-vap-pres-max day-vap-pres-min
items.daily.json:=$(shell for i in ${items.daily}; do echo $$i | ${sed.json}; done )
items.daily.val:=$(subst ${sp},${comma},$(patsubst %,%.Value,${items.daily.json}))
items.daily.qc:=$(subst ${sp},${comma},$(patsubst %,%.Qc,${items.daily.json}))
items.daily.row:=.Station,.Date,${items.daily.val},${items.daily.qc}
items.daily.header:=$(subst .,,${items.daily.row})
items.hourly:=hly-air-tmp hly-dew-pnt hly-eto hly-net-rad hly-asce-eto hly-precip\
hly-rel-hum hly-res-wind hly-soil-tmp hly-sol-rad hly-vap-pres hly-wind-dir hly-wind-spd
items.hourly.json:=$(shell for i in ${items.hourly}; do echo $$i | ${sed.json}; done )
items.hourly.val:=$(subst ${sp},${comma},$(patsubst %,%.Value,${items.hourly.json}))
items.hourly.qc:=$(subst ${sp},${comma},$(patsubst %,%.Qc,${items.hourly.json}))
items.hourly.row:=.Station,.Date,.Hour,${items.hourly.val},${items.hourly.qc}
items.hourly.header:=$(subst .,,${items.hourly.row})
items.cimis:=day_air_tmp_min,day_air_tmp_min_qc,day_air_tmp_max,day_air_tmp_max_qc,day_wind_spd_avg,day_wind_spd_avg_qc,day_rel_hum_max,day_rel_hum_max_qc,day_dew_pnt,day_dew_pnt_qc'
INFO:
@echo ${items.daily.row}
@echo items.daily.qc: ${items.daily.qc}
@echo "target_ids: ${target_ids}"
@echo "dates: ${dates.2015}"
@echo "months: ${months.2015}"
@echo "hourly: ${items.hourly.json}"
@echo "start of wy16: ${start.2016}"
.PHONY:csv
csv: csv.cimis csv.daily csv.hourly
define get_target
$(warning-no get_target $1 $2)
json.daily::daily_station/$1.wy/$2.json
csv.daily::daily_station/$1.wy/$2.csv
json.hourly:hourly_station/$1.wy/$2.json
csv.hourly:hourly_station/$1.wy/$2.csv
csv.cimis::cimis/$1.wy/$2.csv
daily_station/$1.wy/$2.json:
[[ -d daily_station/$1.wy ]] || mkdir -p daily_station/$1.wy
http --timeout=60 GET http://et.water.ca.gov/api/data \
appKey==${appKey} targets==${$2.id} \
startDate==${start.$1} endDate==$1-09-30 \
unitOfMeasure==M dataItems==$(subst ${sp},${comma},${items.daily}) > $$@
daily_station/$1.wy/$2.csv:daily_station/$1.wy/$2.json
echo ${items.daily.header} > $$@
jq -r ".Data.Providers[0].Records[] | [${items.daily.row}] | @csv" < $$< >> $$@
hourly_station/$1.wy/$2.json:
[[ -d hourly_station/$1.wy ]] || mkdir -p hourly_station/$1.wy
http --timeout=240 GET http://et.water.ca.gov/api/data \
appKey==${appKey} targets==${$2.id} \
startDate==${start.$1} endDate==$1-09-30 \
unitOfMeasure==M dataItems==$(subst ${sp},${comma},${items.hourly}) > $$@
hourly_station/$1.wy/$2.csv:hourly_station/$1.wy/$2.json
echo ${items.hourly.header} > $$@
jq -r ".Data.Providers[0].Records[] | [${items.hourly.row}] | @csv" < $$< >> $$@
cimis/$1.wy/$2.csv:cimis/$1.wy/station.csv
head -n 1 $$< > $$@
grep -P '^([\d\-.]+,){3}${$2.id},' < $$< >> $$@
clean-csv.cimis::
rm -f cimis/$1/$2.csv
endef
$(foreach w,${water-years},$(foreach t,${targets},$(eval $(call get_target,$w,$t))))
clean-csv.cimis::
rm -f cimis/2015.wy/station.csv cimis/2014.wy_partial/station.csv cimis/2016.wy/station.csv
cimis/2016.wy/station.csv:
[[ -d cimis/2016.wy ]] || mkdir -p cimis/2016.wy
echo 'x,y,z,station_id,date,day_air_tmp_min,day_air_tmp_min_qc,day_air_tmp_max,day_air_tmp_max_qc,day_wind_spd_avg,day_wind_spd_avg_qc,day_rel_hum_max,day_rel_hum_max_qc,day_dew_pnt,day_dew_pnt_qc' > $@;
for i in `seq 0 365`; do \
ymd=`date --date="2015-10-01 + $$i days" +%Y/%m/%d`; \
echo $$ymd; \
http http://cimis.casil.ucdavis.edu/cimis/$$ymd/station.csv | tail -n +1 >> $@; \
done
cimis/2015.wy/station.csv:
[[ -d cimis/2015.wy ]] || mkdir -p cimis/2015.wy
echo 'x,y,z,station_id,date,day_air_tmp_min,day_air_tmp_min_qc,day_air_tmp_max,day_air_tmp_max_qc,day_wind_spd_avg,day_wind_spd_avg_qc,day_rel_hum_max,day_rel_hum_max_qc,day_dew_pnt,day_dew_pnt_qc' > $@;
for i in `seq 0 364`; do \
ymd=`date --date="2014-10-01 + $$i days" +%Y/%m/%d`; \
echo $$ymd; \
http http://cimis.casil.ucdavis.edu/cimis/$$ymd/station.csv | tail -n +1 >> $@; \
done
cimis/2014.wy/station.csv:
[[ -d cimis/2014.wy ]] || mkdir -p cimis/2014.wy
echo 'x,y,z,station_id,date,day_air_tmp_min,day_air_tmp_min_qc,day_air_tmp_max,day_air_tmp_max_qc,day_wind_spd_avg,day_wind_spd_avg_qc,day_rel_hum_max,day_rel_hum_max_qc,day_dew_pnt,day_dew_pnt_qc' > $@;
for i in `seq 0 29`; do \
ymd=`date --date="2014-09-01 + $$i days" +%Y/%m/%d`; \
echo $$ymd; \
http http://cimis.casil.ucdavis.edu/cimis/$$ymd/station.csv | tail -n +1 >> $@; \
done
rast.cimis:=ETo K Rnl Rs Rso Tdew Tn Tx U2
define cimis_y
cimis.yearly:: cimis/$1.wy/$2.tif
cimis/$1.wy/$2.tif:
gdal_merge.py -separate -o cimis/$1.wy/$2.tif cimis/$(shell let y=$1-1; echo $$y)/1?/$2.tif cimis/$1/0?/$2.tif
endef
define cimis_ym
cimis.monthly:: cimis/$1/$2.tif
cimis/$1/$2.tif:
gdal_merge.py -separate -o cimis/$1/$2.tif cimis/$1/??/$2.tif
endef
define cimis
cimis.daily:: $3
$3:: cimis/$1/$2/$3.tif
cimis/$1/$3.tif:: cimis/$1/$2/$3.tif
cimis/$1/$2/$3.tif:
[[ -d cimis/$1/$2 ]] || mkdir -p cimis/$1/$2
http http://cimis.casil.ucdavis.edu/cimis/$1/$2/$3.asc.gz > cimis/$1/$2/$3.asc
gdal_translate -a_srs EPSG:3310 -projwin -164000 68000 -108000 -44000 cimis/$1/$2/$3.asc cimis/$1/$2/$3.tif
rm cimis/$1/$2/$3.asc
endef
$(foreach d,${dates.2015},$(foreach r,${rast.cimis},$(eval $(call cimis,$(dir $d),$(notdir $d),$r))))
$(foreach m,${months.2015},$(foreach r,${rast.cimis},$(eval $(call cimis_ym,$m,$r))))
$(foreach y,${water-years},$(foreach r,${rast.cimis},$(eval $(call cimis_y,$y,$r))))
$(foreach d,${dates.2016},$(foreach r,${rast.cimis},$(eval $(call cimis,$(dir $d),$(notdir $d),$r))))
$(foreach m,${months.2016},$(foreach r,${rast.cimis},$(eval $(call cimis_ym,$m,$r))))
$(foreach y,2016,$(foreach r,${rast.cimis},$(eval $(call cimis_y,$y,$r))))