-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathsetup-NT3.sh
41 lines (37 loc) · 930 Bytes
/
setup-NT3.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# SETUP NT3
# Shrink the train/test data by 90% (to 10% of original size)
# Only shrinks if the data already exists-
# thus, the first run after auto-download will be slow.
DATA_PILOT1=$BENCHMARKS/Data/Pilot1
CSV=$DATA_PILOT1/nt_test2.csv
if [[ -f $CSV ]]
then
SIZE=$( stat --format "%s" $CSV )
if (( SIZE > 100000000 )) # 100 MB
then
echo "Shrinking test CSV..."
T=$( mktemp --suffix=.csv --tmpdir=$DATA_PILOT1 nt_test-XXX )
(
set -x
head --lines=28 $CSV > $T
mv --backup --force $CSV $CSV.orig
mv $T $CSV
)
fi
fi
CSV=$DATA_PILOT1/nt_train2.csv
if [[ -f $CSV ]]
then
SIZE=$( stat --format "%s" $CSV )
if (( SIZE > 500000000 )) # 500 MB
then
echo "Shrinking train CSV..."
T=$( mktemp --suffix=.csv --tmpdir=$DATA_PILOT1 nt_train-XXX )
(
set -x
head --lines=112 $CSV > $T
mv --backup --force $CSV $CSV.orig
mv $T $CSV
)
fi
fi