-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathDataCleaner.pine
213 lines (189 loc) · 8.72 KB
/
DataCleaner.pine
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
// This source code is subject to the terms of the Mozilla Public License 2.0 at https://mozilla.org/MPL/2.0/
// © Electrified (electrifiedtrading)
// @version=5
// @description Functions for acquiring outlier levels and deriving a cleaned version of a series.
library("DataCleaner")
//////////////////////////////////////////////////
// @type A struct of the result of calculating the average and the standard deviation.
// @field average The average.
// @field stdev The standard deviation.
export type Deviation
float average
float stdev
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Gets the average and standard deviation for a given series.
// @param src The source series
// @param len The The number of bars to measure.
export calcDeviation(
series float src,
simple int len) =>
avg = ta.sma(src, len)
std = ta.stdev(src, len)
Deviation.new(avg, std)
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Gets the (standard deviation) outlier level for a given series.
// @param src The series to average and add a multiple of the standard deviation to.
// @param len The The number of bars to measure.
// @param level The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary.
// @returns The average of the series plus the multiple of the standard deviation.
export outlierLevel(
series float src,
simple int len,
simple float level) =>
if level == 0
runtime.error('Passing a level of (0) zero to "level" is simply the average.')
d = calcDeviation(src, len)
d.average + d.stdev * level
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Returns only values that are within the maximum deviation.
// @param src The series to filter results from.
// @param len The The number of bars to measure.
// @param maxDeviation The maximum deviation before considered an outlier.
export naOutliers(
series float src,
simple int len,
simple float maxDeviation = 4) =>
if maxDeviation == 0
runtime.error('Passing a level of (0) zero to "maxDeviation" is simply the average.')
prev = src[1] // ignore current in measurment as it could throw off result.
avg = ta.sma(prev, len)
dev = ta.stdev(prev, len) * maxDeviation
src > avg + dev or src < avg - dev ? na : src
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Returns the source value adjusted by its standard deviation.
// @param src The series to measure.
// @param len The number of bars to measure the standard deviation.
// @param maxDeviation The maximum deviation before considered an outlier.
// @param baseline The value considered to be at center. Typically zero.
export normalize(
series float src,
simple int len,
simple int maxDeviation = 4,
float baseline = 0) =>
cleaned = naOutliers(src, len, maxDeviation)
d = calcDeviation(cleaned, len)
src / (src >= baseline ? (d.average + d.stdev) : math.abs(d.average - d.stdev))
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Returns an array representing the result series with (outliers provided by the source) removed.
// @param src The source series to read from.
// @param result The result series.
// @param len The maximum size of the resultant array.
// @param maxDeviation The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary.
// @returns An array containing the cleaned series.
export cleanUsing(
series float src,
series float result,
simple int len,
simple float maxDeviation) =>
var cleaned = array.new_float()
if maxDeviation == 0
runtime.error('Cannot clean a series with a "maxDeviation" of (0) zero.')
cleaned
else
// Hold the previous value so it's not included in the set yet.
var raw = array.new_float()
var level = outlierLevel(src, len, maxDeviation)
// Is the value in bounds? Add it to the series.
if(maxDeviation > 0 and src < level or maxDeviation < 0 and src > level)
array.push(cleaned, result)
// Maxed out results?
if(array.size(cleaned) > len)
array.shift(cleaned)
level := outlierLevel(src, len, maxDeviation)
cleaned
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Returns an array representing the source series with outliers removed.
// @param src The source series to read from.
// @param len The maximum size of the resultant array.
// @param maxDeviation The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary.
// @returns An array containing the cleaned series.
export clean(
series float src,
simple int len,
simple float maxDeviation) =>
cleanUsing(src, src, len, maxDeviation)
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Returns an array representing the source array with outliers removed.
// @param src The source series to read from.
// @param maxDeviation The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary.
// @returns An array containing the cleaned series.
export cleanArray(
float[] src,
simple float maxDeviation) =>
if array.size(src)==0
src
else
avg = array.avg(src)
stdev = array.stdev(src)
cleaned = array.new_float()
last = array.size(src) - 1
level = stdev * maxDeviation
hi = avg + level
lo = avg - level
for i = 0 to last
v = array.get(src, i)
if lo < v and v < hi
array.push(cleaned, v)
cleaned
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Returns an array representing the source array with outliers removed.
// @param src The array to set outliers to N/A.
// @param maxDeviation The maximum deviation before considered an outlier.
// @returns True if there were any outliers; otherwise false.
export naArrayOutliers(
float[] src,
simple float maxDeviation) =>
if array.size(src)==0
false
else
avg = array.avg(src)
stdev = array.stdev(src)
last = array.size(src) - 1
level = stdev * maxDeviation
hi = avg + level
lo = avg - level
bool modified = false
for i = 0 to last
v = array.get(src, i)
if lo > v or v > hi
array.set(src, i, na)
modified := true
modified
//////////////////////////////////////////////////
//////////////////////////////////////////////////
// @function Gets the (standard deviation) outlier level for a given series after a single pass of removing any outliers.
// @param src The series to average and add a multiple of the standard deviation to.
// @param len The The number of bars to measure.
// @param level The positive or negative multiple of the standard deviation to apply to the average. A positive number will be the upper boundary and a negative number will be the lower boundary.
// @param maxDeviation The optional standard deviation level to use when cleaning the series. The default is the value of the provided level.
// @returns The average of the series plus the multiple of the standard deviation.
export outlierLevelAdjusted(
series float src,
simple int len,
simple float level,
simple float maxDeviation = 0)=>
if level == 0
runtime.error('Passing a level of (0) zero to "level" is simply the average.')
cleaned = clean(src, len, maxDeviation==0 ? level : maxDeviation)
avg = array.avg(cleaned)
std = array.stdev(cleaned)
avg + std * level
//////////////////////////////////////////////////
len = input.int(200, "Length")
maxDev = input.float(2, "Max Deviation", minval=1)
level = input.float(4, "Max Deviation After Cleaning", minval=1)
//plot(ta.atr(len), "ATR", color.red)
candleHeight = high - low
plot(candleHeight, "Candle Height", color.yellow)
before = outlierLevel(candleHeight, len, maxDev)
plot(before, "Deviation Before Adjustment", color.orange)
after = outlierLevelAdjusted(candleHeight, len, level)
plot(after, "Deviation After Adjustment", color.green)