Fix endlesslop in case of empty response

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 160

.pre-commit-config.yaml

@@ -0,0 +1,4 @@
+- repo: https://gitlab.com/pycqa/flake8
+  rev: ''  # pick a git hash / tag to point to
+  hooks:
+    - id: flake8

README.md

@@ -37,6 +37,19 @@ Black Oak requires Python 3.7 or higher.
 pip install -r requirements.txt
 ```
 
+### Setup Development Environment
+#### Prerequisites
+- [Direnv](https://direnv.net/)
+  - `brew install direnv`
+- [Miniconda](https://docs.conda.io/en/latest/miniconda.html)
+  - `brew cask install miniconda`
+
+#### Installation
+The .envrc takes care of installing the conda environment, dependencies and pre-commit
+```bash
+direnv allow
+```
+
 #### Run Black Oak
 ```bash
 ./bin/run_ohlcv_data_fetcher.sh config.toml
@@ -65,16 +78,16 @@ ratelimit = false
 [exchanges]
 
     [exchanges.bitfinex]
-    filter_symbols = [] # passing an empty list will pull all available symbols!!!
-    filter_resolutions = []    # passing an empty list will pull all available resolutions
+    filter_symbols = []     # passing an empty list will pull all available symbols!!!
+    filter_resolutions = [] # passing an empty list will pull all available resolutions
     since = "2017-01-01 00:00:00"
     until = "2019-06-26 00:00:00"
     limit = 5000
 
     [exchanges.bitmex]
     filter_symbols = ["BTC/USD", "ETH/USD"]
     filter_resolutions = ["1m", "1h", "1d"]
-    since = "2017-01-01 00:00:00"
+    since = "" # leave since empty to fetch data from the first date available
     until = "2019-06-26 00:00:00"
     limit = 750
 ```

config.toml

@@ -1,6 +1,6 @@
 [settings]
 
-debug = true
+debug = false
 
 # we only implemented file as a backend for now
 persistence = "file"
@@ -37,7 +37,7 @@ default_limit = 200
 #    filter_resolutions = ["1d"]
 #    since = "2017-01-01 00:00:00"
 #    until = "2019-06-26 00:00:00"
-#    limit = 750
+#    limit = 1000
 
 #    [exchanges.okex3]
 #    filter_symbols = ["BTC/USDT"]
@@ -51,7 +51,7 @@ default_limit = 200
 #    filter_resolutions = []
 #    since = "2019-07-18 00:00:00"
 #    until = "2019-08-18 00:00:00"
-#    limit = 750
+#    limit = 1000
 
 #    [exchanges.kraken]
 #    filter_symbols = [

ohlcv/generate_bitmex_futures_symbols.py

@@ -0,0 +1,44 @@
+# A simple
+
+import datetime
+
+# available future symbols
+futures = {
+    'ADA': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 17},
+    'BCH': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 17},
+    'EOS': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 17},
+    'ETH': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 16},
+    'LTC': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 19},
+    'TRX': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 17},
+    'XBT': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 15},
+    'XRP': {'timeframes': ['1m', '5m', '1h', '1d'], 'since_year': 17},
+}
+
+# F = January
+# G = February
+# H = March
+# J = April
+# K = May
+# M = June
+# N = July
+# Q = August
+# U = September
+# V = October
+# X = November
+# Z = December
+month_codes = ['F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z']
+
+if __name__ == '__main__':
+    for symbol, futures_config in futures.items():
+
+        # current year as two digits
+        current_year = int(datetime.datetime.now().strftime('%y'))
+        start_year = futures_config['since_year']
+
+        while start_year <= current_year:
+
+            for month in month_codes:
+                print(f'https://www.bitmex.com/api/v1/trade/bucketed?symbol={symbol}{month}{start_year}&binSize=1d')
+
+            # set iteration to next year
+            start_year = start_year + 1

ohlcv/lib/data_fetcher.py

@@ -176,17 +176,15 @@ async def fetch_ohlvc_data(
                 exchange.headers = {'Connection': 'close'}
 
             # exit the loop if we fetched the whole time series
-            if since > until and until is not 0:
+            if since and since > until and until:
                 break
 
             try:
+                date = datetime.utcfromtimestamp(
+                    util.ms_timestamp_to_epoch_timestamp(since)) if since else 'no start date given'
                 logging.info(
-                    'Start fetching %s data points from %s for %s and timestamp %s (%s)',
-                    limit,
-                    exchange.id,
-                    symbol,
-                    since,
-                    datetime.utcfromtimestamp(util.ms_timestamp_to_epoch_timestamp(since))
+                    f'Start fetching {limit} data points from {exchange.id} for {symbol}'
+                    f'and timestamp {since} ({date})'
                 )
 
                 ohlcv_ts = pd.DataFrame(
@@ -211,11 +209,16 @@ async def fetch_ohlvc_data(
                     ccxt.DDoSProtection
             ) as error:
                 logging.error(
-                    f'Got an error {type(error).__name__} {error.args}. Will try to send the same Request again.',
+                    f'Got an error {type(error).__name__} {error.args}. Will try to send the same Request again.'
                 )
                 # skip current iteration and try again if we run into an exception
                 continue
 
+            # exit the loop in case we didn't receive any data for the current configuration
+            if len(ohlcv_ts) == 0:
+                logging.info(f'Didn\'t receive any data for {symbol}_{resolution} from {exchange.id}.')
+                break
+
             # write data frame to csv
             io.write_csv(
                 f'data/{file_name}',