[pull] main from calumrussell:main

.gitignore

@@ -2,3 +2,8 @@
 *.csv
 /.vscode
 Cargo.lock
+data/venv/
+rotala-python/venv/
+test_data/
+*__pycache__*
+.env

Cargo.toml

@@ -1,7 +1,4 @@
 [workspace]
 
 resolver = "2"
-members = [
-    "rotala",
-    "example_clients/alator"
-]
+members = ["rotala", "rotala-client", "rotala-http"]

Dockerfile

@@ -0,0 +1,5 @@
+FROM rust:1.83-bullseye
+
+COPY . .
+
+RUN cargo build --release

data/hl.py

@@ -0,0 +1,92 @@
+import argparse
+import os
+import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
+import lz4framed
+import datetime
+
+
+def path_builder(date, hour, coin):
+    return f"market_data/{date}/{hour}/l2Book/{coin}.lz4"
+
+
+def parse_date(string):
+    return (string[0:4], string[4:6])
+
+
+def zero_padding(number):
+    if number < 10:
+        return "0" + str(number)
+    return str(number)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="HL Data Fetcher",
+        description="Downloads data from HL, unzips and places into directory",
+    )
+
+    parser.add_argument("-o", "--outdir")
+    parser.add_argument("-c", "--coin")
+    parser.add_argument("-s", "--start")
+
+    args = parser.parse_args()
+
+    max_year = 2024
+    hours = list(range(0, 24))
+    days = list(range(1, 32))
+    months = list(range(1, 13))
+    client = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+    bucket_name = "hyperliquid-archive"
+    now = datetime.datetime.now()
+
+    os.makedirs(f"{args.outdir}/{args.coin}", exist_ok=True)
+
+    (start_year, start_month) = parse_date(args.start)
+    for year in range(int(start_year), max_year + 1):
+        iter_start = int(start_month) if int(year) == int(start_year) else 1
+
+        for month in range(iter_start, 13):
+            chunks = []
+            file_path = f"{args.outdir}/{args.coin}/{month}"
+            if os.path.exists(file_path):
+                continue
+
+            for day in days:
+                try:
+                    then = datetime.datetime(year, month, day)
+                except ValueError:
+                    # Occurs if date isn't valid
+                    continue
+
+                if then > now:
+                    print("Reached the present")
+                    print(file_path)
+                    with open(f"{file_path}", "w") as f:
+                        for chunk in chunks:
+                            f.write(chunk)
+                    exit(1)
+
+                for hour in hours:
+                    date_string = str(year) + zero_padding(month) + zero_padding(day)
+                    key = path_builder(date_string, hour, args.coin)
+
+                    try:
+                        response = client.get_object(
+                            Bucket=bucket_name,
+                            Key=key,
+                        )
+                        contents = response["Body"].read()
+                    except Exception:
+                        print(f"Didn't find - {key}")
+                        continue
+
+                    uncompressed = lz4framed.decompress(contents)
+                    print(f"Took - {key}")
+                    chunks.append(uncompressed.decode("utf-8"))
+
+            print(file_path)
+            with open(f"{file_path}", "w") as f:
+                for chunk in chunks:
+                    f.write(chunk)

data/requirements.txt

@@ -0,0 +1,2 @@
+boto3==1.35.29
+py-lz4framed==0.14.0

docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3.9'
+
+services:
+
+  uist_v2:
+    build: .
+    ports:
+      - 0.0.0.0:3000:${PORT}
+    command: cargo run --bin uist_server_v2 0.0.0.0 ${PORT} ${DB_HOST} ${DB_USER} ${DB_PWD} ${DB_NAME

example_clients/alator/Cargo.toml → rotala-client/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "alator"
+name = "rotala-client"
 version = "0.4.1"
 authors = ["Calum Russell <[email protected]>"]
 edition = "2021"
@@ -14,22 +14,20 @@ time = { version = "0.3.17", features = ["macros", "parsing"] }
 rand = "0.8.4"
 rand_distr = "0.4.1"
 log = "0.4.17"
-env_logger = "0.11.0"
-pyo3 = { version = "0.22.1", optional = true }
-async-trait = "0.1.73"
+pyo3 = { version = "0.23.2", optional = true }
 tokio = { version = "1.32.0", features = ["full"] }
 futures = "0.3.28"
-rotala = { path = "../../rotala/" }
+rotala = { path = "../rotala/" }
+rotala-http = { path = "../rotala-http/" }
+anyhow = "1.0.86"
+reqwest = { version = "0.12.5", features = ["blocking", "json"] }
+env_logger = "0.11.5"
 
 [dev-dependencies]
 reqwest = { version = "0.12.5", features = ["blocking"] }
-zip = "0.6.2"
+zip = "2.2.0"
 csv = "1.1.6"
-criterion = { version = "0.5.1", features = ["async_tokio"] }
 
-[lib]
-bench = false
-
-[[bench]]
-name = "sim_benchmark"
-harness = false
+[[bin]]
+name = "uist_client_test"
+path = "./src/bin/uist_client_test.rs"

rotala/src/bin/uist_client_test.rs → rotala-client/src/bin/uist_client_test.rs

@@ -1,7 +1,8 @@
 use std::io::Result;
 
 use rotala::exchange::uist_v1::Order;
-use rotala::http::uist_v1::{Client, HttpClient};
+use rotala_client::client::uist_v1::HttpClient;
+use rotala_http::http::uist_v1::Client;
 
 #[tokio::main]
 async fn main() -> Result<()> {