From 1a5d585e00a7e61cca0811781815bc241576745a Mon Sep 17 00:00:00 2001 From: AJ <34186192+Jurshsmith@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:52:33 +0100 Subject: [PATCH 01/15] feat(publisher): Extend NATS publishing by storing payloads in S3 (#348) * feat(publisher): Publish streams to S3 * feat(publisher): add Localstack for testing s3 locally * feat(publisher): Ignore localstack data * feat(publisher): Fix build errors from integrating S3Client * feat(publisher): Ensure published objects can be streamed Also update tests * feat(publisher): Consistently hash static payload for s3 path * feat(publisher): Use testnet or mainnet buckets based on the env * feat(publisher): Distinct env vars for S3 --- .env.sample | 5 + .github/workflows/ci.yaml | 5 + .gitignore | 3 + Cargo.lock | 828 +- Cargo.toml | 2 + README.md | 2 +- benches/data-parser/benches/deserialize.rs | 1 - .../src/runners/runner_streamable.rs | 20 +- benches/nats-publisher/src/utils/nats.rs | 4 +- cluster/docker/docker-compose.yml | 21 + crates/fuel-data-parser/src/lib.rs | 37 +- crates/fuel-networks/Cargo.toml | 14 + crates/fuel-networks/src/lib.rs | 75 + crates/fuel-streams-core/Cargo.toml | 7 +- crates/fuel-streams-core/README.md | 15 +- crates/fuel-streams-core/src/lib.rs | 12 +- crates/fuel-streams-core/src/stream/error.rs | 3 + .../src/stream/stream_encoding.rs | 33 +- .../src/stream/stream_impl.rs | 265 +- crates/fuel-streams-core/src/types.rs | 4 +- crates/fuel-streams-publisher/Cargo.lock | 7553 ----------------- crates/fuel-streams-publisher/Cargo.toml | 1 + crates/fuel-streams-publisher/src/cli.rs | 9 - crates/fuel-streams-publisher/src/main.rs | 1 - .../src/publisher/fuel_streams.rs | 23 +- .../src/publisher/mod.rs | 29 +- .../fuel-streams-publisher/src/server/http.rs | 9 +- crates/fuel-streams-storage/Cargo.toml | 34 + crates/fuel-streams-storage/src/lib.rs | 7 + .../src/nats/error.rs | 0 .../src/nats/mod.rs | 1 + .../src/nats/nats_client.rs | 16 +- .../src/nats/nats_client_opts.rs | 90 +- .../src/nats/nats_namespace.rs | 6 +- .../src/nats/types.rs | 0 crates/fuel-streams-storage/src/s3/mod.rs | 5 + .../fuel-streams-storage/src/s3/s3_client.rs | 213 + .../src/s3/s3_client_opts.rs | 71 + crates/fuel-streams/README.md | 15 +- crates/fuel-streams/src/client/client_impl.rs | 44 +- crates/fuel-streams/src/client/error.rs | 6 +- crates/fuel-streams/src/client/types.rs | 2 +- crates/fuel-streams/src/lib.rs | 3 +- crates/fuel-streams/src/stream/mod.rs | 2 +- crates/fuel-streams/src/stream/stream_impl.rs | 122 +- examples/blocks.rs | 5 +- examples/inputs.rs | 5 +- examples/logs.rs | 5 +- examples/multiple-streams.rs | 32 +- examples/outputs.rs | 5 +- examples/receipts.rs | 5 +- examples/transactions.rs | 6 +- examples/utxos.rs | 5 +- scripts/run_publisher.sh | 2 +- tests/src/lib.rs | 40 +- tests/src/main.rs | 32 +- tests/tests/client.rs | 70 +- tests/tests/publisher.rs | 37 +- tests/tests/stream.rs | 57 +- 59 files changed, 1845 insertions(+), 8079 deletions(-) create mode 100644 crates/fuel-networks/Cargo.toml create mode 100644 crates/fuel-networks/src/lib.rs delete mode 100644 crates/fuel-streams-publisher/Cargo.lock create mode 100644 crates/fuel-streams-storage/Cargo.toml create mode 100644 crates/fuel-streams-storage/src/lib.rs rename crates/{fuel-streams-core => fuel-streams-storage}/src/nats/error.rs (100%) rename crates/{fuel-streams-core => fuel-streams-storage}/src/nats/mod.rs (96%) rename crates/{fuel-streams-core => fuel-streams-storage}/src/nats/nats_client.rs (85%) rename crates/{fuel-streams-core => fuel-streams-storage}/src/nats/nats_client_opts.rs (56%) rename crates/{fuel-streams-core => fuel-streams-storage}/src/nats/nats_namespace.rs (92%) rename crates/{fuel-streams-core => fuel-streams-storage}/src/nats/types.rs (100%) create mode 100644 crates/fuel-streams-storage/src/s3/mod.rs create mode 100644 crates/fuel-streams-storage/src/s3/s3_client.rs create mode 100644 crates/fuel-streams-storage/src/s3/s3_client_opts.rs diff --git a/.env.sample b/.env.sample index 0f57a6f3..65a2309a 100644 --- a/.env.sample +++ b/.env.sample @@ -3,6 +3,11 @@ KEYPAIR=generated-p2p-secret NATS_URL=nats://localhost:4222 NATS_ADMIN_PASS=generated-secret NATS_SYSTEM_PASS=generated-secret +AWS_ACCESS_KEY_ID=test +AWS_SECRET_ACCESS_KEY=test +AWS_REGION=us-east-1 +AWS_ENDPOINT_URL=http://localhost:4566 +AWS_S3_BUCKET_NAME=fuel-streams-testnet USE_ELASTIC_LOGGING=false USE_PUBLISHER_METRICS=true PUBLISHER_MAX_THREADS=16 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8fd6c99a..6b1a241b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -214,6 +214,11 @@ jobs: NATS_URL: nats://127.0.0.1:4222 NATS_ADMIN_PASS: secret NATS_PUBLIC_PASS: secret + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_REGION: us-east-1 + AWS_ENDPOINT_URL: http://localhost:4566 + AWS_S3_BUCKET_NAME: fuel-streams-test strategy: fail-fast: false matrix: diff --git a/.gitignore b/.gitignore index c3ed4a0a..b46e0b73 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ coverage/ docs/ **/**/charts/**.tgz values-publisher-secrets.yaml +values-publisher-env.yaml +localstack-data +.vscode diff --git a/Cargo.lock b/Cargo.lock index e1cc9d60..d21eef11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,9 +505,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.11" +version = "7.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba6d24703c5adc5ba9116901b92ee4e4c0643c01a56c4fd303f3818638d7449" +checksum = "10db7e8b2042f8d7ebcfebc482622411c23f88f3e9cd7fac74465b78fdab65f0" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -524,7 +524,6 @@ dependencies = [ "mime", "multer", "num-traits", - "once_cell", "pin-project-lite", "regex", "serde", @@ -538,9 +537,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.11" +version = "7.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94c2d176893486bd37cd1b6defadd999f7357bf5804e92f510c08bcf16c538f" +checksum = "ad560d871a344178c35568a15be1bbb40cbcaced57838bf2eb1f654802000df7" dependencies = [ "Inflector", "async-graphql-parser", @@ -555,9 +554,9 @@ dependencies = [ [[package]] name = "async-graphql-parser" -version = "7.0.11" +version = "7.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79272bdbf26af97866e149f05b2b546edb5c00e51b5f916289931ed233e208ad" +checksum = "1df338e3e6469f86cce1e2b0226644e9fd82ec04790e199f8dd06416632d89ea" dependencies = [ "async-graphql-value", "pest", @@ -567,9 +566,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.11" +version = "7.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1" +checksum = "d4cffd8bb84bc7895672c4e9b71d21e35526ffd645a29aedeed165a3f4a7ba9b" dependencies = [ "bytes", "indexmap 2.7.0", @@ -767,6 +766,389 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json 0.60.7", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring 0.17.8", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json 0.61.1", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac 0.12.1", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2 0.10.8", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json 0.61.1", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json 0.61.1", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json 0.61.1", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac 0.12.1", + "http 0.2.12", + "http 1.2.0", + "once_cell", + "p256 0.11.1", + "percent-encoding", + "ring 0.17.8", + "sha2 0.10.8", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2 0.10.8", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper 0.14.31", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.2.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.2.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.5.17" @@ -928,6 +1310,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" @@ -952,6 +1340,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -1200,6 +1598,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytestring" version = "1.4.0" @@ -1270,9 +1678,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -1326,9 +1734,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1584,18 +1992,18 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const_format" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c655d81ff1114fb0dcdea9225ea9f0cc712a6f8d189378e82bdf62a473a64b" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" dependencies = [ "const_format_proc_macros", ] [[package]] name = "const_format_proc_macros" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff1a44b93f47b1bac19a27932f5c591e43d1ba357ee4f61526c8a25603f0eb1" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" dependencies = [ "proc-macro2", "quote", @@ -1739,13 +2147,13 @@ checksum = "210fbe6f98594963b46cc980f126a9ede5db9a3848ca65b71303bebdb01afcd9" dependencies = [ "bip32", "cosmos-sdk-proto", - "ecdsa", + "ecdsa 0.16.9", "eyre", "k256", "rand_core", "serde", "serde_json", - "signature", + "signature 2.2.0", "subtle-encoding", "tendermint 0.40.0", "thiserror 1.0.69", @@ -1900,6 +2308,15 @@ dependencies = [ "wasmtime-types", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1984,6 +2401,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -2229,6 +2658,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.9" @@ -2434,18 +2873,30 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.9", "digest 0.10.7", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -2454,8 +2905,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -2482,7 +2933,7 @@ dependencies = [ "rand_core", "serde", "sha2 0.10.8", - "signature", + "signature 2.2.0", "subtle", "zeroize", ] @@ -2514,21 +2965,41 @@ dependencies = [ "void", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest 0.10.7", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest 0.10.7", - "ff", + "ff 0.13.0", "generic-array", - "group", - "pkcs8", + "group 0.13.0", + "pkcs8 0.10.2", "rand_core", - "sec1", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -2741,7 +3212,7 @@ dependencies = [ "cargo_metadata", "chrono", "const-hex", - "elliptic-curve", + "elliptic-curve 0.13.8", "ethabi", "generic-array", "k256", @@ -2859,9 +3330,19 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] [[package]] name = "ff" @@ -3534,11 +4015,11 @@ version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33548590131674e8f272a3e056be4dbaa1de7cb364eab2b17987cd5c0dc31cb0" dependencies = [ - "ecdsa", + "ecdsa 0.16.9", "ed25519-dalek", "fuel-types 0.56.0", "k256", - "p256", + "p256 0.13.2", "serde", "sha2 0.10.8", "zeroize", @@ -3552,12 +4033,12 @@ checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" dependencies = [ "coins-bip32", "coins-bip39", - "ecdsa", + "ecdsa 0.16.9", "ed25519-dalek", "fuel-types 0.58.2", "k256", "lazy_static", - "p256", + "p256 0.13.2", "rand", "secp256k1", "serde", @@ -3580,7 +4061,7 @@ dependencies = [ "serde_json", "strum 0.26.3", "strum_macros 0.26.4", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", ] @@ -3648,6 +4129,13 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "fuel-networks" +version = "0.0.13" +dependencies = [ + "clap 4.5.23", +] + [[package]] name = "fuel-sequencer-proto" version = "0.1.0" @@ -3679,7 +4167,7 @@ dependencies = [ "displaydoc", "fuel-streams-core", "futures", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", ] @@ -3690,23 +4178,22 @@ dependencies = [ "async-nats", "async-trait", "chrono", - "clap 4.5.23", "displaydoc", - "dotenvy", "fuel-core-client", "fuel-core-importer", "fuel-core-types 0.40.2", "fuel-data-parser", + "fuel-networks", "fuel-streams-macros", + "fuel-streams-storage", "futures", "hex", "pretty_assertions", - "rand", "serde", "serde_json", - "thiserror 2.0.4", + "sha2 0.10.8", + "thiserror 2.0.6", "tokio", - "tracing", ] [[package]] @@ -3752,6 +4239,7 @@ dependencies = [ "fuel-core-types 0.40.2", "fuel-streams", "fuel-streams-core", + "fuel-streams-storage", "futures", "mockall 0.13.1", "mockall_double", @@ -3767,7 +4255,7 @@ dependencies = [ "serde_prometheus", "sha2 0.10.8", "sysinfo", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", "tokio-stream", "tracing", @@ -3775,6 +4263,26 @@ dependencies = [ "url", ] +[[package]] +name = "fuel-streams-storage" +version = "0.0.13" +dependencies = [ + "async-nats", + "aws-config", + "aws-sdk-s3", + "aws-smithy-runtime-api", + "aws-smithy-types", + "displaydoc", + "dotenvy", + "fuel-networks", + "pretty_assertions", + "rand", + "serde_json", + "thiserror 2.0.6", + "tokio", + "tracing", +] + [[package]] name = "fuel-tx" version = "0.56.0" @@ -4145,13 +4653,24 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.0", "rand_core", "subtle", ] @@ -4940,9 +5459,9 @@ dependencies = [ [[package]] name = "impl-tools" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a84bc8d2baf8da56e93b4247067d918e1a44829bbbe3e4b875aaf8d7d3c7bc9" +checksum = "b4739bc9af85c18969eba5e4db90dbf26be140ff2e5628593693f18559e9e5fe" dependencies = [ "autocfg", "impl-tools-lib", @@ -4952,9 +5471,9 @@ dependencies = [ [[package]] name = "impl-tools-lib" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a795a1e201125947a063b967c79de6ae152143ab522f481d4f493c44835ba37a" +checksum = "798fe18a7e727001b30a029ab9cdd485afd325801d4df846f0bb5338b2986a2c" dependencies = [ "proc-macro-error2", "proc-macro2", @@ -5117,9 +5636,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -5152,11 +5671,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "once_cell", "sha2 0.10.8", - "signature", + "signature 2.2.0", ] [[package]] @@ -5194,9 +5713,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.167" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libflate" @@ -5935,6 +6454,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "memchr" version = "2.7.4" @@ -6106,9 +6635,9 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.2" +version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc41f430805af9d1cf4adae4ed2149c759b877b01d909a1f40256188d09345d2" +checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" dependencies = [ "core2", "unsigned-varint 0.8.0", @@ -6554,20 +7083,37 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2 0.10.8", +] + [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.8", ] @@ -6712,12 +7258,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 1.0.69", + "thiserror 2.0.6", "ucd-trie", ] @@ -6773,14 +7319,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", +] + [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -6975,7 +7531,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -7335,7 +7891,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.19", "socket2", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", "tracing", ] @@ -7354,7 +7910,7 @@ dependencies = [ "rustls 0.23.19", "rustls-pki-types", "slab", - "thiserror 2.0.4", + "thiserror 2.0.6", "tinyvec", "tracing", "web-time", @@ -7362,9 +7918,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ "cfg_aliases", "libc", @@ -7671,6 +8227,17 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac 0.12.1", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -7874,15 +8441,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -8118,16 +8685,30 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.9", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -8427,12 +9008,22 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "rand_core", - "signature", + "signature 2.2.0", "zeroize", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest 0.10.7", + "rand_core", +] + [[package]] name = "signature" version = "2.2.0" @@ -8555,6 +9146,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -8562,7 +9163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.9", ] [[package]] @@ -8910,7 +9511,7 @@ dependencies = [ "serde_json", "serde_repr", "sha2 0.10.8", - "signature", + "signature 2.2.0", "subtle", "subtle-encoding", "tendermint-proto 0.36.0", @@ -8940,7 +9541,7 @@ dependencies = [ "serde_json", "serde_repr", "sha2 0.10.8", - "signature", + "signature 2.2.0", "subtle", "subtle-encoding", "tendermint-proto 0.40.0", @@ -9058,11 +9659,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.6", ] [[package]] @@ -9078,9 +9679,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", @@ -9775,6 +10376,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -9832,6 +10439,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -9859,9 +10472,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -9870,13 +10483,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn 2.0.90", @@ -9885,9 +10497,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -9898,9 +10510,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9908,9 +10520,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -9921,9 +10533,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-encoder" @@ -10155,9 +10767,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -10563,6 +11175,12 @@ version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xmltree" version = "0.10.3" diff --git a/Cargo.toml b/Cargo.toml index f5b1c85b..1cf2d1c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,10 +60,12 @@ tracing-actix-web = "0.7" thiserror = "2.0" fuel-streams = { path = "crates/fuel-streams" } +fuel-networks = { path = "crates/fuel-networks" } fuel-data-parser = { version = "0.0.13", path = "crates/fuel-data-parser" } fuel-streams-core = { version = "0.0.13", path = "crates/fuel-streams-core" } fuel-streams-publisher = { version = "0.0.13", path = "crates/fuel-streams-publisher" } fuel-streams-macros = { version = "0.0.13", path = "crates/fuel-streams-macros" } +fuel-streams-storage = { version = "0.0.13", path = "crates/fuel-streams-storage" } subject-derive = { version = "0.0.13", path = "crates/fuel-streams-macros/subject-derive" } # Workspace projects diff --git a/README.md b/README.md index 2a07a7ce..880e5f05 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ With Fuel Data Systems, developers can build sophisticated applications that lev let mut subscription = stream.subscribe().await?; while let Some(entry) = subscription.next().await { let entry = entry.unwrap().clone(); - let block = Block::decode(entry).await; + let block = Block::decode(entry); println!("Received block: {:?}", block); } diff --git a/benches/data-parser/benches/deserialize.rs b/benches/data-parser/benches/deserialize.rs index 0a1ce5d2..7df9efdd 100644 --- a/benches/data-parser/benches/deserialize.rs +++ b/benches/data-parser/benches/deserialize.rs @@ -59,7 +59,6 @@ fn bench_deserialize(c: &mut Criterion) { let result = runtime.block_on(async { data_parser .deserialize::>(&serialized) - .await .expect("deserialization failed") }); // Use black_box to make sure 'result' is considered used by the compiler diff --git a/benches/load-tester/src/runners/runner_streamable.rs b/benches/load-tester/src/runners/runner_streamable.rs index 54092720..bda6f2db 100644 --- a/benches/load-tester/src/runners/runner_streamable.rs +++ b/benches/load-tester/src/runners/runner_streamable.rs @@ -21,21 +21,17 @@ pub async fn run_streamable_consumer( }; // Subscribe to the block stream with the specified configuration - let mut sub = stream.subscribe_with_config(config).await?; + let mut sub = stream.subscribe_raw_with_config(config).await?; // Process incoming blocks while let Some(bytes) = sub.next().await { - match bytes.as_ref() { - Err(_) => load_test_tracker.increment_error_count(), - Ok(message) => { - load_test_tracker.increment_message_count(); - let decoded_msg = S::decode_raw(message.payload.to_vec()).await; - let ts_millis = decoded_msg.ts_as_millis(); - load_test_tracker - .add_publish_time(ts_millis) - .increment_message_count(); - } - } + load_test_tracker.increment_message_count(); + let decoded_msg = S::decode_raw(bytes).unwrap(); + + let ts_millis = decoded_msg.ts_as_millis(); + load_test_tracker + .add_publish_time(ts_millis) + .increment_message_count(); } Ok(()) diff --git a/benches/nats-publisher/src/utils/nats.rs b/benches/nats-publisher/src/utils/nats.rs index a1d66c78..2a6e3ad6 100644 --- a/benches/nats-publisher/src/utils/nats.rs +++ b/benches/nats-publisher/src/utils/nats.rs @@ -7,7 +7,7 @@ use async_nats::{ ConnectOptions, }; use fuel_data_parser::DataParser; -use fuel_streams_core::nats::FuelNetwork; +use fuel_streams_core::prelude::FuelNetwork; #[allow(dead_code)] #[derive(Clone)] @@ -61,7 +61,7 @@ impl NatsHelper { pub async fn connect() -> anyhow::Result { Ok(ConnectOptions::new() .user_and_password("admin".into(), "secret".into()) - .connect(FuelNetwork::Local.to_url()) + .connect(FuelNetwork::Local.to_nats_url()) .await?) } diff --git a/cluster/docker/docker-compose.yml b/cluster/docker/docker-compose.yml index 185bc0c4..e007f202 100644 --- a/cluster/docker/docker-compose.yml +++ b/cluster/docker/docker-compose.yml @@ -18,3 +18,24 @@ services: - -D env_file: - ./../../.env + localstack: + profiles: + - all + - s3 + - dev + - monitoring + image: localstack/localstack:latest + container_name: localstack + restart: always + ports: + - "4566:4566" # LocalStack main gateway port + - "4572:4572" # S3 service port (optional) + environment: + - SERVICES=s3 # Enable just S3 service + - DEBUG=1 + - AWS_ACCESS_KEY_ID=test + - AWS_SECRET_ACCESS_KEY=test + - DEFAULT_REGION=us-east-1 + volumes: + - ./localstack-data:/var/lib/localstack + - /var/run/docker.sock:/var/run/docker.sock diff --git a/crates/fuel-data-parser/src/lib.rs b/crates/fuel-data-parser/src/lib.rs index 1ac52c3b..74b634f0 100644 --- a/crates/fuel-data-parser/src/lib.rs +++ b/crates/fuel-data-parser/src/lib.rs @@ -196,6 +196,13 @@ impl DataParser { }) } + pub fn encode_json( + &self, + data: &T, + ) -> Result, Error> { + self.serialize_json(data) + } + /// Serializes the provided data according to the selected `SerializationType`. /// /// # Arguments @@ -220,6 +227,14 @@ impl DataParser { } } + fn serialize_json( + &self, + raw_data: &T, + ) -> Result, Error> { + serde_json::to_vec(&raw_data) + .map_err(|e| Error::Serde(SerdeError::Json(e))) + } + /// Decodes the provided data by deserializing and optionally decompressing it. /// /// # Arguments @@ -259,10 +274,17 @@ impl DataParser { Some(strategy) => strategy.decompress(data).await?, None => data.to_vec(), }; - let decoded_data = self.deserialize(&data[..]).await?; + let decoded_data = self.deserialize(&data[..])?; Ok(decoded_data) } + pub fn decode_json( + &self, + data: &[u8], + ) -> Result { + self.deserialize_json(data) + } + /// Deserializes the provided data according to the selected `SerializationType`. /// /// # Arguments @@ -273,7 +295,7 @@ impl DataParser { /// /// A `Result` containing either the deserialized data structure, /// or an `Error` if deserialization fails. - pub async fn deserialize<'a, T: serde::Deserialize<'a>>( + pub fn deserialize<'a, T: serde::Deserialize<'a>>( &self, raw_data: &'a [u8], ) -> Result { @@ -282,10 +304,17 @@ impl DataParser { .map_err(|e| Error::Serde(SerdeError::Bincode(*e))), SerializationType::Postcard => postcard::from_bytes(raw_data) .map_err(|e| Error::Serde(SerdeError::Postcard(e))), - SerializationType::Json => serde_json::from_slice(raw_data) - .map_err(|e| Error::Serde(SerdeError::Json(e))), + SerializationType::Json => self.deserialize_json(raw_data), } } + + pub fn deserialize_json<'a, T: serde::Deserialize<'a>>( + &self, + raw_data: &'a [u8], + ) -> Result { + serde_json::from_slice(raw_data) + .map_err(|e| Error::Serde(SerdeError::Json(e))) + } } #[cfg(test)] diff --git a/crates/fuel-networks/Cargo.toml b/crates/fuel-networks/Cargo.toml new file mode 100644 index 00000000..29a165a5 --- /dev/null +++ b/crates/fuel-networks/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "fuel-networks" +description = "Networks for the Fuel blockchain and utilities for tranforming them for different apps" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +clap = { workspace = true } diff --git a/crates/fuel-networks/src/lib.rs b/crates/fuel-networks/src/lib.rs new file mode 100644 index 00000000..4a3ac3d0 --- /dev/null +++ b/crates/fuel-networks/src/lib.rs @@ -0,0 +1,75 @@ +/// FuelStreamsNetworks; shortened to FuelNetworks for brievity and public familiarity + +#[derive(Debug, Clone, Default)] +pub enum FuelNetworkUserRole { + Admin, + #[default] + Default, +} + +#[derive(Debug, Copy, Clone, Default, clap::ValueEnum)] +pub enum FuelNetwork { + #[default] + Local, + Testnet, + Mainnet, +} + +impl std::fmt::Display for FuelNetwork { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FuelNetwork::Local => write!(f, "local"), + FuelNetwork::Testnet => write!(f, "testnet"), + FuelNetwork::Mainnet => write!(f, "mainnet"), + } + } +} + +impl FuelNetwork { + pub fn load_from_env() -> Self { + match std::env::var("NETWORK").as_deref() { + Ok("testnet") => FuelNetwork::Testnet, + Ok("mainnet") => FuelNetwork::Mainnet, + _ => FuelNetwork::Local, + } + } + + pub fn to_nats_url(&self) -> String { + match self { + FuelNetwork::Local => "nats://localhost:4222", + FuelNetwork::Testnet => "nats://stream-testnet.fuel.network:4222", + FuelNetwork::Mainnet => "nats://stream.fuel.network:4222", + } + .to_string() + } + + pub fn to_s3_url(&self) -> String { + match self { + FuelNetwork::Local => "http://localhost:4566".to_string(), + FuelNetwork::Testnet | FuelNetwork::Mainnet => { + let bucket = self.to_s3_bucket(); + let region = self.to_s3_region(); + // TODO: Update for client streaming + format!("https://{bucket}.s3-website-{region}.amazonaws.com") + } + } + } + + pub fn to_s3_region(&self) -> String { + // TODO: Update correctly for client streaming + match self { + FuelNetwork::Local + | FuelNetwork::Testnet + | FuelNetwork::Mainnet => "us-east-1".to_string(), + } + } + + pub fn to_s3_bucket(&self) -> String { + match self { + FuelNetwork::Local => "fuel-streams-local", + FuelNetwork::Testnet => "fuel-streams-testnet", + FuelNetwork::Mainnet => "fuel-streams", + } + .to_string() + } +} diff --git a/crates/fuel-streams-core/Cargo.toml b/crates/fuel-streams-core/Cargo.toml index 1ff4045f..a2f5acb2 100644 --- a/crates/fuel-streams-core/Cargo.toml +++ b/crates/fuel-streams-core/Cargo.toml @@ -14,22 +14,21 @@ rust-version = { workspace = true } async-nats = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } -clap = { workspace = true } displaydoc = { workspace = true } -dotenvy = { workspace = true } fuel-core-client = { workspace = true } fuel-core-importer = { workspace = true } fuel-core-types = { workspace = true } fuel-data-parser = { workspace = true } +fuel-networks = { workspace = true } fuel-streams-macros = { workspace = true } +fuel-streams-storage = { workspace = true } futures = { workspace = true } hex = { workspace = true } pretty_assertions = { workspace = true, optional = true } -rand = { workspace = true } serde = { workspace = true } +sha2 = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tracing = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/crates/fuel-streams-core/README.md b/crates/fuel-streams-core/README.md index 0304865e..221aca64 100644 --- a/crates/fuel-streams-core/README.md +++ b/crates/fuel-streams-core/README.md @@ -54,25 +54,28 @@ fuel-streams-core = "*" Here's a simple example to get you started with Fuel Streams Core: ```rust,no_run +use std::sync::Arc; use fuel_streams_core::prelude::*; use futures::StreamExt; #[tokio::main] async fn main() -> BoxedResult<()> { // Connect to NATS server - let opts = NatsClientOpts::new(Some(FuelNetwork::Local)); - let client = NatsClient::connect(&opts).await?; + let nats_opts = NatsClientOpts::new(FuelNetwork::Local); + let nats_client = NatsClient::connect(&nats_opts).await?; + + let s3_opts = S3ClientOpts::new(FuelNetwork::Local); + let s3_client = Arc::new(S3Client::new(&s3_opts).await?); // Create a stream for blocks - let stream = Stream::::new(&client).await; + let stream = Stream::::new(&nats_client, &s3_client).await; // Subscribe to the stream let wildcard = BlocksSubject::wildcard(None, None); // blocks.*.* - let mut subscription = stream.subscribe(&wildcard).await?; + let mut subscription = stream.subscribe(None).await?; // Process incoming blocks - while let Some(bytes) = subscription.next().await { - let block = Block::decode_raw(bytes.unwrap()).await; + while let Some(block) = subscription.next().await { dbg!(block); } diff --git a/crates/fuel-streams-core/src/lib.rs b/crates/fuel-streams-core/src/lib.rs index 040bae8d..11fe85b5 100644 --- a/crates/fuel-streams-core/src/lib.rs +++ b/crates/fuel-streams-core/src/lib.rs @@ -8,7 +8,14 @@ pub mod receipts; pub mod transactions; pub mod utxos; -pub mod nats; +pub mod nats { + pub use fuel_streams_storage::nats::*; +} + +pub mod s3 { + pub use fuel_streams_storage::s3::*; +} + pub mod stream; pub mod subjects; @@ -20,7 +27,8 @@ pub mod types; pub use stream::*; pub mod prelude { + pub use fuel_networks::*; pub use fuel_streams_macros::subject::*; - pub use crate::{nats::*, stream::*, subjects::*, types::*}; + pub use crate::{nats::*, s3::*, stream::*, subjects::*, types::*}; } diff --git a/crates/fuel-streams-core/src/stream/error.rs b/crates/fuel-streams-core/src/stream/error.rs index e3e6f31e..27994989 100644 --- a/crates/fuel-streams-core/src/stream/error.rs +++ b/crates/fuel-streams-core/src/stream/error.rs @@ -19,6 +19,9 @@ pub enum StreamError { source: error::Error, }, + /// Failed to publish to S3 + S3PublishError(#[from] fuel_streams_storage::s3::S3ClientError), + /// Failed to retrieve last published message from stream GetLastPublishedFailed(#[from] error::Error), diff --git a/crates/fuel-streams-core/src/stream/stream_encoding.rs b/crates/fuel-streams-core/src/stream/stream_encoding.rs index e2f29215..31c7e0b2 100644 --- a/crates/fuel-streams-core/src/stream/stream_encoding.rs +++ b/crates/fuel-streams-core/src/stream/stream_encoding.rs @@ -40,23 +40,40 @@ where #[async_trait] pub trait StreamEncoder: DataParseable { - async fn encode(&self, subject: &str) -> Vec { + // TODO: Should we remove the `StreamData` type and encode/decode the raw data only + fn encode(&self, subject: &str) -> Vec { let data = StreamData::new(subject, self.clone()); Self::data_parser() - .encode(&data) - .await + .encode_json(&data) .expect("Streamable must encode correctly") } - async fn decode(encoded: Vec) -> Self { - Self::decode_raw(encoded).await.payload + fn encode_self(&self) -> Vec { + Self::data_parser() + .encode_json(self) + .expect("Streamable must encode correctly") + } + + fn decode(encoded: Vec) -> Result { + Ok(Self::decode_raw(encoded)?.payload) + } + + fn decode_or_panic(encoded: Vec) -> Self { + Self::decode_raw(encoded) + .expect("Streamable must decode correctly") + .payload + } + + fn decode_raw( + encoded: Vec, + ) -> Result, fuel_data_parser::Error> { + Self::data_parser().decode_json(&encoded) } - async fn decode_raw(encoded: Vec) -> StreamData { + fn decode_raw_or_panic(encoded: Vec) -> StreamData { Self::data_parser() - .decode(&encoded) - .await + .decode_json(&encoded) .expect("Streamable must decode correctly") } diff --git a/crates/fuel-streams-core/src/stream/stream_impl.rs b/crates/fuel-streams-core/src/stream/stream_impl.rs index c3b7e855..984e05e4 100644 --- a/crates/fuel-streams-core/src/stream/stream_impl.rs +++ b/crates/fuel-streams-core/src/stream/stream_impl.rs @@ -1,6 +1,4 @@ -#[cfg(any(test, feature = "test-helpers"))] -use std::pin::Pin; -use std::{fmt::Debug, sync::Arc, time::Duration}; +use std::{fmt::Debug, sync::Arc}; use async_nats::{ jetstream::{ @@ -12,26 +10,28 @@ use async_nats::{ }; use async_trait::async_trait; use fuel_streams_macros::subject::IntoSubject; -use futures::{StreamExt, TryStreamExt}; +use fuel_streams_storage::*; +use futures::{stream::BoxStream, StreamExt, TryStreamExt}; +use sha2::{Digest, Sha256}; use tokio::sync::OnceCell; use super::{error::StreamError, stream_encoding::StreamEncoder}; -use crate::{nats::types::*, prelude::NatsClient}; -pub const FUEL_BLOCK_TIME_SECS: u64 = 1; -pub const MAX_RETENTION_BLOCKS: u64 = 100; - -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct PublishPacket { pub subject: Arc, pub payload: Arc, + pub s3_path: String, } impl PublishPacket { pub fn new(payload: T, subject: Arc) -> Self { + let s3_path = payload.get_s3_path(); + Self { payload: Arc::new(payload), subject, + s3_path, } } } @@ -65,6 +65,19 @@ pub trait Streamable: StreamEncoder { fn to_packet(&self, subject: Arc) -> PublishPacket { PublishPacket::new(self.clone(), subject) } + + fn get_s3_path(&self) -> String { + format!("v1/{}/{}.json", Self::NAME, self.get_consistent_hash()) + } + + fn get_consistent_hash(&self) -> String { + let serialized = self.encode_self(); + + let mut hasher = Sha256::new(); + hasher.update(serialized); + + format!("{:x}", hasher.finalize()) + } } /// Houses nats-agnostic APIs for publishing and consuming a streamable type @@ -72,6 +85,7 @@ pub trait Streamable: StreamEncoder { /// # Examples /// /// ```no_run +/// use std::sync::Arc; /// use fuel_streams_core::prelude::*; /// use fuel_streams_macros::subject::IntoSubject; /// use futures::StreamExt; @@ -89,17 +103,16 @@ pub trait Streamable: StreamEncoder { /// const WILDCARD_LIST: &'static [&'static str] = &["*"]; /// } /// -/// async fn example(client: &NatsClient) { -/// let stream = Stream::::new(client).await; +/// async fn example(nats_client: &NatsClient, s3_client: &Arc) { +/// let stream = Stream::::new(nats_client, s3_client).await; /// /// // Publish -/// let subject = BlocksSubject::new().with_height(Some(23.into())); -/// let payload = MyStreamable { data: "foo".into() }; -/// stream.publish(&subject, &payload).await.unwrap(); +/// let subject = BlocksSubject::new().with_height(Some(23.into())).arc(); +/// let packet = MyStreamable { data: "foo".into() }.to_packet(subject); +/// stream.publish(&packet).await.unwrap(); /// /// // Subscribe -/// let wildcard = BlocksSubject::WILDCARD; -/// let mut subscription = stream.subscribe(wildcard).await.unwrap(); +/// let mut subscription = stream.subscribe(None).await.unwrap(); /// while let Some(message) = subscription.next().await { /// // Process message /// } @@ -110,7 +123,8 @@ pub trait Streamable: StreamEncoder { /// TODO: Rename as FuelStream? #[derive(Debug, Clone)] pub struct Stream { - store: kv::Store, + store: Arc, + s3_client: Arc, _marker: std::marker::PhantomData, } @@ -118,59 +132,70 @@ impl Stream { #[allow(clippy::declare_interior_mutable_const)] const INSTANCE: OnceCell = OnceCell::const_new(); - pub async fn get_or_init(client: &NatsClient) -> Self { + pub async fn get_or_init( + nats_client: &NatsClient, + s3_client: &Arc, + ) -> Self { let cell = Self::INSTANCE; - cell.get_or_init(|| async { Self::new(client).await.to_owned() }) - .await - .to_owned() + cell.get_or_init(|| async { + Self::new(nats_client, s3_client).await.to_owned() + }) + .await + .to_owned() } - pub async fn new(client: &NatsClient) -> Self { - let namespace = &client.namespace; + pub async fn new( + nats_client: &NatsClient, + s3_client: &Arc, + ) -> Self { + let namespace = &nats_client.namespace; let bucket_name = namespace.stream_name(S::NAME); - let store = client + let store = nats_client .get_or_create_kv_store(kv::Config { bucket: bucket_name.to_owned(), storage: stream::StorageType::File, history: 1, - compression: true, - max_age: Duration::from_secs( - FUEL_BLOCK_TIME_SECS * MAX_RETENTION_BLOCKS, - ), ..Default::default() }) .await .expect("Streams must be created"); Self { - store, + store: Arc::new(store), + s3_client: Arc::clone(s3_client), _marker: std::marker::PhantomData, } } pub async fn publish( &self, - subject: &dyn IntoSubject, - payload: &S, + packet: &PublishPacket, ) -> Result { - let subject_name = &subject.parse(); - self.publish_raw(subject_name, payload).await + let payload = &packet.payload; + let s3_path = &packet.s3_path; + let subject_name = &packet.subject.parse(); + + // publish payload to S3 + self.s3_client + .put_object(s3_path, payload.encode(subject_name)) + .await?; + + self.publish_s3_path_to_nats(subject_name, s3_path).await } - /// Publish with subject name with no static guarantees of the subject - pub async fn publish_raw( + async fn publish_s3_path_to_nats( &self, subject_name: &str, - payload: &S, + s3_path: &str, ) -> Result { - let data = payload.encode(subject_name).await; + let data = s3_path.to_string().into_bytes(); let data_size = data.len(); + let result = self.store.create(subject_name, data.into()).await; match result { Ok(_) => Ok(data_size), Err(e) if e.kind() == CreateErrorKind::AlreadyExists => { - // This is a workaround to avoid publish two times the same message Ok(data_size) } Err(e) => Err(StreamError::PublishFailed { @@ -199,17 +224,93 @@ impl Stream { self.store.stream_name.as_str() } - // TODO: This should probably be `subscribe_raw` since it returns pure bytes + // Less performant due to our hybrid use of NATS and S3 + pub async fn subscribe_raw( + &self, + subscription_config: Option, + ) -> Result>, StreamError> { + let mut config = PullConsumerConfig { + filter_subjects: vec![S::WILDCARD_LIST[0].to_string()], + deliver_policy: DeliverPolicy::All, + ack_policy: AckPolicy::None, + ..Default::default() + }; + + if let Some(subscription_config) = subscription_config { + config.filter_subjects = subscription_config.filter_subjects; + config.deliver_policy = subscription_config.deliver_policy; + } + + let config = self.prefix_filter_subjects(config); + let consumer = self.store.stream.create_consumer(config).await?; + + Ok(consumer + .messages() + .await? + .then(|message| { + let s3_client = Arc::clone(&self.s3_client); + + async move { + let nats_payload = message + .expect("Message must be valid") + .payload + .to_vec(); + + // TODO: Bubble up the error to users + let s3_path = String::from_utf8(nats_payload) + .expect("Must be S3 path"); + + s3_client + .get_object(&s3_path) + .await + .expect("S3 object must exist") + } + }) + .boxed()) + } + pub async fn subscribe( &self, - // TODO: Allow encapsulating Subject to return wildcard token type - wildcard: &str, - ) -> Result>>, StreamError> { - Ok(self.store.watch(&wildcard).await.map(|stream| { - stream.map(|entry| { - entry.ok().map(|entry_item| entry_item.value.to_vec()) + subscription_config: Option, + ) -> Result, StreamError> { + let mut config = PullConsumerConfig { + filter_subjects: vec![S::WILDCARD_LIST[0].to_string()], + deliver_policy: DeliverPolicy::All, + ack_policy: AckPolicy::None, + ..Default::default() + }; + + if let Some(subscription_config) = subscription_config { + config.filter_subjects = subscription_config.filter_subjects; + config.deliver_policy = subscription_config.deliver_policy; + } + + let config = self.prefix_filter_subjects(config); + let consumer = self.store.stream.create_consumer(config).await?; + + Ok(consumer + .messages() + .await? + .map(|item| { + String::from_utf8( + item.expect("Message must be valid").payload.to_vec(), + ) + .expect("Must be S3 path") }) - })?) + .then(|s3_path| { + let s3_client = Arc::clone(&self.s3_client); + + async move { + // TODO: Bubble up the error? + S::decode_or_panic( + s3_client + .get_object(&s3_path) + .await + .expect("Could not get S3 object"), + ) + } + }) + .boxed()) } #[cfg(feature = "test-helpers")] @@ -217,10 +318,7 @@ impl Stream { pub async fn catchup( &self, number_of_messages: usize, - ) -> Result< - Pin> + Send>>, - StreamError, - > { + ) -> Result>, StreamError> { let config = PullConsumerConfig { filter_subjects: self.all_filter_subjects(), deliver_policy: DeliverPolicy::All, @@ -230,36 +328,24 @@ impl Stream { let config = self.prefix_filter_subjects(config); let consumer = self.store.stream.create_consumer(config).await?; - let stream = consumer.messages().await?.take(number_of_messages).then( - |message| async { + let stream = consumer + .messages() + .await? + .take(number_of_messages) + .then(|message| async { if let Ok(message) = message { - Some(S::decode(message.payload.to_vec()).await) + Some( + self.get_payload_from_s3(message.payload.to_vec()) + .await + .unwrap(), + ) } else { None } - }, - ); - - // Use Box::pin to pin the stream on the heap - Ok(Box::pin(stream)) - } - - // TODO: Make this interface more Stream-like and Nats agnostic - // TODO: This should probably be removed in favor of `subscribe` - pub async fn subscribe_consumer( - &self, - config: SubscribeConsumerConfig, - ) -> Result { - let config = PullConsumerConfig { - filter_subjects: config.filter_subjects, - deliver_policy: config.deliver_policy, - ack_policy: AckPolicy::None, - ..Default::default() - }; + }) + .boxed(); - let config = self.prefix_filter_subjects(config); - let consumer = self.store.stream.create_consumer(config).await?; - Ok(consumer.messages().await?) + Ok(stream) } // TODO: Make this interface more Stream-like and Nats agnostic @@ -299,11 +385,9 @@ impl Stream { .await; match message { - Ok(message) => { - let payload = S::decode(message.payload.to_vec()).await; - - Ok(Some(payload)) - } + Ok(message) => Ok(Some( + self.get_payload_from_s3(message.payload.to_vec()).await?, + )), Err(error) => match &error.kind() { LastRawMessageErrorKind::NoMessageFound => Ok(None), _ => Err(error.into()), @@ -311,6 +395,21 @@ impl Stream { } } + async fn get_payload_from_s3( + &self, + nats_payload: Vec, + ) -> Result { + let s3_path = String::from_utf8(nats_payload).expect("Must be S3 path"); + + let s3_object = self + .s3_client + .get_object(&s3_path) + .await + .expect("S3 object must exist"); + + Ok(S::decode_or_panic(s3_object)) + } + #[cfg(any(test, feature = "test-helpers"))] pub async fn assert_has_stream( &self, @@ -352,16 +451,16 @@ impl Stream { /// # Examples /// /// ``` -/// use fuel_streams_core::stream::SubscribeConsumerConfig; +/// use fuel_streams_core::stream::SubscriptionConfig; /// use async_nats::jetstream::consumer::DeliverPolicy; /// -/// let config = SubscribeConsumerConfig { +/// let config = SubscriptionConfig { /// filter_subjects: vec!["example.*".to_string()], /// deliver_policy: DeliverPolicy::All, /// }; /// ``` #[derive(Debug, Clone, Default)] -pub struct SubscribeConsumerConfig { +pub struct SubscriptionConfig { pub filter_subjects: Vec, pub deliver_policy: DeliverPolicy, } diff --git a/crates/fuel-streams-core/src/types.rs b/crates/fuel-streams-core/src/types.rs index 9b9c16fb..efacbacc 100644 --- a/crates/fuel-streams-core/src/types.rs +++ b/crates/fuel-streams-core/src/types.rs @@ -1,5 +1,3 @@ -use std::error::Error; - pub use crate::{ blocks::types::*, fuel_core_types::*, @@ -16,4 +14,4 @@ pub use crate::{ // ------------------------------------------------------------------------ // General // ------------------------------------------------------------------------ -pub type BoxedResult = Result>; +pub type BoxedResult = Result>; diff --git a/crates/fuel-streams-publisher/Cargo.lock b/crates/fuel-streams-publisher/Cargo.lock deleted file mode 100644 index 046abc4a..00000000 --- a/crates/fuel-streams-publisher/Cargo.lock +++ /dev/null @@ -1,7553 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -dependencies = [ - "lazy_static", - "regex", -] - -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common", - "generic-array", -] - -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - -[[package]] -name = "aes-gcm" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" - -[[package]] -name = "anstyle-parse" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - -[[package]] -name = "anyhow" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" - -[[package]] -name = "arbitrary" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" - -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" - -[[package]] -name = "asn1-rs" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ad1373757efa0f70ec53939aabc7152e1591cb485208052993070ac8d2429d" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror", - "time", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7378575ff571966e99a744addeff0bff98b8ada0dedf1956d59e634db95eaac1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "asn1_der" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" - -[[package]] -name = "async-graphql" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ed522678d412d77effe47b3c82314ac36952a35e6e852093dd48287c421f80" -dependencies = [ - "async-graphql-derive", - "async-graphql-parser", - "async-graphql-value", - "async-stream", - "async-trait", - "base64 0.13.1", - "bytes", - "fnv", - "futures-util", - "http", - "indexmap 1.9.3", - "mime", - "multer", - "num-traits", - "once_cell", - "pin-project-lite", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "static_assertions", - "tempfile", - "thiserror", - "tracing", - "tracing-futures", -] - -[[package]] -name = "async-graphql-derive" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c121a894495d7d3fc3d4e15e0a9843e422e4d1d9e3c514d8062a1c94b35b005d" -dependencies = [ - "Inflector", - "async-graphql-parser", - "darling 0.14.4", - "proc-macro-crate 1.3.1", - "proc-macro2", - "quote", - "syn 1.0.109", - "thiserror", -] - -[[package]] -name = "async-graphql-parser" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b6c386f398145c6180206c1869c2279f5a3d45db5be4e0266148c6ac5c6ad68" -dependencies = [ - "async-graphql-value", - "pest", - "serde", - "serde_json", -] - -[[package]] -name = "async-graphql-value" -version = "4.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a941b499fead4a3fb5392cabf42446566d18c86313f69f2deab69560394d65f" -dependencies = [ - "bytes", - "indexmap 1.9.3", - "serde", - "serde_json", -] - -[[package]] -name = "async-io" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" -dependencies = [ - "async-lock", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite", - "parking", - "polling", - "rustix", - "slab", - "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "async-lock" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-nats" -version = "0.35.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab8df97cb8fc4a884af29ab383e9292ea0939cfcdd7d2a17179086dc6c427e7f" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures", - "memchr", - "nkeys", - "nuid", - "once_cell", - "portable-atomic", - "rand", - "regex", - "ring 0.17.8", - "rustls-native-certs", - "rustls-pemfile 2.1.2", - "rustls-webpki 0.102.4", - "serde", - "serde_json", - "serde_nanos", - "serde_repr", - "thiserror", - "time", - "tokio", - "tokio-rustls 0.26.0", - "tracing", - "tryhard", - "url", -] - -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "async-trait" -version = "0.1.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "async_io_stream" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" -dependencies = [ - "futures", - "pharos", - "rustc_version", -] - -[[package]] -name = "asynchronous-codec" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568" -dependencies = [ - "bytes", - "futures-sink", - "futures-util", - "memchr", - "pin-project-lite", -] - -[[package]] -name = "asynchronous-codec" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233" -dependencies = [ - "bytes", - "futures-sink", - "futures-util", - "memchr", - "pin-project-lite", -] - -[[package]] -name = "atomic-polyfill" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" -dependencies = [ - "critical-section", -] - -[[package]] -name = "attohttpc" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" -dependencies = [ - "http", - "log", - "url", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - -[[package]] -name = "auto_impl" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "autocfg" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" - -[[package]] -name = "axum" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" -dependencies = [ - "async-trait", - "axum-core", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-http", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "tower-layer", - "tower-service", -] - -[[package]] -name = "backtrace" -version = "0.3.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "serde", -] - -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "bech32" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" - -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bindgen" -version = "0.65.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.66", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" -dependencies = [ - "serde", -] - -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "sha2 0.10.8", - "tinyvec", -] - -[[package]] -name = "bumpalo" -version = "3.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" - -[[package]] -name = "byte-slice-cast" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" -dependencies = [ - "serde", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "camino" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo-platform" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo_metadata" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" -dependencies = [ - "camino", - "cargo-platform", - "semver", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "cc" -version = "1.0.98" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" -dependencies = [ - "jobserver", - "libc", - "once_cell", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chacha20" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - -[[package]] -name = "chacha20poly1305" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" -dependencies = [ - "aead", - "chacha20", - "cipher", - "poly1305", - "zeroize", -] - -[[package]] -name = "chrono" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "serde", - "windows-targets 0.52.5", -] - -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", - "zeroize", -] - -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "atty", - "bitflags 1.3.2", - "clap_derive 3.2.25", - "clap_lex 0.2.4", - "indexmap 1.9.3", - "once_cell", - "strsim 0.10.0", - "termcolor", - "textwrap", -] - -[[package]] -name = "clap" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" -dependencies = [ - "clap_builder", - "clap_derive 4.5.4", -] - -[[package]] -name = "clap_builder" -version = "4.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" -dependencies = [ - "anstream", - "anstyle", - "clap_lex 0.7.0", - "strsim 0.11.1", -] - -[[package]] -name = "clap_derive" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" -dependencies = [ - "heck 0.4.1", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "clap_derive" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "clap_lex" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" - -[[package]] -name = "cobs" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" - -[[package]] -name = "coins-bip32" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b6be4a5df2098cd811f3194f64ddb96c267606bffd9689ac7b0160097b01ad3" -dependencies = [ - "bs58", - "coins-core", - "digest 0.10.7", - "hmac 0.12.1", - "k256", - "serde", - "sha2 0.10.8", - "thiserror", -] - -[[package]] -name = "coins-bip39" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db8fba409ce3dc04f7d804074039eb68b960b0829161f8e06c95fea3f122528" -dependencies = [ - "bitvec", - "coins-bip32", - "hmac 0.12.1", - "once_cell", - "pbkdf2", - "rand", - "sha2 0.10.8", - "thiserror", -] - -[[package]] -name = "coins-core" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5286a0843c21f8367f7be734f89df9b822e0321d8bcce8d6e735aadff7d74979" -dependencies = [ - "base64 0.21.7", - "bech32", - "bs58", - "digest 0.10.7", - "generic-array", - "hex", - "ripemd", - "serde", - "serde_derive", - "sha2 0.10.8", - "sha3", - "thiserror", -] - -[[package]] -name = "colorchoice" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "console" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.52.0", -] - -[[package]] -name = "const-hex" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8a24a26d37e1ffd45343323dc9fe6654ceea44c12f2fcb3d7ac29e610bc6" -dependencies = [ - "cfg-if", - "cpufeatures", - "hex", - "proptest", - "serde", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "const_format" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673" -dependencies = [ - "const_format_proc_macros", -] - -[[package]] -name = "const_format_proc_macros" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" - -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - -[[package]] -name = "cpp_demangle" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "cpufeatures" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" -dependencies = [ - "libc", -] - -[[package]] -name = "cranelift-bforest" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496c993b62bdfbe9b4c518b8b3e1fdba9f89ef89fcccc050ab61d91dfba9fbaf" -dependencies = [ - "cranelift-entity", -] - -[[package]] -name = "cranelift-codegen" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b922abb6be41fc383f5e9da65b58d32d0d0a32c87dfe3bbbcb61a09119506c" -dependencies = [ - "bumpalo", - "cranelift-bforest", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-control", - "cranelift-entity", - "cranelift-isle", - "gimli", - "hashbrown 0.14.5", - "log", - "regalloc2", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-codegen-meta" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634c2ed9ef8a04ca42535a3e2e7917e4b551f2f306f4df2d935a6e71e346c167" -dependencies = [ - "cranelift-codegen-shared", -] - -[[package]] -name = "cranelift-codegen-shared" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00cde1425b4da28bb0d5ff010030ea9cc9be7aded342ae099b394284f17cefce" - -[[package]] -name = "cranelift-control" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1622125c99f1864aaf44e57971770c4a918d081d4b4af0bb597bdf624660ed66" -dependencies = [ - "arbitrary", -] - -[[package]] -name = "cranelift-entity" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea97887aca1c0cbe7f8513874dc3603e9744fb1cfa78840ca8897bd2766bd35b" -dependencies = [ - "serde", - "serde_derive", -] - -[[package]] -name = "cranelift-frontend" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdade4c14183fe41482071ed77d6a38cb95a17c7a0a05e629152e6292c4f8cb" -dependencies = [ - "cranelift-codegen", - "log", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-isle" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbe4d3ad7bd4bf4a8d916c8460b441cf92417f5cdeacce4dd1d96eee70b18a2" - -[[package]] -name = "cranelift-native" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c46be4ed1fc8f36df4e2a442b8c30a39d8c03c1868182978f4c04ba2c25c9d4f" -dependencies = [ - "cranelift-codegen", - "libc", - "target-lexicon", -] - -[[package]] -name = "cranelift-wasm" -version = "0.105.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d4c4a785a7866da89d20df159e3c4f96a5f14feb83b1f5998cfd5fe2e74d06" -dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "itertools 0.10.5", - "log", - "smallvec", - "wasmparser", - "wasmtime-types", -] - -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "critical-section" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "rand_core", - "typenum", -] - -[[package]] -name = "crypto-mac" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher", -] - -[[package]] -name = "curve25519-dalek" -version = "4.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" -dependencies = [ - "cfg-if", - "cpufeatures", - "curve25519-dalek-derive", - "digest 0.10.7", - "fiat-crypto", - "platforms", - "rustc_version", - "subtle", - "zeroize", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" -dependencies = [ - "darling_core 0.20.9", - "darling_macro 0.20.9", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", -] - -[[package]] -name = "darling_core" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.11.1", - "syn 2.0.66", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core 0.14.4", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" -dependencies = [ - "darling_core 0.20.9", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "data-encoding" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" - -[[package]] -name = "data-encoding-macro" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" -dependencies = [ - "data-encoding", - "data-encoding-macro-internal", -] - -[[package]] -name = "data-encoding-macro-internal" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" -dependencies = [ - "data-encoding", - "syn 1.0.109", -] - -[[package]] -name = "debugid" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" -dependencies = [ - "uuid", -] - -[[package]] -name = "der" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "der-parser" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - -[[package]] -name = "deranged" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" -dependencies = [ - "powerfmt", - "serde", -] - -[[package]] -name = "derivative" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_more" -version = "0.99.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", -] - -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "directories-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "displaydoc" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - -[[package]] -name = "dtoa" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" - -[[package]] -name = "dunce" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" - -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der", - "digest 0.10.7", - "elliptic-curve", - "rfc6979", - "signature", - "spki", -] - -[[package]] -name = "ed25519" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" -dependencies = [ - "pkcs8", - "signature", -] - -[[package]] -name = "ed25519-dalek" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" -dependencies = [ - "curve25519-dalek", - "ed25519", - "rand_core", - "serde", - "sha2 0.10.8", - "signature", - "subtle", - "zeroize", -] - -[[package]] -name = "either" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" - -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest 0.10.7", - "ff", - "generic-array", - "group", - "pkcs8", - "rand_core", - "sec1", - "subtle", - "zeroize", -] - -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "enr" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3d8dc56e02f954cac8eb489772c552c473346fc34f67412bb6244fd647f7e4" -dependencies = [ - "base64 0.21.7", - "bytes", - "hex", - "k256", - "log", - "rand", - "rlp", - "serde", - "sha3", - "zeroize", -] - -[[package]] -name = "enum-as-inner" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "enum-iterator" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" -dependencies = [ - "enum-iterator-derive", -] - -[[package]] -name = "enum-iterator-derive" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "errno" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "ethabi" -version = "18.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7413c5f74cc903ea37386a8965a936cbeb334bd270862fdece542c1b2dcbc898" -dependencies = [ - "ethereum-types", - "hex", - "once_cell", - "regex", - "serde", - "serde_json", - "sha3", - "thiserror", - "uint", -] - -[[package]] -name = "ethbloom" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60" -dependencies = [ - "crunchy", - "fixed-hash", - "impl-codec", - "impl-rlp", - "impl-serde", - "scale-info", - "tiny-keccak", -] - -[[package]] -name = "ethereum-types" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d215cbf040552efcbe99a38372fe80ab9d00268e20012b79fcd0f073edd8ee" -dependencies = [ - "ethbloom", - "fixed-hash", - "impl-codec", - "impl-rlp", - "impl-serde", - "primitive-types", - "scale-info", - "uint", -] - -[[package]] -name = "ethers-contract" -version = "2.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fceafa3578c836eeb874af87abacfb041f92b4da0a78a5edd042564b8ecdaaa" -dependencies = [ - "const-hex", - "ethers-contract-abigen", - "ethers-contract-derive", - "ethers-core", - "futures-util", - "once_cell", - "pin-project", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "ethers-contract-abigen" -version = "2.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04ba01fbc2331a38c429eb95d4a570166781f14290ef9fdb144278a90b5a739b" -dependencies = [ - "Inflector", - "const-hex", - "dunce", - "ethers-core", - "eyre", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "serde", - "serde_json", - "syn 2.0.66", - "toml 0.8.13", - "walkdir", -] - -[[package]] -name = "ethers-contract-derive" -version = "2.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87689dcabc0051cde10caaade298f9e9093d65f6125c14575db3fd8c669a168f" -dependencies = [ - "Inflector", - "const-hex", - "ethers-contract-abigen", - "ethers-core", - "proc-macro2", - "quote", - "serde_json", - "syn 2.0.66", -] - -[[package]] -name = "ethers-core" -version = "2.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d80cc6ad30b14a48ab786523af33b37f28a8623fc06afd55324816ef18fb1f" -dependencies = [ - "arrayvec", - "bytes", - "cargo_metadata", - "chrono", - "const-hex", - "elliptic-curve", - "ethabi", - "generic-array", - "k256", - "num_enum", - "once_cell", - "open-fastrlp", - "rand", - "rlp", - "serde", - "serde_json", - "strum 0.26.2", - "syn 2.0.66", - "tempfile", - "thiserror", - "tiny-keccak", - "unicode-xid", -] - -[[package]] -name = "ethers-providers" -version = "2.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6434c9a33891f1effc9c75472e12666db2fa5a0fec4b29af6221680a6fe83ab2" -dependencies = [ - "async-trait", - "auto_impl", - "base64 0.21.7", - "bytes", - "const-hex", - "enr", - "ethers-core", - "futures-channel", - "futures-core", - "futures-timer", - "futures-util", - "hashers", - "http", - "instant", - "jsonwebtoken", - "once_cell", - "pin-project", - "reqwest", - "serde", - "serde_json", - "thiserror", - "tokio", - "tokio-tungstenite", - "tracing", - "tracing-futures", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "ws_stream_wasm", -] - -[[package]] -name = "ethnum" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" - -[[package]] -name = "event-listener" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener", - "pin-project-lite", -] - -[[package]] -name = "eyre" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" -dependencies = [ - "indenter", - "once_cell", -] - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fastrand" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" - -[[package]] -name = "ff" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" -dependencies = [ - "rand_core", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" - -[[package]] -name = "findshlibs" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" -dependencies = [ - "cc", - "lazy_static", - "libc", - "winapi", -] - -[[package]] -name = "fixed-hash" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" -dependencies = [ - "byteorder", - "rand", - "rustc-hex", - "static_assertions", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fuel-asm" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42df651415e443094f86102473b7f9fa23633ab6c3c98dd3f713adde251acf0f" -dependencies = [ - "bitflags 2.5.0", - "fuel-types", - "serde", - "strum 0.24.1", -] - -[[package]] -name = "fuel-core" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b030e12851d70598e12722886b899e28884d168367fc20d9a809951dd599004" -dependencies = [ - "anyhow", - "async-graphql", - "async-trait", - "axum", - "clap 4.5.4", - "derive_more", - "enum-iterator", - "fuel-core-chain-config", - "fuel-core-consensus-module", - "fuel-core-database", - "fuel-core-executor", - "fuel-core-importer", - "fuel-core-metrics", - "fuel-core-p2p", - "fuel-core-poa", - "fuel-core-producer", - "fuel-core-relayer", - "fuel-core-services", - "fuel-core-storage", - "fuel-core-sync", - "fuel-core-txpool", - "fuel-core-types", - "fuel-core-upgradable-executor", - "futures", - "hex", - "hyper", - "indicatif", - "itertools 0.12.1", - "num_cpus", - "rand", - "rocksdb", - "serde", - "serde_json", - "strum 0.25.0", - "strum_macros 0.25.3", - "tempfile", - "thiserror", - "tokio", - "tokio-stream", - "tokio-util", - "tower-http", - "tracing", - "uuid", -] - -[[package]] -name = "fuel-core-bin" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5ff2b6ce36b11e79f338b22fe436962d8d587b60a8d751c6bef2b7cb5d89bb" -dependencies = [ - "anyhow", - "async-trait", - "clap 4.5.4", - "const_format", - "dirs", - "dotenvy", - "fuel-core", - "fuel-core-chain-config", - "fuel-core-types", - "hex", - "humantime", - "pyroscope", - "pyroscope_pprofrs", - "serde_json", - "tikv-jemallocator", - "tokio", - "tokio-util", - "tracing", - "tracing-subscriber", - "url", -] - -[[package]] -name = "fuel-core-chain-config" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d202fe1dfeb98882bdc5a0206a58e469d76fd09d952c4050bb979102bd690398" -dependencies = [ - "anyhow", - "bech32", - "derivative", - "fuel-core-storage", - "fuel-core-types", - "itertools 0.12.1", - "postcard", - "serde", - "serde_json", - "serde_with", - "tracing", -] - -[[package]] -name = "fuel-core-consensus-module" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f99179c08972efffe7628f0ff8d59028218b126347a6f9eba86f71e20966eeb" -dependencies = [ - "anyhow", - "fuel-core-chain-config", - "fuel-core-poa", - "fuel-core-storage", - "fuel-core-types", -] - -[[package]] -name = "fuel-core-database" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5b1fd08a72609ebf0c8106359a37a4b205055be15e9f4fc30a4c0b5f0644c6b" -dependencies = [ - "anyhow", - "derive_more", - "fuel-core-storage", - "fuel-core-types", -] - -[[package]] -name = "fuel-core-executor" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f98d89798007bc781d56e02681144683f5c645ee0725e7717e38694e8e5e31d" -dependencies = [ - "anyhow", - "fuel-core-storage", - "fuel-core-types", - "hex", - "parking_lot", - "serde", - "tracing", -] - -[[package]] -name = "fuel-core-importer" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51837a53f2d8b78a701aee61b99c7f1873f23e864f01f4b4d0644a06e1f7c41" -dependencies = [ - "anyhow", - "derive_more", - "fuel-core-metrics", - "fuel-core-storage", - "fuel-core-types", - "tokio", - "tokio-rayon", - "tracing", -] - -[[package]] -name = "fuel-core-metrics" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bacc62bc4fec2fe6a818a1a7145b892bd486d69266190ca8dd31a036a3a327b7" -dependencies = [ - "axum", - "once_cell", - "pin-project-lite", - "prometheus-client", - "regex", - "tracing", -] - -[[package]] -name = "fuel-streams-publisher" -version = "0.26.0" -dependencies = [ - "anyhow", - "async-nats", - "clap 4.5.4", - "fuel-core", - "fuel-core-bin", - "fuel-core-services", - "fuel-core-types", - "serde_json", - "tokio", - "tracing", -] - -[[package]] -name = "fuel-core-p2p" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6496068f0f5736f9e51bba8f8bb04cb83f68df2f6142e410fe62854b47621b3" -dependencies = [ - "anyhow", - "async-trait", - "fuel-core-chain-config", - "fuel-core-metrics", - "fuel-core-services", - "fuel-core-storage", - "fuel-core-types", - "futures", - "hex", - "ip_network", - "libp2p", - "libp2p-mplex", - "postcard", - "prometheus-client", - "quick-protobuf", - "quick-protobuf-codec", - "rand", - "serde", - "serde_with", - "sha2 0.10.8", - "thiserror", - "tokio", - "tracing", - "void", -] - -[[package]] -name = "fuel-core-poa" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d189ecd635688ddc896b44c8497b29c04bb4a3719a24eea0ca9691a6f76d5e" -dependencies = [ - "anyhow", - "async-trait", - "fuel-core-chain-config", - "fuel-core-services", - "fuel-core-storage", - "fuel-core-types", - "tokio", - "tokio-stream", - "tracing", -] - -[[package]] -name = "fuel-core-producer" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2901a7ba2c0e724bbb88a3111fdb9844f5faf9f0bd4005944f61f093730b4d" -dependencies = [ - "anyhow", - "async-trait", - "derive_more", - "fuel-core-storage", - "fuel-core-types", - "tokio", - "tokio-rayon", - "tracing", -] - -[[package]] -name = "fuel-core-relayer" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da1aa686ec4a05b77f4c4dfae68e6f17701b5bc21a49a8a505c6d9dc6dcf9183" -dependencies = [ - "anyhow", - "async-trait", - "enum-iterator", - "ethers-contract", - "ethers-core", - "ethers-providers", - "fuel-core-services", - "fuel-core-storage", - "fuel-core-types", - "futures", - "once_cell", - "strum 0.25.0", - "strum_macros 0.25.3", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "fuel-core-services" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2ab4d3931b8cafdb2e69fe8ca97918a168d74c73c070481ca0e552cc37bb93" -dependencies = [ - "anyhow", - "async-trait", - "fuel-core-metrics", - "futures", - "parking_lot", - "tokio", - "tracing", -] - -[[package]] -name = "fuel-core-storage" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e039c1c6ebef314c74c34728e1f2199dcf9ede041d6f5c6e11479517c8f4d320" -dependencies = [ - "anyhow", - "derive_more", - "enum-iterator", - "fuel-core-types", - "fuel-vm", - "impl-tools", - "itertools 0.12.1", - "num_enum", - "paste", - "postcard", - "primitive-types", - "serde", - "strum 0.25.0", - "strum_macros 0.25.3", -] - -[[package]] -name = "fuel-core-sync" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7059ee4c3dbf1e9340d7cd88eccf2e10c631306d8038ab20160e6434deb79c1b" -dependencies = [ - "anyhow", - "async-trait", - "fuel-core-services", - "fuel-core-types", - "futures", - "rand", - "tokio", - "tracing", -] - -[[package]] -name = "fuel-core-txpool" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985684e2d67d5018e9227a4f9ed79cac02b23b207e457ee95833ab047769c2ac" -dependencies = [ - "anyhow", - "async-trait", - "fuel-core-metrics", - "fuel-core-services", - "fuel-core-storage", - "fuel-core-types", - "futures", - "parking_lot", - "tokio", - "tokio-rayon", - "tokio-stream", - "tracing", -] - -[[package]] -name = "fuel-core-types" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf038dd8df8d3aa665a13295c9ef888ba8118600cccdf8fb4587410e0e102fdf" -dependencies = [ - "anyhow", - "bs58", - "derivative", - "derive_more", - "fuel-vm", - "secrecy", - "serde", - "tai64", - "thiserror", - "zeroize", -] - -[[package]] -name = "fuel-core-upgradable-executor" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc54c84a7dc13f76930761ebca391b167caa096dc2bdb2413b5a2400bf65f99d" -dependencies = [ - "anyhow", - "fuel-core-executor", - "fuel-core-storage", - "fuel-core-types", - "fuel-core-wasm-executor", - "parking_lot", - "postcard", - "tracing", - "wasmtime", -] - -[[package]] -name = "fuel-core-wasm-executor" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a201e4fa5f94efb36cda172947875483d90a88b060cb9c6f9a496313d171aae8" -dependencies = [ - "anyhow", - "fuel-core-executor", - "fuel-core-storage", - "fuel-core-types", - "postcard", - "serde", -] - -[[package]] -name = "fuel-crypto" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71cef93970fb8a26d3a683ae211833c6bbf391066887f501bd5859f29992b59a" -dependencies = [ - "coins-bip32", - "coins-bip39", - "ecdsa", - "ed25519-dalek", - "fuel-types", - "k256", - "lazy_static", - "p256", - "rand", - "secp256k1", - "serde", - "sha2 0.10.8", - "zeroize", -] - -[[package]] -name = "fuel-derive" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b85e8e508b26d088262075fcfe9921b7009c931fef1cc55fe1dafb116c99884" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", -] - -[[package]] -name = "fuel-merkle" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5198b4eab5a19b0034971da88199dae7dd61806ebd8df366d6af1f17cda2e151" -dependencies = [ - "derive_more", - "digest 0.10.7", - "fuel-storage", - "hashbrown 0.13.2", - "hex", - "serde", - "sha2 0.10.8", -] - -[[package]] -name = "fuel-storage" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa738e9c244f3f312af09faef108ec9a285f02afcefbc579c19c242cea742dd0" - -[[package]] -name = "fuel-tx" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e4b4ea79ffe711af7bbf363b25f383fc6e481e652cf55a5ef8b5a458fcf4ef9" -dependencies = [ - "bitflags 2.5.0", - "derivative", - "derive_more", - "fuel-asm", - "fuel-crypto", - "fuel-merkle", - "fuel-types", - "hashbrown 0.14.5", - "itertools 0.10.5", - "postcard", - "rand", - "serde", - "serde_json", - "strum 0.24.1", - "strum_macros 0.24.3", -] - -[[package]] -name = "fuel-types" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455cf5275d96f6907e81ed1825c4e6a9dd79f7c1c37a4e15134562f83024c7e7" -dependencies = [ - "fuel-derive", - "hex", - "serde", -] - -[[package]] -name = "fuel-vm" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8811f949db8ce61cc68dcf81644047df4ee23be55879efcfe9f1aa5adc378965" -dependencies = [ - "async-trait", - "backtrace", - "bitflags 2.5.0", - "derivative", - "derive_more", - "ethnum", - "fuel-asm", - "fuel-crypto", - "fuel-merkle", - "fuel-storage", - "fuel-tx", - "fuel-types", - "hashbrown 0.14.5", - "itertools 0.10.5", - "libm", - "paste", - "percent-encoding", - "primitive-types", - "serde", - "serde_with", - "sha3", - "static_assertions", - "strum 0.24.1", - "tai64", -] - -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - -[[package]] -name = "futures" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-bounded" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e2774cc104e198ef3d3e1ff4ab40f86fa3245d6cb6a3a46174f21463cee173" -dependencies = [ - "futures-timer", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" - -[[package]] -name = "futures-executor" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", - "num_cpus", -] - -[[package]] -name = "futures-io" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" - -[[package]] -name = "futures-lite" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" -dependencies = [ - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "futures-macro" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "futures-rustls" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28" -dependencies = [ - "futures-io", - "rustls 0.21.12", -] - -[[package]] -name = "futures-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" -dependencies = [ - "futures-io", - "rustls 0.23.8", - "rustls-pki-types", -] - -[[package]] -name = "futures-sink" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" - -[[package]] -name = "futures-task" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" - -[[package]] -name = "futures-ticker" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9763058047f713632a52e916cc7f6a4b3fc6e9fc1ff8c5b1dc49e5a89041682e" -dependencies = [ - "futures", - "futures-timer", - "instant", -] - -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" -dependencies = [ - "gloo-timers", - "send_wrapper 0.4.0", -] - -[[package]] -name = "futures-util" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", - "zeroize", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "ghash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" -dependencies = [ - "opaque-debug", - "polyval", -] - -[[package]] -name = "gimli" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -dependencies = [ - "fallible-iterator", - "indexmap 2.2.6", - "stable_deref_trait", -] - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core", - "subtle", -] - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.2.6", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hash32" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" -dependencies = [ - "byteorder", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", - "serde", -] - -[[package]] -name = "hashers" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2bca93b15ea5a746f220e56587f71e73c6165eab783df9e26590069953e3c30" -dependencies = [ - "fxhash", -] - -[[package]] -name = "heapless" -version = "0.7.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" -dependencies = [ - "atomic-polyfill", - "hash32", - "rustc_version", - "serde", - "spin 0.9.8", - "stable_deref_trait", -] - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -dependencies = [ - "serde", -] - -[[package]] -name = "hex_fmt" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" - -[[package]] -name = "hickory-proto" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.4.0", - "ipnet", - "once_cell", - "rand", - "socket2", - "thiserror", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "hickory-resolver" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" -dependencies = [ - "cfg-if", - "futures-util", - "hickory-proto", - "ipconfig", - "lru-cache", - "once_cell", - "parking_lot", - "rand", - "resolv-conf", - "smallvec", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac 0.12.1", -] - -[[package]] -name = "hmac" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" -dependencies = [ - "crypto-mac", - "digest 0.9.0", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "hmac-drbg" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" -dependencies = [ - "digest 0.9.0", - "generic-array", - "hmac 0.8.1", -] - -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "http-range-header" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.14.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http", - "hyper", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core 0.52.0", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "if-addrs" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cabb0019d51a643781ff15c9c8a3e5dedc365c47211270f4e8f82812fedd8f0a" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "if-watch" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b0422c86d7ce0e97169cc42e04ae643caf278874a7a3c87b8150a220dc7e1e" -dependencies = [ - "async-io", - "core-foundation", - "fnv", - "futures", - "if-addrs", - "ipnet", - "log", - "rtnetlink", - "system-configuration", - "tokio", - "windows", -] - -[[package]] -name = "igd-next" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064d90fec10d541084e7b39ead8875a5a80d9114a2b18791565253bae25f49e4" -dependencies = [ - "async-trait", - "attohttpc", - "bytes", - "futures", - "http", - "hyper", - "log", - "rand", - "tokio", - "url", - "xmltree", -] - -[[package]] -name = "impl-codec" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" -dependencies = [ - "parity-scale-codec", -] - -[[package]] -name = "impl-rlp" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" -dependencies = [ - "rlp", -] - -[[package]] -name = "impl-serde" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd" -dependencies = [ - "serde", -] - -[[package]] -name = "impl-tools" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82c305b1081f1a99fda262883c788e50ab57d36c00830bdd7e0a82894ad965c" -dependencies = [ - "autocfg", - "impl-tools-lib", - "proc-macro-error", - "syn 2.0.66", -] - -[[package]] -name = "impl-tools-lib" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85d3946d886eaab0702fa0c6585adcced581513223fa9df7ccfabbd9fa331a88" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "impl-trait-for-tuples" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "indenter" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" -dependencies = [ - "equivalent", - "hashbrown 0.14.5", - "serde", -] - -[[package]] -name = "indicatif" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" -dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic", - "unicode-width", -] - -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "ip_network" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2f047c0a98b2f299aa5d6d7088443570faae494e9ae1305e48be000c9e0eb1" - -[[package]] -name = "ipconfig" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" -dependencies = [ - "socket2", - "widestring", - "windows-sys 0.48.0", - "winreg", -] - -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - -[[package]] -name = "jobserver" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "json" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" - -[[package]] -name = "jsonwebtoken" -version = "8.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" -dependencies = [ - "base64 0.21.7", - "pem 1.1.1", - "ring 0.16.20", - "serde", - "serde_json", - "simple_asn1", -] - -[[package]] -name = "k256" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956ff9b67e26e1a6a866cb758f12c6f8746208489e3e4a4b5580802f2f0a587b" -dependencies = [ - "cfg-if", - "ecdsa", - "elliptic-curve", - "once_cell", - "sha2 0.10.8", - "signature", -] - -[[package]] -name = "keccak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" -dependencies = [ - "cpufeatures", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "leb128" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" - -[[package]] -name = "libc" -version = "0.2.155" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" - -[[package]] -name = "libflate" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" -dependencies = [ - "adler32", - "crc32fast", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" -dependencies = [ - "rle-decode-fast", -] - -[[package]] -name = "libloading" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" -dependencies = [ - "cfg-if", - "windows-targets 0.52.5", -] - -[[package]] -name = "libm" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" - -[[package]] -name = "libp2p" -version = "0.53.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681fb3f183edfbedd7a57d32ebe5dcdc0b9f94061185acf3c30249349cc6fc99" -dependencies = [ - "bytes", - "either", - "futures", - "futures-timer", - "getrandom", - "instant", - "libp2p-allow-block-list", - "libp2p-connection-limits", - "libp2p-core", - "libp2p-dns", - "libp2p-gossipsub", - "libp2p-identify", - "libp2p-identity", - "libp2p-kad", - "libp2p-mdns", - "libp2p-metrics", - "libp2p-noise", - "libp2p-quic", - "libp2p-request-response", - "libp2p-swarm", - "libp2p-tcp", - "libp2p-upnp", - "libp2p-websocket", - "libp2p-yamux", - "multiaddr", - "pin-project", - "rw-stream-sink", - "thiserror", -] - -[[package]] -name = "libp2p-allow-block-list" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "107b238b794cb83ab53b74ad5dcf7cca3200899b72fe662840cfb52f5b0a32e6" -dependencies = [ - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "void", -] - -[[package]] -name = "libp2p-connection-limits" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cd50a78ccfada14de94cbacd3ce4b0138157f376870f13d3a8422cd075b4fd" -dependencies = [ - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "void", -] - -[[package]] -name = "libp2p-core" -version = "0.41.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8130a8269e65a2554d55131c770bdf4bcd94d2b8d4efb24ca23699be65066c05" -dependencies = [ - "either", - "fnv", - "futures", - "futures-timer", - "instant", - "libp2p-identity", - "multiaddr", - "multihash", - "multistream-select", - "once_cell", - "parking_lot", - "pin-project", - "quick-protobuf", - "rand", - "rw-stream-sink", - "smallvec", - "thiserror", - "tracing", - "unsigned-varint 0.8.0", - "void", -] - -[[package]] -name = "libp2p-dns" -version = "0.41.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d17cbcf7160ff35c3e8e560de4a068fe9d6cb777ea72840e48eb76ff9576c4b6" -dependencies = [ - "async-trait", - "futures", - "hickory-resolver", - "libp2p-core", - "libp2p-identity", - "parking_lot", - "smallvec", - "tracing", -] - -[[package]] -name = "libp2p-gossipsub" -version = "0.46.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d665144a616dadebdc5fff186b1233488cdcd8bfb1223218ff084b6d052c94f7" -dependencies = [ - "asynchronous-codec 0.7.0", - "base64 0.21.7", - "byteorder", - "bytes", - "either", - "fnv", - "futures", - "futures-ticker", - "getrandom", - "hex_fmt", - "instant", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "prometheus-client", - "quick-protobuf", - "quick-protobuf-codec", - "rand", - "regex", - "sha2 0.10.8", - "smallvec", - "tracing", - "void", -] - -[[package]] -name = "libp2p-identify" -version = "0.44.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5d635ebea5ca0c3c3e77d414ae9b67eccf2a822be06091b9c1a0d13029a1e2f" -dependencies = [ - "asynchronous-codec 0.7.0", - "either", - "futures", - "futures-bounded", - "futures-timer", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "lru", - "quick-protobuf", - "quick-protobuf-codec", - "smallvec", - "thiserror", - "tracing", - "void", -] - -[[package]] -name = "libp2p-identity" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "999ec70441b2fb35355076726a6bc466c932e9bdc66f6a11c6c0aa17c7ab9be0" -dependencies = [ - "asn1_der", - "bs58", - "ed25519-dalek", - "hkdf", - "libsecp256k1", - "multihash", - "quick-protobuf", - "rand", - "sha2 0.10.8", - "thiserror", - "tracing", - "zeroize", -] - -[[package]] -name = "libp2p-kad" -version = "0.45.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc5767727d062c4eac74dd812c998f0e488008e82cce9c33b463d38423f9ad2" -dependencies = [ - "arrayvec", - "asynchronous-codec 0.7.0", - "bytes", - "either", - "fnv", - "futures", - "futures-bounded", - "futures-timer", - "instant", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "quick-protobuf", - "quick-protobuf-codec", - "rand", - "sha2 0.10.8", - "smallvec", - "thiserror", - "tracing", - "uint", - "void", -] - -[[package]] -name = "libp2p-mdns" -version = "0.45.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49007d9a339b3e1d7eeebc4d67c05dbf23d300b7d091193ec2d3f26802d7faf2" -dependencies = [ - "data-encoding", - "futures", - "hickory-proto", - "if-watch", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "rand", - "smallvec", - "socket2", - "tokio", - "tracing", - "void", -] - -[[package]] -name = "libp2p-metrics" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdac91ae4f291046a3b2660c039a2830c931f84df2ee227989af92f7692d3357" -dependencies = [ - "futures", - "instant", - "libp2p-core", - "libp2p-gossipsub", - "libp2p-identify", - "libp2p-identity", - "libp2p-kad", - "libp2p-swarm", - "pin-project", - "prometheus-client", -] - -[[package]] -name = "libp2p-mplex" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e895765e27e30217b25f7cb7ac4686dad1ff80bf2fdeffd1d898566900a924" -dependencies = [ - "asynchronous-codec 0.6.2", - "bytes", - "futures", - "libp2p-core", - "libp2p-identity", - "nohash-hasher", - "parking_lot", - "rand", - "smallvec", - "tracing", - "unsigned-varint 0.7.2", -] - -[[package]] -name = "libp2p-noise" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecd0545ce077f6ea5434bcb76e8d0fe942693b4380aaad0d34a358c2bd05793" -dependencies = [ - "asynchronous-codec 0.7.0", - "bytes", - "curve25519-dalek", - "futures", - "libp2p-core", - "libp2p-identity", - "multiaddr", - "multihash", - "once_cell", - "quick-protobuf", - "rand", - "sha2 0.10.8", - "snow", - "static_assertions", - "thiserror", - "tracing", - "x25519-dalek", - "zeroize", -] - -[[package]] -name = "libp2p-quic" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67296ad4e092e23f92aea3d2bdb6f24eab79c0929ed816dfb460ea2f4567d2b" -dependencies = [ - "bytes", - "futures", - "futures-timer", - "if-watch", - "libp2p-core", - "libp2p-identity", - "libp2p-tls", - "parking_lot", - "quinn", - "rand", - "ring 0.17.8", - "rustls 0.23.8", - "socket2", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "libp2p-request-response" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6946e5456240b3173187cc37a17cb40c3cd1f7138c76e2c773e0d792a42a8de1" -dependencies = [ - "async-trait", - "futures", - "futures-bounded", - "futures-timer", - "instant", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "rand", - "smallvec", - "tracing", - "void", -] - -[[package]] -name = "libp2p-swarm" -version = "0.44.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80cae6cb75f89dbca53862f9ebe0b9f463aa7b302762fcfaafb9e51dcc9b0f7e" -dependencies = [ - "either", - "fnv", - "futures", - "futures-timer", - "instant", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm-derive", - "lru", - "multistream-select", - "once_cell", - "rand", - "smallvec", - "tokio", - "tracing", - "void", -] - -[[package]] -name = "libp2p-swarm-derive" -version = "0.34.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5daceb9dd908417b6dfcfe8e94098bc4aac54500c282e78120b885dadc09b999" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "libp2p-tcp" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2460fc2748919adff99ecbc1aab296e4579e41f374fb164149bd2c9e529d4c" -dependencies = [ - "futures", - "futures-timer", - "if-watch", - "libc", - "libp2p-core", - "libp2p-identity", - "socket2", - "tokio", - "tracing", -] - -[[package]] -name = "libp2p-tls" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "251b17aebdd29df7e8f80e4d94b782fae42e934c49086e1a81ba23b60a8314f2" -dependencies = [ - "futures", - "futures-rustls 0.26.0", - "libp2p-core", - "libp2p-identity", - "rcgen", - "ring 0.17.8", - "rustls 0.23.8", - "rustls-webpki 0.101.7", - "thiserror", - "x509-parser", - "yasna", -] - -[[package]] -name = "libp2p-upnp" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccf04b0e3ff3de52d07d5fd6c3b061d0e7f908ffc683c32d9638caedce86fc8" -dependencies = [ - "futures", - "futures-timer", - "igd-next", - "libp2p-core", - "libp2p-swarm", - "tokio", - "tracing", - "void", -] - -[[package]] -name = "libp2p-websocket" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4846d51afd08180e164291c3754ba30dd4fbac6fac65571be56403c16431a5e" -dependencies = [ - "either", - "futures", - "futures-rustls 0.24.0", - "libp2p-core", - "libp2p-identity", - "parking_lot", - "pin-project-lite", - "rw-stream-sink", - "soketto", - "tracing", - "url", - "webpki-roots", -] - -[[package]] -name = "libp2p-yamux" -version = "0.45.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200cbe50349a44760927d50b431d77bed79b9c0a3959de1af8d24a63434b71e5" -dependencies = [ - "either", - "futures", - "libp2p-core", - "thiserror", - "tracing", - "yamux 0.12.1", - "yamux 0.13.2", -] - -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags 2.5.0", - "libc", -] - -[[package]] -name = "librocksdb-sys" -version = "0.11.0+8.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" -dependencies = [ - "bindgen", - "bzip2-sys", - "cc", - "glob", - "libc", - "libz-sys", - "lz4-sys", -] - -[[package]] -name = "libsecp256k1" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b09eff1b35ed3b33b877ced3a691fc7a481919c7e29c53c906226fcf55e2a1" -dependencies = [ - "arrayref", - "base64 0.13.1", - "digest 0.9.0", - "hmac-drbg", - "libsecp256k1-core", - "libsecp256k1-gen-ecmult", - "libsecp256k1-gen-genmult", - "rand", - "serde", - "sha2 0.9.9", - "typenum", -] - -[[package]] -name = "libsecp256k1-core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be9b9bb642d8522a44d533eab56c16c738301965504753b03ad1de3425d5451" -dependencies = [ - "crunchy", - "digest 0.9.0", - "subtle", -] - -[[package]] -name = "libsecp256k1-gen-ecmult" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3038c808c55c87e8a172643a7d87187fc6c4174468159cb3090659d55bcb4809" -dependencies = [ - "libsecp256k1-core", -] - -[[package]] -name = "libsecp256k1-gen-genmult" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db8d6ba2cec9eacc40e6e8ccc98931840301f1006e95647ceb2dd5c3aa06f7c" -dependencies = [ - "libsecp256k1-core", -] - -[[package]] -name = "libz-sys" -version = "1.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - -[[package]] -name = "linux-raw-sys" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" - -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" - -[[package]] -name = "lru" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" -dependencies = [ - "hashbrown 0.14.5", -] - -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - -[[package]] -name = "lz4-sys" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - -[[package]] -name = "matchit" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" - -[[package]] -name = "memchr" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" - -[[package]] -name = "memfd" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" -dependencies = [ - "rustix", -] - -[[package]] -name = "memmap2" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" -dependencies = [ - "libc", -] - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - -[[package]] -name = "multer" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "log", - "memchr", - "mime", - "spin 0.9.8", - "version_check", -] - -[[package]] -name = "multiaddr" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b852bc02a2da5feed68cd14fa50d0774b92790a5bdbfa932a813926c8472070" -dependencies = [ - "arrayref", - "byteorder", - "data-encoding", - "libp2p-identity", - "multibase", - "multihash", - "percent-encoding", - "serde", - "static_assertions", - "unsigned-varint 0.7.2", - "url", -] - -[[package]] -name = "multibase" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" -dependencies = [ - "base-x", - "data-encoding", - "data-encoding-macro", -] - -[[package]] -name = "multihash" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" -dependencies = [ - "core2", - "unsigned-varint 0.7.2", -] - -[[package]] -name = "multistream-select" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" -dependencies = [ - "bytes", - "futures", - "log", - "pin-project", - "smallvec", - "unsigned-varint 0.7.2", -] - -[[package]] -name = "names" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bddcd3bf5144b6392de80e04c347cd7fab2508f6df16a85fc496ecd5cec39bc" -dependencies = [ - "clap 3.2.25", - "rand", -] - -[[package]] -name = "netlink-packet-core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297" -dependencies = [ - "anyhow", - "byteorder", - "libc", - "netlink-packet-utils", -] - -[[package]] -name = "netlink-packet-route" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab" -dependencies = [ - "anyhow", - "bitflags 1.3.2", - "byteorder", - "libc", - "netlink-packet-core", - "netlink-packet-utils", -] - -[[package]] -name = "netlink-packet-utils" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" -dependencies = [ - "anyhow", - "byteorder", - "paste", - "thiserror", -] - -[[package]] -name = "netlink-proto" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" -dependencies = [ - "bytes", - "futures", - "log", - "netlink-packet-core", - "netlink-sys", - "thiserror", - "tokio", -] - -[[package]] -name = "netlink-sys" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416060d346fbaf1f23f9512963e3e878f1a78e707cb699ba9215761754244307" -dependencies = [ - "bytes", - "futures", - "libc", - "log", - "tokio", -] - -[[package]] -name = "nix" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", -] - -[[package]] -name = "nix" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", -] - -[[package]] -name = "nkeys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc522a19199a0795776406619aa6aa78e1e55690fbeb3181b8db5265fd0e89ce" -dependencies = [ - "data-encoding", - "ed25519", - "ed25519-dalek", - "getrandom", - "log", - "rand", - "signatory", -] - -[[package]] -name = "nohash-hasher" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - -[[package]] -name = "nuid" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83" -dependencies = [ - "rand", -] - -[[package]] -name = "num-bigint" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi 0.3.9", - "libc", -] - -[[package]] -name = "num_enum" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" -dependencies = [ - "num_enum_derive", -] - -[[package]] -name = "num_enum_derive" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" -dependencies = [ - "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - -[[package]] -name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "crc32fast", - "hashbrown 0.14.5", - "indexmap 2.2.6", - "memchr", -] - -[[package]] -name = "oid-registry" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" -dependencies = [ - "asn1-rs", -] - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - -[[package]] -name = "open-fastrlp" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786393f80485445794f6043fd3138854dd109cc6c4bd1a6383db304c9ce9b9ce" -dependencies = [ - "arrayvec", - "auto_impl", - "bytes", - "ethereum-types", - "open-fastrlp-derive", -] - -[[package]] -name = "open-fastrlp-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "003b2be5c6c53c1cfeb0a238b8a1c3915cd410feb684457a36c10038f764bb1c" -dependencies = [ - "bytes", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2 0.10.8", -] - -[[package]] -name = "parity-scale-codec" -version = "3.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" -dependencies = [ - "arrayvec", - "bitvec", - "byte-slice-cast", - "impl-trait-for-tuples", - "parity-scale-codec-derive", - "serde", -] - -[[package]] -name = "parity-scale-codec-derive" -version = "3.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" -dependencies = [ - "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "parking" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" - -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.52.5", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest 0.10.7", - "hmac 0.12.1", -] - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - -[[package]] -name = "pem" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" -dependencies = [ - "base64 0.22.1", - "serde", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pest" -version = "2.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" -dependencies = [ - "memchr", - "thiserror", - "ucd-trie", -] - -[[package]] -name = "pharos" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" -dependencies = [ - "futures", - "rustc_version", -] - -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - -[[package]] -name = "platforms" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" - -[[package]] -name = "polling" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645493cf344456ef24219d02a768cf1fb92ddf8c92161679ae3d91b91a637be3" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi 0.3.9", - "pin-project-lite", - "rustix", - "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "poly1305" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" -dependencies = [ - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - -[[package]] -name = "postcard" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" -dependencies = [ - "cobs", - "embedded-io", - "heapless", - "serde", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "pprof" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978385d59daf9269189d052ca8a84c1acfd0715c0599a5d5188d4acc078ca46a" -dependencies = [ - "backtrace", - "cfg-if", - "findshlibs", - "libc", - "log", - "nix 0.26.4", - "once_cell", - "parking_lot", - "smallvec", - "symbolic-demangle", - "tempfile", - "thiserror", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "prettyplease" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" -dependencies = [ - "proc-macro2", - "syn 2.0.66", -] - -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - -[[package]] -name = "primitive-types" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" -dependencies = [ - "fixed-hash", - "impl-codec", - "impl-rlp", - "impl-serde", - "scale-info", - "uint", -] - -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - -[[package]] -name = "proc-macro-crate" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" -dependencies = [ - "toml_edit 0.21.1", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "1.0.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prometheus-client" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ca959da22a332509f2a73ae9e5f23f9dcfc31fd3a54d71f159495bd5909baa" -dependencies = [ - "dtoa", - "itoa", - "parking_lot", - "prometheus-client-derive-encode", -] - -[[package]] -name = "prometheus-client-derive-encode" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "proptest" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" -dependencies = [ - "bitflags 2.5.0", - "lazy_static", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax 0.8.3", - "unarray", -] - -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools 0.10.5", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "psm" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" -dependencies = [ - "cc", -] - -[[package]] -name = "pyroscope" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8a53ce01af1087eaeee6ce7c4fbf50ea4040ab1825c0115c4bafa039644ba9" -dependencies = [ - "json", - "libc", - "libflate", - "log", - "names", - "prost", - "reqwest", - "thiserror", - "url", - "winapi", -] - -[[package]] -name = "pyroscope_pprofrs" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f010b2a981a7f8449a650f25f309e520b5206ea2d89512dcb146aaa5518ff4" -dependencies = [ - "log", - "pprof", - "pyroscope", - "thiserror", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quick-protobuf" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" -dependencies = [ - "byteorder", -] - -[[package]] -name = "quick-protobuf-codec" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474" -dependencies = [ - "asynchronous-codec 0.7.0", - "bytes", - "quick-protobuf", - "thiserror", - "unsigned-varint 0.8.0", -] - -[[package]] -name = "quinn" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904e3d3ba178131798c6d9375db2b13b34337d489b089fc5ba0825a2ff1bee73" -dependencies = [ - "bytes", - "futures-io", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls 0.23.8", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "quinn-proto" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e974563a4b1c2206bbc61191ca4da9c22e4308b4c455e8906751cc7828393f08" -dependencies = [ - "bytes", - "rand", - "ring 0.17.8", - "rustc-hash", - "rustls 0.23.8", - "slab", - "thiserror", - "tinyvec", - "tracing", -] - -[[package]] -name = "quinn-udp" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f0def2590301f4f667db5a77f9694fb004f82796dc1a8b1508fafa3d0e8b72" -dependencies = [ - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "quote" -version = "1.0.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rand_xorshift" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" -dependencies = [ - "rand_core", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rcgen" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6" -dependencies = [ - "pem 3.0.4", - "ring 0.16.20", - "time", - "yasna", -] - -[[package]] -name = "redox_syscall" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" -dependencies = [ - "bitflags 2.5.0", -] - -[[package]] -name = "redox_users" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" -dependencies = [ - "getrandom", - "libredox", - "thiserror", -] - -[[package]] -name = "regalloc2" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6" -dependencies = [ - "hashbrown 0.13.2", - "log", - "rustc-hash", - "slice-group-by", - "smallvec", -] - -[[package]] -name = "regex" -version = "1.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - -[[package]] -name = "regex-automata" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax 0.8.3", -] - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" - -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls 0.21.12", - "rustls-pemfile 1.0.4", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-rustls 0.24.1", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", - "winreg", -] - -[[package]] -name = "resolv-conf" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" -dependencies = [ - "hostname", - "quick-error", -] - -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys 0.52.0", -] - -[[package]] -name = "ripemd" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" -dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - -[[package]] -name = "rlp" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" -dependencies = [ - "bytes", - "rlp-derive", - "rustc-hex", -] - -[[package]] -name = "rlp-derive" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33d7b2abe0c340d8797fe2907d3f20d3b5ea5908683618bfe80df7f621f672a" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "rocksdb" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" -dependencies = [ - "libc", - "librocksdb-sys", -] - -[[package]] -name = "rtnetlink" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0" -dependencies = [ - "futures", - "log", - "netlink-packet-route", - "netlink-proto", - "nix 0.24.3", - "thiserror", - "tokio", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc-hex" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" - -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - -[[package]] -name = "rustix" -version = "0.38.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" -dependencies = [ - "bitflags 2.5.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", -] - -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring 0.17.8", - "rustls-webpki 0.101.7", - "sct", -] - -[[package]] -name = "rustls" -version = "0.23.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79adb16721f56eb2d843e67676896a61ce7a0fa622dc18d3e372477a029d2740" -dependencies = [ - "once_cell", - "ring 0.17.8", - "rustls-pki-types", - "rustls-webpki 0.102.4", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" -dependencies = [ - "openssl-probe", - "rustls-pemfile 2.1.2", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - -[[package]] -name = "rustls-pemfile" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" -dependencies = [ - "base64 0.22.1", - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", -] - -[[package]] -name = "rustls-webpki" -version = "0.102.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" -dependencies = [ - "ring 0.17.8", - "rustls-pki-types", - "untrusted 0.9.0", -] - -[[package]] -name = "rustversion" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" - -[[package]] -name = "rw-stream-sink" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1" -dependencies = [ - "futures", - "pin-project", - "static_assertions", -] - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "scale-info" -version = "2.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca070c12893629e2cc820a9761bedf6ce1dcddc9852984d1dc734b8bd9bd024" -dependencies = [ - "cfg-if", - "derive_more", - "parity-scale-codec", - "scale-info-derive", -] - -[[package]] -name = "scale-info-derive" -version = "2.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d35494501194174bda522a32605929eefc9ecf7e0a326c26db1fdd85881eb62" -dependencies = [ - "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "schannel" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", -] - -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", - "subtle", - "zeroize", -] - -[[package]] -name = "secp256k1" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4124a35fe33ae14259c490fd70fa199a32b9ce9502f2ee6bc4f81ec06fa65894" -dependencies = [ - "rand", - "secp256k1-sys", -] - -[[package]] -name = "secp256k1-sys" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" -dependencies = [ - "cc", -] - -[[package]] -name = "secrecy" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" -dependencies = [ - "zeroize", -] - -[[package]] -name = "security-framework" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" -dependencies = [ - "bitflags 2.5.0", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" -dependencies = [ - "serde", -] - -[[package]] -name = "send_wrapper" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" - -[[package]] -name = "send_wrapper" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" - -[[package]] -name = "serde" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "serde_json" -version = "1.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_nanos" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a93142f0367a4cc53ae0fead1bcda39e85beccfad3dcd717656cacab94b12985" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_repr" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "serde_spanned" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" -dependencies = [ - "base64 0.22.1", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.2.6", - "serde", - "serde_derive", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" -dependencies = [ - "darling 0.20.9", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "sha-1" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha3" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" -dependencies = [ - "digest 0.10.7", - "keccak", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - -[[package]] -name = "signatory" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" -dependencies = [ - "pkcs8", - "rand_core", - "signature", - "zeroize", -] - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest 0.10.7", - "rand_core", -] - -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror", - "time", -] - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "slice-group-by" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "snow" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "850948bee068e713b8ab860fe1adc4d109676ab4c3b621fd8147f06b261f2f85" -dependencies = [ - "aes-gcm", - "blake2", - "chacha20poly1305", - "curve25519-dalek", - "rand_core", - "ring 0.17.8", - "rustc_version", - "sha2 0.10.8", - "subtle", -] - -[[package]] -name = "socket2" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "soketto" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" -dependencies = [ - "base64 0.13.1", - "bytes", - "futures", - "httparse", - "log", - "rand", - "sha-1", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sptr" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" -dependencies = [ - "strum_macros 0.24.3", -] - -[[package]] -name = "strum" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" -dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" -dependencies = [ - "strum_macros 0.26.2", -] - -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.66", -] - -[[package]] -name = "strum_macros" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.66", -] - -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - -[[package]] -name = "symbolic-common" -version = "12.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cccfffbc6bb3bb2d3a26cd2077f4d055f6808d266f9d4d158797a4c60510dfe" -dependencies = [ - "debugid", - "memmap2", - "stable_deref_trait", - "uuid", -] - -[[package]] -name = "symbolic-demangle" -version = "12.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a99812da4020a67e76c4eb41f08c87364c14170495ff780f30dd519c221a68" -dependencies = [ - "cpp_demangle", - "rustc-demangle", - "symbolic-common", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tai64" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed7401421025f4132e6c1f7af5e7f8287383969f36e6628016cd509b8d3da9dc" -dependencies = [ - "serde", -] - -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "target-lexicon" -version = "0.12.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" - -[[package]] -name = "tempfile" -version = "3.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" -dependencies = [ - "cfg-if", - "fastrand", - "rustix", - "windows-sys 0.52.0", -] - -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" - -[[package]] -name = "thiserror" -version = "1.0.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "thread_local" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" -dependencies = [ - "cfg-if", - "once_cell", -] - -[[package]] -name = "tikv-jemalloc-sys" -version = "0.5.4+5.3.0-patched" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "tikv-jemallocator" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" -dependencies = [ - "libc", - "tikv-jemalloc-sys", -] - -[[package]] -name = "time" -version = "0.3.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" - -[[package]] -name = "time-macros" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "tokio-rayon" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cf33a76e0b1dd03b778f83244137bd59887abf25c0e87bc3e7071105f457693" -dependencies = [ - "rayon", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" -dependencies = [ - "rustls 0.23.8", - "rustls-pki-types", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" -dependencies = [ - "futures-util", - "log", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", - "tungstenite", - "webpki-roots", -] - -[[package]] -name = "tokio-util" -version = "0.7.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml" -version = "0.8.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.22.13", -] - -[[package]] -name = "toml_datetime" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c" -dependencies = [ - "indexmap 2.2.6", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.6.9", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" -dependencies = [ - "bitflags 1.3.2", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tokio", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "futures", - "futures-task", - "pin-project", - "tracing", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-serde" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex", - "serde", - "serde_json", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", - "tracing-serde", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tryhard" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9f0a709784e86923586cff0d872dba54cd2d2e116b3bc57587d15737cfce9d" -dependencies = [ - "futures", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tungstenite" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand", - "rustls 0.21.12", - "sha1", - "thiserror", - "url", - "utf-8", -] - -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - -[[package]] -name = "ucd-trie" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" - -[[package]] -name = "uint" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" -dependencies = [ - "byteorder", - "crunchy", - "hex", - "static_assertions", -] - -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-width" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" - -[[package]] -name = "unicode-xid" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" - -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common", - "subtle", -] - -[[package]] -name = "unsigned-varint" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" -dependencies = [ - "asynchronous-codec 0.6.2", - "bytes", -] - -[[package]] -name = "unsigned-varint" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" -dependencies = [ - "form_urlencoded", - "idna 0.5.0", - "percent-encoding", -] - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - -[[package]] -name = "uuid" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" -dependencies = [ - "getrandom", -] - -[[package]] -name = "valuable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.66", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" - -[[package]] -name = "wasm-encoder" -version = "0.41.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "972f97a5d8318f908dded23594188a90bcd09365986b1163e66d70170e5287ae" -dependencies = [ - "leb128", -] - -[[package]] -name = "wasmparser" -version = "0.121.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" -dependencies = [ - "bitflags 2.5.0", - "indexmap 2.2.6", - "semver", -] - -[[package]] -name = "wasmtime" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69472708b96ee90579a482bdbb908ce97e53a9e5ebbcab59cc29c3977bcab512" -dependencies = [ - "anyhow", - "bincode", - "bumpalo", - "cfg-if", - "gimli", - "indexmap 2.2.6", - "libc", - "log", - "object", - "once_cell", - "paste", - "rayon", - "rustix", - "serde", - "serde_derive", - "serde_json", - "target-lexicon", - "wasmparser", - "wasmtime-cache", - "wasmtime-cranelift", - "wasmtime-environ", - "wasmtime-jit-icache-coherence", - "wasmtime-runtime", - "windows-sys 0.52.0", -] - -[[package]] -name = "wasmtime-asm-macros" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86292d6a9bf30c669582a40c4a4b8e0b8640e951f3635ee8e0acf7f87809961e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "wasmtime-cache" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a180017db1233c902b992fea9484640d265f2fedf03db60eed57894cb2effcc" -dependencies = [ - "anyhow", - "base64 0.21.7", - "bincode", - "directories-next", - "log", - "rustix", - "serde", - "serde_derive", - "sha2 0.10.8", - "toml 0.5.11", - "windows-sys 0.52.0", - "zstd", -] - -[[package]] -name = "wasmtime-cranelift" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b57d58e220ae223855c5d030ef20753377bc716d0c81b34c1fe74c9f44268774" -dependencies = [ - "anyhow", - "cfg-if", - "cranelift-codegen", - "cranelift-control", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", - "gimli", - "log", - "object", - "target-lexicon", - "thiserror", - "wasmparser", - "wasmtime-cranelift-shared", - "wasmtime-environ", - "wasmtime-versioned-export-macros", -] - -[[package]] -name = "wasmtime-cranelift-shared" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba2cfdfdbde42f0f3baeddb62f3555524dee9f836c96da8d466e299f75f5eee" -dependencies = [ - "anyhow", - "cranelift-codegen", - "cranelift-control", - "cranelift-native", - "gimli", - "object", - "target-lexicon", - "wasmtime-environ", -] - -[[package]] -name = "wasmtime-environ" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abbf3075d9ee7eb1263dc67949aced64d0f0bf27be8098d34d8e5826cf0ff0f2" -dependencies = [ - "anyhow", - "bincode", - "cranelift-entity", - "gimli", - "indexmap 2.2.6", - "log", - "object", - "serde", - "serde_derive", - "target-lexicon", - "thiserror", - "wasmparser", - "wasmtime-types", -] - -[[package]] -name = "wasmtime-jit-icache-coherence" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dacd2aa30fb20fd8cd0eb4e664024a1ab28a02958529fa05bf52117532a098fc" -dependencies = [ - "cfg-if", - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "wasmtime-runtime" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d14e97c4bb36d91bcdd194745446d595e67ce8b89916806270fdbee640c747fd" -dependencies = [ - "anyhow", - "cc", - "cfg-if", - "indexmap 2.2.6", - "libc", - "log", - "mach", - "memfd", - "memoffset", - "paste", - "psm", - "rustix", - "sptr", - "wasm-encoder", - "wasmtime-asm-macros", - "wasmtime-environ", - "wasmtime-versioned-export-macros", - "wasmtime-wmemcheck", - "windows-sys 0.52.0", -] - -[[package]] -name = "wasmtime-types" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "530b94c627a454d24f520173d3145112d1b807c44c82697a57e1d8e28390cde4" -dependencies = [ - "cranelift-entity", - "serde", - "serde_derive", - "thiserror", - "wasmparser", -] - -[[package]] -name = "wasmtime-versioned-export-macros" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5399c175ddba4a471b9da45105dea3493059d52b2d54860eadb0df04c813948d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "wasmtime-wmemcheck" -version = "18.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1711f429111e782fac0537e0b3eb2ab6f821613cf1ec3013f2a0ff3fde41745" - -[[package]] -name = "web-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - -[[package]] -name = "widestring" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.51.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" -dependencies = [ - "windows-core 0.51.1", - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-core" -version = "0.51.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" - -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" -dependencies = [ - "memchr", -] - -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "ws_stream_wasm" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5" -dependencies = [ - "async_io_stream", - "futures", - "js-sys", - "log", - "pharos", - "rustc_version", - "send_wrapper 0.6.0", - "thiserror", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - -[[package]] -name = "x25519-dalek" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" -dependencies = [ - "curve25519-dalek", - "rand_core", - "serde", - "zeroize", -] - -[[package]] -name = "x509-parser" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" -dependencies = [ - "asn1-rs", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "rusticata-macros", - "thiserror", - "time", -] - -[[package]] -name = "xml-rs" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" - -[[package]] -name = "xmltree" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" -dependencies = [ - "xml-rs", -] - -[[package]] -name = "yamux" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed0164ae619f2dc144909a9f082187ebb5893693d8c0196e8085283ccd4b776" -dependencies = [ - "futures", - "log", - "nohash-hasher", - "parking_lot", - "pin-project", - "rand", - "static_assertions", -] - -[[package]] -name = "yamux" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f97202f6b125031b95d83e01dc57292b529384f80bfae4677e4bbc10178cf72" -dependencies = [ - "futures", - "instant", - "log", - "nohash-hasher", - "parking_lot", - "pin-project", - "rand", - "static_assertions", -] - -[[package]] -name = "yasna" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" -dependencies = [ - "time", -] - -[[package]] -name = "zerocopy" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/crates/fuel-streams-publisher/Cargo.toml b/crates/fuel-streams-publisher/Cargo.toml index 3982fc93..fb19b174 100644 --- a/crates/fuel-streams-publisher/Cargo.toml +++ b/crates/fuel-streams-publisher/Cargo.toml @@ -32,6 +32,7 @@ fuel-core-storage = { workspace = true } fuel-core-types = { workspace = true } fuel-streams = { workspace = true, features = ["test-helpers"] } fuel-streams-core = { workspace = true, features = ["test-helpers"] } +fuel-streams-storage = { workspace = true, features = ["test-helpers"] } futures = { workspace = true } num_cpus = "1.16" parking_lot = { version = "0.12", features = ["serde"] } diff --git a/crates/fuel-streams-publisher/src/cli.rs b/crates/fuel-streams-publisher/src/cli.rs index 92d5765c..7363eb81 100644 --- a/crates/fuel-streams-publisher/src/cli.rs +++ b/crates/fuel-streams-publisher/src/cli.rs @@ -9,15 +9,6 @@ use clap::Parser; /// - `fuel_core_config`: Configuration for the Fuel Core service, parsed using a flattened command. #[derive(Clone, Parser)] pub struct Cli { - /// Fuel Network to connect to. - #[arg( - long, - value_name = "NATS_URL", - env = "NATS_URL", - default_value = "localhost:4222", - help = "NATS URL to connect to." - )] - pub nats_url: String, /// Flattened command structure for Fuel Core configuration. #[command(flatten)] pub fuel_core_config: fuel_core_bin::cli::run::Command, diff --git a/crates/fuel-streams-publisher/src/main.rs b/crates/fuel-streams-publisher/src/main.rs index db5c5366..2d2a1602 100644 --- a/crates/fuel-streams-publisher/src/main.rs +++ b/crates/fuel-streams-publisher/src/main.rs @@ -27,7 +27,6 @@ async fn main() -> anyhow::Result<()> { let publisher = fuel_streams_publisher::Publisher::new( Arc::clone(&fuel_core), - cli.nats_url, telemetry.clone(), ) .await?; diff --git a/crates/fuel-streams-publisher/src/publisher/fuel_streams.rs b/crates/fuel-streams-publisher/src/publisher/fuel_streams.rs index 6900c82a..05f9fd0f 100644 --- a/crates/fuel-streams-publisher/src/publisher/fuel_streams.rs +++ b/crates/fuel-streams-publisher/src/publisher/fuel_streams.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use async_nats::{jetstream::stream::State as StreamState, RequestErrorKind}; use fuel_streams::types::Log; use fuel_streams_core::prelude::*; @@ -14,17 +16,20 @@ pub struct FuelStreams { pub logs: Stream, } -#[cfg_attr(test, mockall::automock)] impl FuelStreams { - pub async fn new(nats_client: &NatsClient) -> Self { + pub async fn new( + nats_client: &NatsClient, + s3_client: &Arc, + ) -> Self { Self { - transactions: Stream::::new(nats_client).await, - blocks: Stream::::new(nats_client).await, - inputs: Stream::::new(nats_client).await, - outputs: Stream::::new(nats_client).await, - receipts: Stream::::new(nats_client).await, - utxos: Stream::::new(nats_client).await, - logs: Stream::::new(nats_client).await, + transactions: Stream::::new(nats_client, s3_client) + .await, + blocks: Stream::::new(nats_client, s3_client).await, + inputs: Stream::::new(nats_client, s3_client).await, + outputs: Stream::::new(nats_client, s3_client).await, + receipts: Stream::::new(nats_client, s3_client).await, + utxos: Stream::::new(nats_client, s3_client).await, + logs: Stream::::new(nats_client, s3_client).await, } } } diff --git a/crates/fuel-streams-publisher/src/publisher/mod.rs b/crates/fuel-streams-publisher/src/publisher/mod.rs index 4a7df81f..9666053a 100644 --- a/crates/fuel-streams-publisher/src/publisher/mod.rs +++ b/crates/fuel-streams-publisher/src/publisher/mod.rs @@ -12,6 +12,7 @@ use blocks_streams::build_blocks_stream; pub use fuel_core_like::{FuelCore, FuelCoreLike}; pub use fuel_streams::{FuelStreams, FuelStreamsExt}; use fuel_streams_core::prelude::*; +use fuel_streams_storage::S3Client; use futures::{future::try_join_all, stream::FuturesUnordered, StreamExt}; use tokio::sync::Semaphore; @@ -28,18 +29,22 @@ pub struct Publisher { pub nats_client: NatsClient, pub fuel_streams: Arc, pub telemetry: Arc, + pub s3_client: Arc, } impl Publisher { pub async fn new( fuel_core: Arc, - nats_url: String, telemetry: Arc, ) -> anyhow::Result { - let nats_client_opts = - NatsClientOpts::admin_opts(None).with_custom_url(nats_url); + let nats_client_opts = NatsClientOpts::admin_opts(); let nats_client = NatsClient::connect(&nats_client_opts).await?; - let fuel_streams = Arc::new(FuelStreams::new(&nats_client).await); + + let s3_client_opts = S3ClientOpts::admin_opts(); + let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); + + let fuel_streams = + Arc::new(FuelStreams::new(&nats_client, &s3_client).await); telemetry.record_streams_count( fuel_core.chain_id(), @@ -51,6 +56,7 @@ impl Publisher { fuel_streams, nats_client, telemetry, + s3_client, }) } @@ -60,15 +66,19 @@ impl Publisher { } #[cfg(feature = "test-helpers")] - pub async fn default( + pub async fn new_for_testing( nats_client: &NatsClient, + s3_client: &Arc, fuel_core: Arc, ) -> anyhow::Result { Ok(Publisher { fuel_core, - fuel_streams: Arc::new(FuelStreams::new(nats_client).await), + fuel_streams: Arc::new( + FuelStreams::new(nats_client, s3_client).await, + ), nats_client: nats_client.clone(), telemetry: Telemetry::new().await?, + s3_client: Arc::clone(s3_client), }) } @@ -185,6 +195,7 @@ impl Publisher { let fuel_streams = &*self.fuel_streams; let blocks_stream = Arc::new(fuel_streams.blocks().to_owned()); + let opts = &Arc::new(PublishOpts { semaphore, chain_id, @@ -246,15 +257,15 @@ pub fn publish( opts: &Arc, ) -> JoinHandle> { let opts = Arc::clone(opts); - let payload = Arc::clone(&packet.payload); - let subject = Arc::clone(&packet.subject); + let packet = packet.clone(); let telemetry = Arc::clone(&opts.telemetry); let wildcard = packet.subject.wildcard(); tokio::spawn(async move { let _permit = opts.semaphore.acquire().await?; - match stream.publish(&*subject, &payload).await { + // Publish to NATS + match stream.publish(&packet).await { Ok(published_data_size) => { telemetry.log_info(&format!( "Successfully published for stream: {}", diff --git a/crates/fuel-streams-publisher/src/server/http.rs b/crates/fuel-streams-publisher/src/server/http.rs index 32d0ec73..50933e3d 100644 --- a/crates/fuel-streams-publisher/src/server/http.rs +++ b/crates/fuel-streams-publisher/src/server/http.rs @@ -83,13 +83,8 @@ mod tests { let telemetry = Telemetry::new().await.unwrap(); let fuel_core = FuelCore::from(fuel_service); - let publisher = Publisher::new( - fuel_core.arc(), - "nats://localhost:4222".to_string(), - telemetry, - ) - .await - .unwrap(); + let publisher = + Publisher::new(fuel_core.arc(), telemetry).await.unwrap(); let state = ServerState::new(publisher).await; assert!(state.publisher.nats_client.is_connected()); diff --git a/crates/fuel-streams-storage/Cargo.toml b/crates/fuel-streams-storage/Cargo.toml new file mode 100644 index 00000000..c8478563 --- /dev/null +++ b/crates/fuel-streams-storage/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "fuel-streams-storage" +description = "strategies and adapters for storing fuel streams in transient and file storage systems (i.e. NATS and S3)" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +async-nats = { workspace = true } +aws-config = { version = "1.5.10", features = ["behavior-version-latest"] } +aws-sdk-s3 = "1.65.0" +aws-smithy-runtime-api = "1.7.3" +aws-smithy-types = "=1.2.9" +displaydoc = { workspace = true } +dotenvy = { workspace = true } +fuel-networks = { workspace = true } +rand = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } + +[features] +default = [] +test-helpers = [] +bench-helpers = [] diff --git a/crates/fuel-streams-storage/src/lib.rs b/crates/fuel-streams-storage/src/lib.rs new file mode 100644 index 00000000..15f3b0ec --- /dev/null +++ b/crates/fuel-streams-storage/src/lib.rs @@ -0,0 +1,7 @@ +// TODO: Introduce Adapters for Transient and FileStorage (NATS and S3 clients would implement those) + +pub mod nats; +pub mod s3; + +pub use nats::*; +pub use s3::*; diff --git a/crates/fuel-streams-core/src/nats/error.rs b/crates/fuel-streams-storage/src/nats/error.rs similarity index 100% rename from crates/fuel-streams-core/src/nats/error.rs rename to crates/fuel-streams-storage/src/nats/error.rs diff --git a/crates/fuel-streams-core/src/nats/mod.rs b/crates/fuel-streams-storage/src/nats/mod.rs similarity index 96% rename from crates/fuel-streams-core/src/nats/mod.rs rename to crates/fuel-streams-storage/src/nats/mod.rs index c15e193b..63359fd2 100644 --- a/crates/fuel-streams-core/src/nats/mod.rs +++ b/crates/fuel-streams-storage/src/nats/mod.rs @@ -13,3 +13,4 @@ pub use error::*; pub use nats_client::*; pub use nats_client_opts::*; pub use nats_namespace::*; +pub use types::*; diff --git a/crates/fuel-streams-core/src/nats/nats_client.rs b/crates/fuel-streams-storage/src/nats/nats_client.rs similarity index 85% rename from crates/fuel-streams-core/src/nats/nats_client.rs rename to crates/fuel-streams-storage/src/nats/nats_client.rs index 3daae364..3c09323b 100644 --- a/crates/fuel-streams-core/src/nats/nats_client.rs +++ b/crates/fuel-streams-storage/src/nats/nats_client.rs @@ -14,10 +14,11 @@ use super::{types::*, NatsClientOpts, NatsError, NatsNamespace}; /// Creating a new `NatsClient`: /// /// ```no_run -/// use fuel_streams_core::prelude::*; +/// use fuel_streams_storage::nats::*; +/// use fuel_networks::FuelNetwork; /// -/// async fn example() -> BoxedResult<()> { -/// let opts = NatsClientOpts::new(Some(FuelNetwork::Local)); +/// async fn example() -> Result<(), Box> { +/// let opts = NatsClientOpts::new(FuelNetwork::Local); /// let client = NatsClient::connect(&opts).await?; /// Ok(()) /// } @@ -26,11 +27,12 @@ use super::{types::*, NatsClientOpts, NatsError, NatsNamespace}; /// Creating a key-value store: /// /// ```no_run -/// use fuel_streams_core::prelude::*; +/// use fuel_streams_storage::nats::*; +/// use fuel_networks::FuelNetwork; /// use async_nats::jetstream::kv; /// -/// async fn example() -> BoxedResult<()> { -/// let opts = NatsClientOpts::new(Some(FuelNetwork::Local)); +/// async fn example() -> Result<(), Box> { +/// let opts = NatsClientOpts::new(FuelNetwork::Local); /// let client = NatsClient::connect(&opts).await?; /// let kv_config = kv::Config { /// bucket: "my-bucket".into(), @@ -55,7 +57,7 @@ pub struct NatsClient { impl NatsClient { pub async fn connect(opts: &NatsClientOpts) -> Result { - let url = opts.get_url(); + let url = &opts.get_url(); let namespace = opts.namespace.clone(); let nats_client = opts.connect_opts().connect(url).await.map_err(|e| { diff --git a/crates/fuel-streams-core/src/nats/nats_client_opts.rs b/crates/fuel-streams-storage/src/nats/nats_client_opts.rs similarity index 56% rename from crates/fuel-streams-core/src/nats/nats_client_opts.rs rename to crates/fuel-streams-storage/src/nats/nats_client_opts.rs index 139ae32f..1c8cee04 100644 --- a/crates/fuel-streams-core/src/nats/nats_client_opts.rs +++ b/crates/fuel-streams-storage/src/nats/nats_client_opts.rs @@ -1,38 +1,10 @@ use std::time::Duration; use async_nats::ConnectOptions; +use fuel_networks::{FuelNetwork, FuelNetworkUserRole}; use super::NatsNamespace; -#[derive(Debug, Clone, Default)] -pub enum NatsUserRole { - Admin, - #[default] - Default, -} - -#[derive(Debug, Copy, Clone, Default, clap::ValueEnum)] -pub enum FuelNetwork { - Local, - #[default] - Testnet, - Mainnet, -} - -impl FuelNetwork { - pub fn to_url(&self) -> String { - match self { - FuelNetwork::Local => "nats://localhost:4222".to_string(), - FuelNetwork::Testnet => { - "nats://stream-testnet.fuel.network:4222".to_string() - } - FuelNetwork::Mainnet => { - "nats://stream.fuel.network:4222".to_string() - } - } - } -} - /// Represents options for configuring a NATS client. /// /// # Examples @@ -40,34 +12,36 @@ impl FuelNetwork { /// Creating a new `NatsClientOpts` instance: /// /// ``` -/// use fuel_streams_core::nats::{NatsClientOpts, FuelNetwork}; +/// use fuel_streams_storage::nats::NatsClientOpts; +/// use fuel_networks::FuelNetwork; /// -/// let opts = NatsClientOpts::new(Some(FuelNetwork::Local)); +/// let opts = NatsClientOpts::new(FuelNetwork::Local); /// ``` /// /// Creating a public `NatsClientOpts`: /// /// ``` -/// use fuel_streams_core::nats::{NatsClientOpts, FuelNetwork}; +/// use fuel_streams_storage::nats::NatsClientOpts; +/// use fuel_networks::FuelNetwork; /// -/// let opts = NatsClientOpts::default_opts(Some(FuelNetwork::Local)); +/// let opts = NatsClientOpts::new(FuelNetwork::Local); /// ``` /// /// Modifying `NatsClientOpts`: /// /// ``` -/// use fuel_streams_core::nats::{NatsClientOpts, NatsUserRole, FuelNetwork}; +/// use fuel_streams_storage::nats::NatsClientOpts; +/// use fuel_networks::{FuelNetwork, FuelNetworkUserRole}; /// -/// let opts = NatsClientOpts::new(Some(FuelNetwork::Local)) -/// .with_role(NatsUserRole::Admin) +/// let opts = NatsClientOpts::new(FuelNetwork::Local) +/// .with_role(FuelNetworkUserRole::Admin) /// .with_timeout(10); /// ``` #[derive(Debug, Clone)] pub struct NatsClientOpts { - /// The URL of the NATS server to connect to. - url: String, + pub network: FuelNetwork, /// The role of the user connecting to the NATS server (Admin or Public). - pub(crate) role: NatsUserRole, + pub(crate) role: FuelNetworkUserRole, /// The namespace used as a prefix for NATS streams, consumers, and subject names. pub(crate) namespace: NatsNamespace, /// The timeout in seconds for NATS operations. @@ -75,42 +49,32 @@ pub struct NatsClientOpts { } impl NatsClientOpts { - pub fn new(network: Option) -> Self { + pub fn new(network: FuelNetwork) -> Self { Self { - url: network.unwrap_or_default().to_url(), - role: NatsUserRole::default(), + network, + role: FuelNetworkUserRole::default(), namespace: NatsNamespace::default(), timeout_secs: 5, } } - pub fn default_opts(network: Option) -> Self { - Self::new(network).with_role(NatsUserRole::Default) + pub fn admin_opts() -> Self { + Self::new(FuelNetwork::load_from_env()) + .with_role(FuelNetworkUserRole::Admin) } - #[cfg(any(test, feature = "test-helpers"))] - pub fn admin_opts(network: Option) -> Self { - Self::new(network).with_role(NatsUserRole::Admin) - } - - pub fn with_role(self, role: NatsUserRole) -> Self { + pub fn with_role(self, role: FuelNetworkUserRole) -> Self { Self { role, ..self } } - pub fn get_url(&self) -> &str { - &self.url - } - pub fn with_fuel_network(self, network: FuelNetwork) -> Self { - Self { - url: network.to_url(), - ..self + pub fn get_url(&self) -> String { + match self.role { + FuelNetworkUserRole::Admin => dotenvy::var("NATS_URL") + .expect("NATS_URL must be set for admin role"), + FuelNetworkUserRole::Default => self.network.to_nats_url(), } } - pub fn with_custom_url(self, url: String) -> Self { - Self { url, ..self } - } - #[cfg(any(test, feature = "test-helpers"))] pub fn with_rdn_namespace(self) -> Self { let namespace = format!(r"namespace-{}", Self::random_int()); @@ -132,14 +96,14 @@ impl NatsClientOpts { pub(super) fn connect_opts(&self) -> ConnectOptions { let (user, pass) = match self.role { - NatsUserRole::Admin => ( + FuelNetworkUserRole::Admin => ( Some("admin".to_string()), Some( dotenvy::var("NATS_ADMIN_PASS") .expect("`NATS_ADMIN_PASS` env must be set"), ), ), - NatsUserRole::Default => { + FuelNetworkUserRole::Default => { (Some("default_user".to_string()), Some("".to_string())) } }; diff --git a/crates/fuel-streams-core/src/nats/nats_namespace.rs b/crates/fuel-streams-storage/src/nats/nats_namespace.rs similarity index 92% rename from crates/fuel-streams-core/src/nats/nats_namespace.rs rename to crates/fuel-streams-storage/src/nats/nats_namespace.rs index 8aa7da89..c89f5fd2 100644 --- a/crates/fuel-streams-core/src/nats/nats_namespace.rs +++ b/crates/fuel-streams-storage/src/nats/nats_namespace.rs @@ -7,7 +7,7 @@ static DEFAULT_NAMESPACE: &str = "fuel"; /// # Examples /// /// ``` -/// use fuel_streams_core::nats::NatsNamespace; +/// use fuel_streams_storage::nats::NatsNamespace; /// /// let default_namespace = NatsNamespace::default(); /// assert_eq!(default_namespace.to_string(), "fuel"); @@ -44,7 +44,7 @@ impl NatsNamespace { /// # Examples /// /// ``` - /// use fuel_streams_core::nats::NatsNamespace; + /// use fuel_streams_storage::nats::NatsNamespace; /// /// let namespace = NatsNamespace::default(); /// assert_eq!(namespace.subject_name("test"), "fuel.test"); @@ -61,7 +61,7 @@ impl NatsNamespace { /// # Examples /// /// ``` - /// use fuel_streams_core::nats::NatsNamespace; + /// use fuel_streams_storage::nats::NatsNamespace; /// /// let namespace = NatsNamespace::default(); /// assert_eq!(namespace.stream_name("test"), "fuel_test"); diff --git a/crates/fuel-streams-core/src/nats/types.rs b/crates/fuel-streams-storage/src/nats/types.rs similarity index 100% rename from crates/fuel-streams-core/src/nats/types.rs rename to crates/fuel-streams-storage/src/nats/types.rs diff --git a/crates/fuel-streams-storage/src/s3/mod.rs b/crates/fuel-streams-storage/src/s3/mod.rs new file mode 100644 index 00000000..ff459c80 --- /dev/null +++ b/crates/fuel-streams-storage/src/s3/mod.rs @@ -0,0 +1,5 @@ +mod s3_client; +mod s3_client_opts; + +pub use s3_client::*; +pub use s3_client_opts::*; diff --git a/crates/fuel-streams-storage/src/s3/s3_client.rs b/crates/fuel-streams-storage/src/s3/s3_client.rs new file mode 100644 index 00000000..ea07be30 --- /dev/null +++ b/crates/fuel-streams-storage/src/s3/s3_client.rs @@ -0,0 +1,213 @@ +use aws_config::{meta::region::RegionProviderChain, Region}; +use aws_sdk_s3::{ + config::http::HttpResponse, + operation::{ + create_bucket::CreateBucketError, + delete_bucket::DeleteBucketError, + delete_object::DeleteObjectError, + get_object::GetObjectError, + put_object::PutObjectError, + }, + Client, +}; +use aws_smithy_runtime_api::client::result::SdkError; +use aws_smithy_types::byte_stream::error::Error as BytesStreamError; +use thiserror::Error; + +use super::s3_client_opts::S3ClientOpts; + +#[derive(Error, Debug)] +pub enum S3ClientError { + #[error("AWS SDK Create Error: {0}")] + CreateBucketError(#[from] SdkError), + #[error("AWS SDK Delete bucket Error: {0}")] + DeleteBucketError(#[from] SdkError), + #[error("AWS SDK Put Error: {0}")] + PutObjectError(#[from] SdkError), + #[error("AWS SDK Get Error: {0}")] + GetObjectError(#[from] SdkError), + #[error("Error aggregating bytes from S3: {0}")] + BuildObjectAfterGettingError(#[from] BytesStreamError), + #[error("AWS SDK Delete object Error: {0}")] + DeleteObjectError(#[from] SdkError), + #[error("Environment variable missing: {0}")] + MissingEnvVar(String), + #[error("Failed to stream objects because: {0}")] + StreamingError(String), + #[error("IO Error: {0}")] + IoError(#[from] std::io::Error), +} + +#[derive(Debug, Clone)] +pub struct S3Client { + client: Client, + bucket: String, +} + +impl S3Client { + pub async fn new(opts: &S3ClientOpts) -> Result { + // Load AWS configuration + let mut aws_config = aws_config::from_env(); + + if let Some(endpoint_url) = opts.endpoint_url() { + aws_config = aws_config.endpoint_url(endpoint_url); + } + + if let Some(region) = opts.region() { + let region_provider = + RegionProviderChain::first_try(Region::new(region)); + let region = region_provider.region().await.unwrap(); + + aws_config = aws_config.region(region); + } + + let s3_config = + aws_sdk_s3::config::Builder::from(&aws_config.load().await) + .force_path_style(true) + .build(); + + let client = aws_sdk_s3::Client::from_conf(s3_config); + + Ok(Self { + client, + bucket: opts.bucket(), + }) + } + + pub fn arc(self) -> std::sync::Arc { + std::sync::Arc::new(self) + } + + pub async fn put_object( + &self, + key: &str, + object: Vec, + ) -> Result<(), S3ClientError> { + self.client + .put_object() + .bucket(&self.bucket) + .key(key) + .body(object.into()) + .send() + .await?; + + Ok(()) + } + + pub async fn get_object( + &self, + key: &str, + ) -> Result, S3ClientError> { + let result = self + .client + .get_object() + .bucket(&self.bucket) + .key(key) + .send() + .await?; + + Ok(result.body.collect().await?.into_bytes().to_vec()) + } + + /// Delete a single object from S3. + pub async fn delete_object(&self, key: &str) -> Result<(), S3ClientError> { + self.client + .delete_object() + .bucket(&self.bucket) + .key(key) + .send() + .await?; + + Ok(()) + } + + #[cfg(any(test, feature = "test-helpers"))] + pub async fn create_bucket(&self) -> Result<(), S3ClientError> { + // Create bucket + self.client + .create_bucket() + .bucket(&self.bucket) + .send() + .await?; + + Ok(()) + } + + #[cfg(any(test, feature = "test-helpers"))] + pub async fn new_for_testing() -> Self { + use fuel_networks::FuelNetwork; + + dotenvy::dotenv().expect(".env file not found"); + + let s3_client = Self::new( + &S3ClientOpts::new(FuelNetwork::Local).with_random_namespace(), + ) + .await + .expect( + "S3Client creation failed. Check AWS Env vars and Localstack setup", + ); + + s3_client + .create_bucket() + .await + .expect("Failed to create bucket"); + + s3_client + } + + #[cfg(any(test, feature = "test-helpers"))] + pub async fn cleanup_after_testing(&self) { + let client = &self.client; + let bucket = &self.bucket; + + let objects = client + .list_objects_v2() + .bucket(bucket) + .send() + .await + .unwrap(); + + for object in objects.contents() { + if let Some(key) = object.key() { + client + .delete_object() + .bucket(bucket) + .key(key) + .send() + .await + .unwrap(); + } + } + + client.delete_bucket().bucket(bucket).send().await.unwrap(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_put_and_get_object() { + let s3_client = S3Client::new_for_testing().await; + + // Put object + let key = "test-key"; + let content = b"Hello, LocalStack!".to_vec(); + s3_client + .put_object(key, content.clone()) + .await + .expect("Failed to put object"); + + // Get object + let result = s3_client + .get_object(key) + .await + .expect("Failed to get object"); + + assert_eq!(result, content); + + // Cleanup + s3_client.cleanup_after_testing().await; + } +} diff --git a/crates/fuel-streams-storage/src/s3/s3_client_opts.rs b/crates/fuel-streams-storage/src/s3/s3_client_opts.rs new file mode 100644 index 00000000..c937a032 --- /dev/null +++ b/crates/fuel-streams-storage/src/s3/s3_client_opts.rs @@ -0,0 +1,71 @@ +use fuel_networks::{FuelNetwork, FuelNetworkUserRole}; + +// Introduced for consistency. +// TODO: make it more ergonomic by probably using FuelNetwork in S3Client directly +#[derive(Debug, Clone, Default)] +pub struct S3ClientOpts { + pub fuel_network: FuelNetwork, + pub role: FuelNetworkUserRole, + pub namespace: Option, +} + +impl S3ClientOpts { + pub fn new(fuel_network: FuelNetwork) -> Self { + Self { + fuel_network, + role: FuelNetworkUserRole::default(), + namespace: None, + } + } + + pub fn admin_opts() -> Self { + Self::new(FuelNetwork::load_from_env()) + .with_role(FuelNetworkUserRole::Admin) + } + + pub fn with_role(self, role: FuelNetworkUserRole) -> Self { + Self { role, ..self } + } + + pub fn endpoint_url(&self) -> Option { + match self.role { + FuelNetworkUserRole::Admin => dotenvy::var("AWS_ENDPOINT_URL").ok(), + FuelNetworkUserRole::Default => Some(self.fuel_network.to_s3_url()), + } + } + + pub fn region(&self) -> Option { + match self.role { + FuelNetworkUserRole::Admin => dotenvy::var("AWS_S3_REGION").ok(), + FuelNetworkUserRole::Default => { + Some(self.fuel_network.to_s3_region()) + } + } + } + + // TODO: Consider revamping and reusing NATs' Namespace here + #[cfg(any(test, feature = "test-helpers"))] + pub fn with_random_namespace(mut self) -> Self { + let random_namespace = { + use rand::Rng; + let random_int: u32 = rand::thread_rng().gen(); + format!("namespace-{}", random_int) + }; + self.namespace = Some(random_namespace); + self + } + + pub fn bucket(&self) -> String { + let bucket = match self.role { + FuelNetworkUserRole::Admin => dotenvy::var("AWS_S3_BUCKET_NAME") + .expect("AWS_S3_BUCKET_NAME must be set for admin role"), + FuelNetworkUserRole::Default => self.fuel_network.to_s3_bucket(), + }; + + format!( + "{}-{}", + bucket, + self.namespace.to_owned().unwrap_or_default() + ) + } +} diff --git a/crates/fuel-streams/README.md b/crates/fuel-streams/README.md index c293a78e..410e1e37 100644 --- a/crates/fuel-streams/README.md +++ b/crates/fuel-streams/README.md @@ -75,8 +75,7 @@ async fn main() -> Result<(), fuel_streams::Error> { let stream = fuel_streams::Stream::::new(&client).await; let mut subscription = stream.subscribe().await?; - while let Some(bytes) = subscription.next().await { - let block = Block::decode(bytes.unwrap()).await; + while let Some(block) = subscription.next().await { println!("Received block: {:?}", block); } @@ -107,9 +106,7 @@ async fn main() -> Result<(), fuel_streams::Error> { .subscribe_with_config(StreamConfig::default()) .await?; - while let Some(message) = subscription.next().await { - let payload = message?.payload.clone(); - let transaction = Transaction::decode(payload.into()).await; + while let Some(transaction) = subscription.next().await { println!("Received transaction: {:?}", transaction); } @@ -146,9 +143,7 @@ async fn main() -> Result<(), fuel_streams::Error> { }) .await?; - while let Some(message) = subscription.next().await { - let payload = message?.payload.clone(); - let block = Block::decode(payload.into()).await; + while let Some(block) = subscription.next().await { println!("Received block: {:?}", block); } @@ -195,9 +190,7 @@ async fn main() -> Result<(), fuel_streams::Error> { .subscribe_with_config(StreamConfig::default()) .await?; - while let Some(message) = subscription.next().await { - let payload = message?.payload.clone(); - let transaction = Transaction::decode(payload.into()).await; + while let Some(transaction) = subscription.next().await { println!("Received filtered transaction: {:?}", transaction); } diff --git a/crates/fuel-streams/src/client/client_impl.rs b/crates/fuel-streams/src/client/client_impl.rs index f3773e6c..620d1a7d 100644 --- a/crates/fuel-streams/src/client/client_impl.rs +++ b/crates/fuel-streams/src/client/client_impl.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use fuel_streams_core::prelude::*; use super::ClientError; @@ -8,7 +10,8 @@ use super::ClientError; #[derive(Debug, Clone)] pub struct Client { /// The underlying NATS client connection. - pub conn: NatsClient, + pub nats_conn: Arc, + pub s3_conn: Arc, } impl Client { @@ -33,11 +36,20 @@ impl Client { /// # } /// ``` pub async fn connect(network: FuelNetwork) -> Result { - let opts = NatsClientOpts::new(Some(network)); - let conn = NatsClient::connect(&opts) + let nats_opts = NatsClientOpts::new(network); + let nats_client = NatsClient::connect(&nats_opts) + .await + .map_err(ClientError::NatsConnectionFailed)?; + + let s3_client_opts = S3ClientOpts::new(network); + let s3_client = S3Client::new(&s3_client_opts) .await - .map_err(ClientError::ConnectionFailed)?; - Ok(Self { conn }) + .map_err(ClientError::S3ConnectionFailed)?; + + Ok(Self { + nats_conn: Arc::new(nats_client), + s3_conn: Arc::new(s3_client), + }) } /// Connects to a NATS server using the provided options. @@ -55,19 +67,29 @@ impl Client { /// ```no_run /// use fuel_streams::client::{Client, FuelNetwork}; /// use fuel_streams_core::nats::NatsClientOpts; + /// use fuel_streams_core::s3::S3ClientOpts; /// /// # async fn example() -> Result<(), fuel_streams::Error> { - /// let opts = NatsClientOpts::new(Some(FuelNetwork::Local)); - /// let client = Client::with_opts(&opts).await?; + /// let nats_opts = NatsClientOpts::new(FuelNetwork::Local); + /// let s3_opts = S3ClientOpts::new(FuelNetwork::Local); + /// + /// let client = Client::with_opts(&nats_opts, &s3_opts).await?; /// # Ok(()) /// # } /// ``` pub async fn with_opts( - opts: &NatsClientOpts, + nats_opts: &NatsClientOpts, + s3_opts: &S3ClientOpts, ) -> Result { - let conn = NatsClient::connect(opts) + let nats_client = NatsClient::connect(nats_opts) + .await + .map_err(ClientError::NatsConnectionFailed)?; + let s3_client = S3Client::new(s3_opts) .await - .map_err(ClientError::ConnectionFailed)?; - Ok(Self { conn }) + .map_err(ClientError::S3ConnectionFailed)?; + Ok(Self { + nats_conn: Arc::new(nats_client), + s3_conn: Arc::new(s3_client), + }) } } diff --git a/crates/fuel-streams/src/client/error.rs b/crates/fuel-streams/src/client/error.rs index fa89767c..216f855e 100644 --- a/crates/fuel-streams/src/client/error.rs +++ b/crates/fuel-streams/src/client/error.rs @@ -1,9 +1,11 @@ use displaydoc::Display as DisplayDoc; -use fuel_streams_core::nats::NatsError; +use fuel_streams_core::{nats::NatsError, s3::S3ClientError}; use thiserror::Error; #[derive(Debug, Error, DisplayDoc)] pub enum ClientError { /// Failed to establish connection with the NATS server: {0} - ConnectionFailed(#[from] NatsError), + NatsConnectionFailed(#[from] NatsError), + /// Failed to establish connection with S3: {0} + S3ConnectionFailed(#[from] S3ClientError), } diff --git a/crates/fuel-streams/src/client/types.rs b/crates/fuel-streams/src/client/types.rs index d3e2632e..f2e5c976 100644 --- a/crates/fuel-streams/src/client/types.rs +++ b/crates/fuel-streams/src/client/types.rs @@ -1,4 +1,4 @@ -pub use fuel_streams_core::nats::FuelNetwork; +pub use fuel_streams_core::prelude::FuelNetwork; #[derive(Debug, Clone, Eq, PartialEq, Default)] pub enum ClientStatus { diff --git a/crates/fuel-streams/src/lib.rs b/crates/fuel-streams/src/lib.rs index c86c099f..9bd61ae9 100644 --- a/crates/fuel-streams/src/lib.rs +++ b/crates/fuel-streams/src/lib.rs @@ -13,7 +13,8 @@ pub mod subjects { pub mod types { pub use fuel_streams_core::{ - nats::{types::*, FuelNetwork, NatsClientOpts}, + nats::{types::*, NatsClientOpts}, + prelude::FuelNetwork, types::*, }; diff --git a/crates/fuel-streams/src/stream/mod.rs b/crates/fuel-streams/src/stream/mod.rs index 511a2b19..755cda45 100644 --- a/crates/fuel-streams/src/stream/mod.rs +++ b/crates/fuel-streams/src/stream/mod.rs @@ -6,6 +6,6 @@ pub use fuel_streams_core::stream::{ StreamData, StreamEncoder, Streamable, - SubscribeConsumerConfig, + SubscriptionConfig, }; pub use stream_impl::*; diff --git a/crates/fuel-streams/src/stream/stream_impl.rs b/crates/fuel-streams/src/stream/stream_impl.rs index c10d78ab..e899b68e 100644 --- a/crates/fuel-streams/src/stream/stream_impl.rs +++ b/crates/fuel-streams/src/stream/stream_impl.rs @@ -1,8 +1,10 @@ +use std::pin::Pin; + use fuel_streams_core::{ prelude::{IntoSubject, SubjectBuildable}, - types::{DeliverPolicy, PullConsumerStream}, + types::DeliverPolicy, Streamable, - SubscribeConsumerConfig, + SubscriptionConfig, }; use crate::{client::Client, stream::StreamError}; @@ -79,8 +81,11 @@ impl Stream { /// # } /// ``` pub async fn new(client: &Client) -> Self { - let stream = - fuel_streams_core::Stream::::get_or_init(&client.conn).await; + let stream = fuel_streams_core::Stream::::get_or_init( + &client.nats_conn, + &client.s3_conn, + ) + .await; Self { stream, filter_subjects: Vec::new(), @@ -121,7 +126,7 @@ impl Stream { self } - /// Subscribes to the stream. + /// Subscribes to the stream item. /// /// # Returns /// @@ -143,19 +148,57 @@ impl Stream { /// # Ok(()) /// # } /// ``` - pub async fn subscribe( - &self, - ) -> Result>>, StreamError> { + pub async fn subscribe<'a>( + &'a self, + ) -> Result + Send + 'a>>, StreamError> + { // TODO: Why implicitly select a stream for the user? // TODO: Should this be a combination of streams self.stream // TODO: Improve DX by ensuring the stream returns the streamable entity directly - .subscribe(S::WILDCARD_LIST[0]) + .subscribe(None) .await .map_err(|source| StreamError::Subscribe { source }) } - /// Subscribes to the stream with custom configuration options. + /// Subscribes to the stream bytes. + /// + /// # Returns + /// + /// Returns a `Result` containing a `futures::Stream` of byte vectors on success, + /// or a `StreamError` on failure. + /// + /// # Examples + /// + /// ```no_run + /// use fuel_streams::types::FuelNetwork; + /// use fuel_streams::client::Client; + /// use fuel_streams::stream::Stream; + /// use fuel_streams::blocks::Block; + /// + /// # async fn example() -> Result<(), fuel_streams::Error> { + /// # let client = Client::connect(FuelNetwork::Local).await?; + /// # let stream = Stream::::new(&client).await; + /// let subscription = stream.subscribe().await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn subscribe_raw<'a>( + &'a self, + ) -> Result< + Pin> + Send + 'a>>, + StreamError, + > { + // TODO: Why implicitly select a stream for the user? + // TODO: Should this be a combination of streams + self.stream + // TODO: Improve DX by ensuring the stream returns the streamable entity directly + .subscribe_raw(None) + .await + .map_err(|source| StreamError::Subscribe { source }) + } + + /// Subscribes to the stream item with custom configuration options. /// /// # Parameters /// @@ -185,15 +228,64 @@ impl Stream { /// # Ok(()) /// # } /// ``` - pub async fn subscribe_with_config( - &self, + pub async fn subscribe_with_config<'a>( + &'a self, opts: StreamConfig, - ) -> Result { + ) -> Result + Send + 'a>>, StreamError> + { self.stream - .subscribe_consumer(SubscribeConsumerConfig { + // TODO: Improve DX by ensuring the stream returns the streamable entity directly + .subscribe(Some(SubscriptionConfig { + deliver_policy: opts.deliver_policy, + filter_subjects: self.filter_subjects.to_owned(), + })) + .await + .map_err(|source| StreamError::SubscribeWithOpts { source }) + } + + /// Subscribes to the stream bytes with custom configuration options. + /// + /// # Parameters + /// + /// * `opts`: A `StreamConfig` instance containing custom configuration options. + /// + /// # Returns + /// + /// Returns a `Result` containing a `PullConsumerStream` on success, + /// or a `StreamError` on failure. + /// + /// # Examples + /// + /// ```no_run + /// use fuel_streams::types::FuelNetwork; + /// use fuel_streams::client::Client; + /// use fuel_streams::stream::{Stream, StreamConfig}; + /// use fuel_streams::blocks::Block; + /// use fuel_streams::types::DeliverPolicy; + /// + /// # async fn example() -> Result<(), fuel_streams::Error> { + /// # let client = Client::connect(FuelNetwork::Local).await?; + /// # let stream = Stream::::new(&client).await; + /// let config = StreamConfig { + /// deliver_policy: DeliverPolicy::All, + /// }; + /// let subscription = stream.subscribe_with_config(config).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn subscribe_raw_with_config<'a>( + &'a self, + opts: StreamConfig, + ) -> Result< + Pin> + Send + 'a>>, + StreamError, + > { + self.stream + // TODO: Improve DX by ensuring the stream returns the streamable entity directly + .subscribe_raw(Some(SubscriptionConfig { deliver_policy: opts.deliver_policy, filter_subjects: self.filter_subjects.to_owned(), - }) + })) .await .map_err(|source| StreamError::SubscribeWithOpts { source }) } diff --git a/examples/blocks.rs b/examples/blocks.rs index e92a72e7..48e882e4 100644 --- a/examples/blocks.rs +++ b/examples/blocks.rs @@ -31,14 +31,13 @@ async fn main() -> anyhow::Result<()> { }; // Subscribe to the block stream with the specified configuration - let mut sub = stream.subscribe_with_config(config).await?; + let mut sub = stream.subscribe_raw_with_config(config).await?; println!("Listening for blocks..."); // Process incoming blocks while let Some(bytes) = sub.next().await { - let message = bytes?; - let decoded_msg = Block::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Block::decode_raw(bytes).unwrap(); let tx_subject = decoded_msg.subject; let tx_published_at = decoded_msg.timestamp; diff --git a/examples/inputs.rs b/examples/inputs.rs index 10fcbac9..da448d9f 100644 --- a/examples/inputs.rs +++ b/examples/inputs.rs @@ -31,14 +31,13 @@ async fn main() -> anyhow::Result<()> { }; // Subscribe to the input stream with the specified configuration - let mut sub = stream.subscribe_with_config(config).await?; + let mut sub = stream.subscribe_raw_with_config(config).await?; println!("Listening for inputs..."); // Process incoming inputs while let Some(bytes) = sub.next().await { - let message = bytes?; - let decoded_msg = Input::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Input::decode_raw(bytes).unwrap(); let tx_subject = decoded_msg.subject; let tx_published_at = decoded_msg.timestamp; diff --git a/examples/logs.rs b/examples/logs.rs index 5fc53cdd..f9e1100b 100644 --- a/examples/logs.rs +++ b/examples/logs.rs @@ -31,14 +31,13 @@ async fn main() -> anyhow::Result<()> { }; // Subscribe to the log stream with the specified configuration - let mut sub = stream.subscribe_with_config(config).await?; + let mut sub = stream.subscribe_raw_with_config(config).await?; println!("Listening for logs..."); // Process incoming logs while let Some(bytes) = sub.next().await { - let message = bytes?; - let decoded_msg = Log::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Log::decode_raw(bytes).unwrap(); let log_subject = decoded_msg.subject; let log_published_at = decoded_msg.timestamp; diff --git a/examples/multiple-streams.rs b/examples/multiple-streams.rs index 0511434b..1bb1d4ca 100644 --- a/examples/multiple-streams.rs +++ b/examples/multiple-streams.rs @@ -123,11 +123,13 @@ async fn stream_blocks( let mut block_stream = fuel_streams::Stream::::new(client).await; let mut sub = match filter { - Some(filter) => block_stream.with_filter(filter).subscribe().await?, - None => block_stream.subscribe().await?, + Some(filter) => { + block_stream.with_filter(filter).subscribe_raw().await? + } + None => block_stream.subscribe_raw().await?, }; while let Some(bytes) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes.unwrap()).await; + let decoded_msg = Block::decode_raw(bytes).unwrap(); let block_height = decoded_msg.payload.height; let block_subject = decoded_msg.subject; let block_published_at = decoded_msg.timestamp; @@ -155,16 +157,14 @@ async fn stream_transactions( Some(filter) => { txs_stream .with_filter(filter) - .subscribe_with_config(config) + .subscribe_raw_with_config(config) .await? } - None => txs_stream.subscribe_with_config(config).await?, + None => txs_stream.subscribe_raw_with_config(config).await?, }; while let Some(bytes) = sub.next().await { - let message = bytes?; - let decoded_msg = - Transaction::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Transaction::decode_raw(bytes).unwrap(); let tx = decoded_msg.payload; let tx_subject = decoded_msg.subject; let tx_published_at = decoded_msg.timestamp; @@ -201,10 +201,10 @@ async fn stream_transactions_by_contract( .with_id_value(Some((*contract_id).into())); // Filtered stream - let mut sub = txs_stream.with_filter(filter).subscribe().await?; + let mut sub = txs_stream.with_filter(filter).subscribe_raw().await?; while let Some(bytes) = sub.next().await { - let decoded_msg = Transaction::decode_raw(bytes.unwrap()).await; + let decoded_msg = Transaction::decode_raw(bytes).unwrap(); let tx = decoded_msg.payload; let tx_subject = decoded_msg.subject; let tx_published_at = decoded_msg.timestamp; @@ -274,10 +274,10 @@ async fn stream_contract( ReceiptsMintSubject::new().with_contract_id(Some(contract_id.into())), ); - let mut sub = receipt_stream.subscribe().await?; + let mut sub = receipt_stream.subscribe_raw().await?; while let Some(bytes) = sub.next().await { - let decoded_msg = Receipt::decode_raw(bytes.unwrap().to_vec()).await; + let decoded_msg = Receipt::decode_raw(bytes).unwrap(); let receipt = decoded_msg.payload; // Check if the receipt has a contract_id and if it matches our target @@ -321,10 +321,10 @@ async fn stream_inputs_by_contract( .with_id_value(Some((*contract_id).into())), ); - let mut sub = inputs_stream.subscribe().await?; + let mut sub = inputs_stream.subscribe_raw().await?; while let Some(bytes) = sub.next().await { - let decoded_msg = Input::decode_raw(bytes.unwrap().to_vec()).await; + let decoded_msg = Input::decode_raw(bytes).unwrap(); let input = decoded_msg.payload; let input_subject = decoded_msg.subject; let input_published_at = decoded_msg.timestamp; @@ -362,10 +362,10 @@ async fn stream_receipts_by_contract( .with_id_value(Some((*contract_id).into())), ); - let mut sub = receipt_stream.subscribe().await?; + let mut sub = receipt_stream.subscribe_raw().await?; while let Some(bytes) = sub.next().await { - let decoded_msg = Receipt::decode_raw(bytes.unwrap().to_vec()).await; + let decoded_msg = Receipt::decode_raw(bytes).unwrap(); let receipt = decoded_msg.payload; let receipt_subject = decoded_msg.subject; let receipt_published_at = decoded_msg.timestamp; diff --git a/examples/outputs.rs b/examples/outputs.rs index 83f2be46..f9836030 100644 --- a/examples/outputs.rs +++ b/examples/outputs.rs @@ -31,14 +31,13 @@ async fn main() -> anyhow::Result<()> { }; // Subscribe to the output stream with the specified configuration - let mut sub = stream.subscribe_with_config(config).await?; + let mut sub = stream.subscribe_raw_with_config(config).await?; println!("Listening for outputs..."); // Process incoming outputs while let Some(bytes) = sub.next().await { - let message = bytes?; - let decoded_msg = Output::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Output::decode_raw(bytes).unwrap(); let tx_subject = decoded_msg.subject; let tx_published_at = decoded_msg.timestamp; diff --git a/examples/receipts.rs b/examples/receipts.rs index 8160e436..09430779 100644 --- a/examples/receipts.rs +++ b/examples/receipts.rs @@ -97,14 +97,13 @@ async fn main() -> Result<()> { }; // Subscribe to the receipt stream - let mut sub = receipt_stream.subscribe_with_config(config).await?; + let mut sub = receipt_stream.subscribe_raw_with_config(config).await?; println!("Listening for receipts..."); // Process incoming receipts while let Some(bytes) = sub.next().await { - let message = bytes.unwrap(); - let decoded_msg = Receipt::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Receipt::decode_raw(bytes).unwrap(); let receipt = decoded_msg.payload; let receipt_subject = decoded_msg.subject; let receipt_published_at = decoded_msg.timestamp; diff --git a/examples/transactions.rs b/examples/transactions.rs index 5294ec44..44025b13 100644 --- a/examples/transactions.rs +++ b/examples/transactions.rs @@ -31,15 +31,13 @@ async fn main() -> anyhow::Result<()> { }; // Subscribe to the transaction stream with the specified configuration - let mut sub = stream.subscribe_with_config(config).await?; + let mut sub = stream.subscribe_raw_with_config(config).await?; println!("Listening for transactions..."); // Process incoming transactions while let Some(bytes) = sub.next().await { - let message = bytes?; - let decoded_msg = - Transaction::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Transaction::decode_raw(bytes).unwrap(); let tx = decoded_msg.payload; let tx_subject = decoded_msg.subject; let tx_published_at = decoded_msg.timestamp; diff --git a/examples/utxos.rs b/examples/utxos.rs index fabcc31f..ea46b64e 100644 --- a/examples/utxos.rs +++ b/examples/utxos.rs @@ -31,14 +31,13 @@ async fn main() -> anyhow::Result<()> { }; // Subscribe to the UTXO stream with the specified configuration - let mut sub = stream.subscribe_with_config(config).await?; + let mut sub = stream.subscribe_raw_with_config(config).await?; println!("Listening for UTXOs..."); // Process incoming UTXOs while let Some(bytes) = sub.next().await { - let message = bytes?; - let decoded_msg = Utxo::decode_raw(message.payload.to_vec()).await; + let decoded_msg = Utxo::decode_raw(bytes).unwrap(); let utxo_subject = decoded_msg.subject; let utxo_published_at = decoded_msg.timestamp; diff --git a/scripts/run_publisher.sh b/scripts/run_publisher.sh index 7d29d677..b566ea34 100755 --- a/scripts/run_publisher.sh +++ b/scripts/run_publisher.sh @@ -9,7 +9,7 @@ set -e usage() { echo "Usage: $0 [options]" echo "Options:" - echo " --network : Specify the network (mainnet|testnet)" + echo " --network : Specify the network (mainnet|testnet|local)" echo " Default: testnet" echo " --mode : Specify the run mode (dev|profiling)" echo " Default: profiling" diff --git a/tests/src/lib.rs b/tests/src/lib.rs index ac6267df..bbce5580 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -1,5 +1,6 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; +use fuel_streams::client::Client; use fuel_streams_core::{ nats::NatsClient, prelude::*, @@ -20,9 +21,13 @@ pub struct Streams { } impl Streams { - pub async fn new(client: &NatsClient) -> Self { - let blocks = Stream::::get_or_init(client).await; - let transactions = Stream::::get_or_init(client).await; + pub async fn new( + nats_client: &NatsClient, + s3_client: &Arc, + ) -> Self { + let blocks = Stream::::get_or_init(nats_client, s3_client).await; + let transactions = + Stream::::get_or_init(nats_client, s3_client).await; Self { transactions, blocks, @@ -30,12 +35,21 @@ impl Streams { } } -pub async fn server_setup() -> BoxedResult<(NatsClient, Streams)> { - let opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) - .with_rdn_namespace(); - let client = NatsClient::connect(&opts).await?; - let streams = Streams::new(&client).await; - Ok((client, streams)) +pub async fn server_setup() -> BoxedResult<(NatsClient, Streams, Client)> { + let nats_client_opts = NatsClientOpts::admin_opts().with_rdn_namespace(); + let nats_client = NatsClient::connect(&nats_client_opts).await?; + + let s3_client_opts = S3ClientOpts::admin_opts().with_random_namespace(); + let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); + s3_client.create_bucket().await?; + + let streams = Streams::new(&nats_client, &s3_client).await; + + let client = Client::with_opts(&nats_client_opts, &s3_client_opts) + .await + .unwrap(); + + Ok((nats_client, streams, client)) } pub fn publish_items( @@ -49,8 +63,10 @@ pub fn publish_items( for item in items { tokio::time::sleep(Duration::from_millis(50)).await; let payload = item.1.clone(); - let subject = item.0; - stream.publish(&subject, &payload).await.unwrap(); + let subject = Arc::new(item.0); + let packet = payload.to_packet(subject); + + stream.publish(&packet).await.unwrap(); } } }) diff --git a/tests/src/main.rs b/tests/src/main.rs index 159bb8af..e48fa797 100644 --- a/tests/src/main.rs +++ b/tests/src/main.rs @@ -75,13 +75,16 @@ async fn main() -> BoxedResult<()> { .expect("Failed to change directory to workspace root"); // ensure nats is connected and running - let client_opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) + let client_opts = NatsClientOpts::admin_opts() .with_rdn_namespace() .with_timeout(1); - let is_connected = Client::with_opts(&client_opts) + + let s3_client_opts = S3ClientOpts::admin_opts(); + + let is_connected = Client::with_opts(&client_opts, &s3_client_opts) .await .ok() - .map(|c| c.conn.is_connected()) + .map(|c| c.nats_conn.is_connected()) .unwrap_or_default(); if !is_connected { println!("Starting nats ..."); @@ -89,10 +92,9 @@ async fn main() -> BoxedResult<()> { } // create a subscription - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let stream = fuel_streams::Stream::::new(&client).await; - let mut sub = stream.subscribe().await.unwrap().enumerate(); + let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); // publish all items in a separate thread let (items, publish_join_handle) = @@ -108,26 +110,24 @@ async fn main() -> BoxedResult<()> { loop { tokio::select! { - bytes = sub.next() => { - let (index, bytes) = bytes.unzip(); - if let Some(bytes) = bytes.flatten() { + Some((index, bytes)) = sub.next() => { println!("Valid subscription"); - let decoded_msg = Block::decode_raw(bytes).await; - let (subject, block) = items[index.unwrap()].to_owned(); + let decoded_msg = Block::decode_raw(bytes).unwrap(); + let (subject, block) = items[index].to_owned(); let height = decoded_msg.payload.height; assert_eq!(decoded_msg.subject, subject.parse()); assert_eq!(decoded_msg.payload, block); - assert_eq!(height, index.unwrap() as u32); - if index.unwrap() == 9 { + assert_eq!(height, index as u32); + if index == 9 { break; } - } } _ = action_interval.tick() => { - let client_opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) + let client_opts = NatsClientOpts::admin_opts() .with_rdn_namespace() .with_timeout(1); - let is_nats_connected = Client::with_opts(&client_opts).await.ok().map(|c| c.conn.is_connected()).unwrap_or_default(); + + let is_nats_connected = Client::with_opts(&client_opts, &s3_client_opts).await.ok().map(|c| c.nats_conn.is_connected()).unwrap_or_default(); if is_nats_connected { stop_nats(&makefile_path); } else { diff --git a/tests/tests/client.rs b/tests/tests/client.rs index 06831bbd..a0178a7c 100644 --- a/tests/tests/client.rs +++ b/tests/tests/client.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, time::Duration}; +use std::{collections::HashSet, sync::Arc, time::Duration}; use fuel_streams::prelude::*; use fuel_streams_core::prelude::{types, *}; @@ -22,7 +22,7 @@ fn gen_random_string(size: usize) -> String { #[tokio::test] async fn conn_streams_has_required_streams() -> BoxedResult<()> { - let (client, streams) = server_setup().await.unwrap(); + let (client, streams, _) = server_setup().await.unwrap(); let mut context_streams = client.jetstream.stream_names(); let mut names = HashSet::new(); @@ -41,23 +41,27 @@ async fn conn_streams_has_required_streams() -> BoxedResult<()> { #[tokio::test] async fn fuel_streams_client_connection() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)); - let client = NatsClient::connect(&opts).await?; + let nats_opts = NatsClientOpts::admin_opts(); + let client = NatsClient::connect(&nats_opts).await?; assert!(client.is_connected()); - let client = Client::with_opts(&opts).await?; - assert!(client.conn.is_connected()); + let s3_opts = Arc::new(S3ClientOpts::admin_opts()); + let client = Client::with_opts(&nats_opts, &s3_opts).await?; + assert!(client.nats_conn.is_connected()); Ok(()) } #[tokio::test] async fn multiple_client_connections() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)); + let nats_opts = NatsClientOpts::admin_opts(); + let s3_opts = Arc::new(S3ClientOpts::admin_opts()); let tasks: Vec<_> = (0..100) .map(|_| { - let opts = opts.clone(); + let nats_opts = nats_opts.clone(); + let s3_opts = s3_opts.clone(); async move { - let client = Client::with_opts(&opts).await.unwrap(); - assert!(client.conn.is_connected()); + let client = + Client::with_opts(&nats_opts, &s3_opts).await.unwrap(); + assert!(client.nats_conn.is_connected()); Ok::<(), NatsError>(()) } }) @@ -69,7 +73,7 @@ async fn multiple_client_connections() -> BoxedResult<()> { #[tokio::test] async fn public_user_cannot_create_streams() -> BoxedResult<()> { - let opts = NatsClientOpts::default_opts(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::new(FuelNetwork::Local) .with_rdn_namespace() .with_timeout(1); let client = NatsClient::connect(&opts).await?; @@ -91,7 +95,7 @@ async fn public_user_cannot_create_streams() -> BoxedResult<()> { #[tokio::test] async fn public_user_cannot_create_stores() -> BoxedResult<()> { - let opts = NatsClientOpts::default_opts(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::new(FuelNetwork::Local) .with_rdn_namespace() .with_timeout(1); @@ -112,7 +116,7 @@ async fn public_user_cannot_create_stores() -> BoxedResult<()> { #[tokio::test] async fn public_user_cannot_delete_stores() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::admin_opts() .with_rdn_namespace() .with_timeout(1); @@ -127,7 +131,7 @@ async fn public_user_cannot_delete_stores() -> BoxedResult<()> { }) .await?; - let opts = NatsClientOpts::default_opts(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::new(FuelNetwork::Local) .with_rdn_namespace() .with_timeout(1); let client = NatsClient::connect(&opts).await?; @@ -143,7 +147,7 @@ async fn public_user_cannot_delete_stores() -> BoxedResult<()> { #[tokio::test] async fn public_user_cannot_delete_stream() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::admin_opts() .with_rdn_namespace() .with_timeout(1); let client = NatsClient::connect(&opts).await?; @@ -160,7 +164,7 @@ async fn public_user_cannot_delete_stream() -> BoxedResult<()> { }) .await?; - let public_opts = opts.clone().with_role(NatsUserRole::Default); + let public_opts = opts.clone().with_role(FuelNetworkUserRole::Default); let public_client = NatsClient::connect(&public_opts).await?; assert!( @@ -177,40 +181,44 @@ async fn public_user_cannot_delete_stream() -> BoxedResult<()> { #[tokio::test] async fn public_user_can_access_streams_after_created() { - let opts = NatsClientOpts::new(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::new(FuelNetwork::Local) .with_rdn_namespace() .with_timeout(1); - let admin_opts = opts.clone().with_role(NatsUserRole::Admin); + let admin_opts = opts.clone().with_role(FuelNetworkUserRole::Admin); assert!(NatsClient::connect(&admin_opts).await.is_ok()); - let public_opts = opts.clone().with_role(NatsUserRole::Default); + let public_opts = opts.clone().with_role(FuelNetworkUserRole::Default); assert!(NatsClient::connect(&public_opts).await.is_ok()); } #[tokio::test] async fn public_and_admin_user_can_access_streams_after_created( ) -> BoxedResult<()> { - let admin_opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)); + let admin_opts = NatsClientOpts::admin_opts(); + let s3_opts = Arc::new(S3ClientOpts::admin_opts()); let admin_tasks: Vec>> = (0..100) .map(|_| { let opts: NatsClientOpts = admin_opts.clone(); + let s3_opts = s3_opts.clone(); async move { - let client = Client::with_opts(&opts).await.unwrap(); - assert!(client.conn.is_connected()); + let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); + assert!(client.nats_conn.is_connected()); Ok::<(), NatsError>(()) } .boxed() }) .collect(); - let public_opts = NatsClientOpts::default_opts(Some(FuelNetwork::Local)); + let public_opts = NatsClientOpts::new(FuelNetwork::Local); + let s3_public_opts = Arc::new(S3ClientOpts::new(FuelNetwork::Local)); let public_tasks: Vec>> = (0..100) .map(|_| { let opts: NatsClientOpts = public_opts.clone(); + let s3_opts = s3_public_opts.clone(); async move { - let client = Client::with_opts(&opts).await.unwrap(); - assert!(client.conn.is_connected()); + let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); + assert!(client.nats_conn.is_connected()); Ok::<(), NatsError>(()) } .boxed() @@ -229,7 +237,7 @@ async fn public_and_admin_user_can_access_streams_after_created( #[tokio::test] async fn admin_user_can_delete_stream() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::admin_opts() .with_rdn_namespace() .with_timeout(1); let client = NatsClient::connect(&opts).await?; @@ -254,7 +262,7 @@ async fn admin_user_can_delete_stream() -> BoxedResult<()> { #[tokio::test] async fn admin_user_can_delete_stores() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) + let opts = NatsClientOpts::admin_opts() .with_rdn_namespace() .with_timeout(1); @@ -280,8 +288,7 @@ async fn admin_user_can_delete_stores() -> BoxedResult<()> { #[tokio::test] async fn ensure_deduplication_when_publishing() -> BoxedResult<()> { - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let stream = fuel_streams::Stream::::new(&client).await; let producer = Some(Address::zeroed()); let const_block_height = 1001; @@ -290,15 +297,14 @@ async fn ensure_deduplication_when_publishing() -> BoxedResult<()> { .unwrap() .0; - let mut sub = stream.subscribe().await.unwrap().enumerate(); + let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); let timeout_duration = Duration::from_secs(1); // ensure just one message was published 'l: loop { match timeout(timeout_duration, sub.next()).await { Ok(Some((idx, entry))) => { - assert!(entry.is_some()); - let decoded_msg = Block::decode_raw(entry.unwrap()).await; + let decoded_msg = Block::decode_raw(entry).unwrap(); let (subject, _block) = items[idx].to_owned(); let height = decoded_msg.payload.height; assert_eq!(decoded_msg.subject, subject.parse()); diff --git a/tests/tests/publisher.rs b/tests/tests/publisher.rs index 42ae9e85..52e0709c 100644 --- a/tests/tests/publisher.rs +++ b/tests/tests/publisher.rs @@ -96,7 +96,8 @@ impl FuelCoreLike for TestFuelCore { #[tokio::test(flavor = "multi_thread")] async fn doesnt_publish_any_message_when_no_block_has_been_mined() { let (blocks_broadcaster, _) = broadcast::channel::(1); - let publisher = new_publisher(blocks_broadcaster.clone()).await; + let s3_client = Arc::new(S3Client::new_for_testing().await); + let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; let shutdown_controller = start_publisher(&publisher).await; stop_publisher(shutdown_controller).await; @@ -107,7 +108,8 @@ async fn doesnt_publish_any_message_when_no_block_has_been_mined() { #[tokio::test(flavor = "multi_thread")] async fn publishes_a_block_message_when_a_single_block_has_been_mined() { let (blocks_broadcaster, _) = broadcast::channel::(1); - let publisher = new_publisher(blocks_broadcaster.clone()).await; + let s3_client = Arc::new(S3Client::new_for_testing().await); + let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; publish_block(&publisher, &blocks_broadcaster).await; @@ -117,12 +119,14 @@ async fn publishes_a_block_message_when_a_single_block_has_been_mined() { .get_last_published(BlocksSubject::WILDCARD) .await .is_ok_and(|result| result.is_some())); + s3_client.cleanup_after_testing().await; } #[tokio::test(flavor = "multi_thread")] async fn publishes_transaction_for_each_published_block() { let (blocks_broadcaster, _) = broadcast::channel::(1); - let publisher = new_publisher(blocks_broadcaster.clone()).await; + let s3_client = Arc::new(S3Client::new_for_testing().await); + let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; publish_block(&publisher, &blocks_broadcaster).await; @@ -132,6 +136,7 @@ async fn publishes_transaction_for_each_published_block() { .get_last_published(TransactionsSubject::WILDCARD) .await .is_ok_and(|result| result.is_some())); + s3_client.cleanup_after_testing().await; } #[tokio::test(flavor = "multi_thread")] @@ -227,9 +232,11 @@ async fn publishes_receipts() { .with_receipts(receipts.to_vec()) .arc(); - let publisher = Publisher::default(&nats_client().await, fuel_core) - .await - .unwrap(); + let s3_client = Arc::new(S3Client::new_for_testing().await); + let publisher = + Publisher::new_for_testing(&nats_client().await, &s3_client, fuel_core) + .await + .unwrap(); publish_block(&publisher, &blocks_broadcaster).await; @@ -244,12 +251,15 @@ async fn publishes_receipts() { while let Some(Some(receipt)) = receipts_stream.next().await { assert!(receipts.contains(&receipt)); } + + s3_client.cleanup_after_testing().await; } #[tokio::test(flavor = "multi_thread")] async fn publishes_inputs() { let (blocks_broadcaster, _) = broadcast::channel::(1); - let publisher = new_publisher(blocks_broadcaster.clone()).await; + let s3_client = Arc::new(S3Client::new_for_testing().await); + let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; publish_block(&publisher, &blocks_broadcaster).await; @@ -259,11 +269,15 @@ async fn publishes_inputs() { .get_last_published(InputsByIdSubject::WILDCARD) .await .is_ok_and(|result| result.is_some())); + s3_client.cleanup_after_testing().await; } -async fn new_publisher(broadcaster: Sender) -> Publisher { +async fn new_publisher( + broadcaster: Sender, + s3_client: &Arc, +) -> Publisher { let fuel_core = TestFuelCore::default(broadcaster).arc(); - Publisher::default(&nats_client().await, fuel_core) + Publisher::new_for_testing(&nats_client().await, s3_client, fuel_core) .await .unwrap() } @@ -325,9 +339,8 @@ fn create_test_block() -> ImporterResult { } async fn nats_client() -> NatsClient { - let nats_client_opts = NatsClientOpts::admin_opts(Some(FuelNetwork::Local)) - .with_rdn_namespace(); - NatsClient::connect(&nats_client_opts) + let opts = NatsClientOpts::admin_opts().with_rdn_namespace(); + NatsClient::connect(&opts) .await .expect("NATS connection failed") } diff --git a/tests/tests/stream.rs b/tests/tests/stream.rs index 73bdbf1e..f8f11ac5 100644 --- a/tests/tests/stream.rs +++ b/tests/tests/stream.rs @@ -6,15 +6,15 @@ use streams_tests::{publish_blocks, publish_transactions, server_setup}; #[tokio::test] async fn blocks_streams_subscribe() { - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let stream = fuel_streams::Stream::::new(&client).await; let producer = Some(Address::zeroed()); let items = publish_blocks(stream.stream(), producer, None).unwrap().0; - let mut sub = stream.subscribe().await.unwrap().enumerate(); + let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); + while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes.unwrap()).await; + let decoded_msg = Block::decode_raw(bytes).unwrap(); let (subject, block) = items[i].to_owned(); let height = decoded_msg.payload.height; @@ -29,8 +29,7 @@ async fn blocks_streams_subscribe() { #[tokio::test] async fn blocks_streams_subscribe_with_filter() { - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let mut stream = fuel_streams::Stream::::new(&client).await; let producer = Some(Address::zeroed()); @@ -45,16 +44,14 @@ async fn blocks_streams_subscribe_with_filter() { // creating subscription let mut sub = stream .with_filter(filter) - .subscribe_with_config(StreamConfig::default()) + .subscribe_raw_with_config(StreamConfig::default()) .await .unwrap() .take(10); // result should be just 1 single message with height 5 - while let Some(message) = sub.next().await { - let message = message.unwrap(); - let decoded_msg = - Block::decode_raw(message.payload.clone().into()).await; + while let Some(bytes) = sub.next().await { + let decoded_msg = Block::decode_raw(bytes).unwrap(); let height = decoded_msg.payload.height; assert_eq!(height, 5); if height == 5 { @@ -65,8 +62,7 @@ async fn blocks_streams_subscribe_with_filter() { #[tokio::test] async fn transactions_streams_subscribe() { - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let stream = fuel_streams::Stream::::new(&client).await; let mock_block = MockBlock::build(1); @@ -74,10 +70,9 @@ async fn transactions_streams_subscribe() { .unwrap() .0; - let mut sub = stream.subscribe().await.unwrap().enumerate(); + let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); while let Some((i, bytes)) = sub.next().await { - let decoded_msg = - Transaction::decode_raw(bytes.unwrap().to_vec()).await; + let decoded_msg = Transaction::decode_raw(bytes).unwrap(); let (_, transaction) = items[i].to_owned(); assert_eq!(decoded_msg.payload, transaction); @@ -89,8 +84,7 @@ async fn transactions_streams_subscribe() { #[tokio::test] async fn transactions_streams_subscribe_with_filter() { - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let mut stream = fuel_streams::Stream::::new(&client).await; // publishing 10 transactions @@ -106,17 +100,15 @@ async fn transactions_streams_subscribe_with_filter() { // creating subscription let mut sub = stream .with_filter(filter) - .subscribe_with_config(StreamConfig::default()) + .subscribe_raw_with_config(StreamConfig::default()) .await .unwrap() .take(10) .enumerate(); // result should be 10 transactions messages - while let Some((i, message)) = sub.next().await { - let message = message.unwrap(); - let payload = message.payload.clone().into(); - let decoded_msg = Transaction::decode(payload).await; + while let Some((i, bytes)) = sub.next().await { + let decoded_msg = Transaction::decode(bytes).unwrap(); let (_, transaction) = items[i].to_owned(); assert_eq!(decoded_msg, transaction); @@ -128,8 +120,7 @@ async fn transactions_streams_subscribe_with_filter() { #[tokio::test] async fn multiple_subscribers_same_subject() { - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let stream = fuel_streams::Stream::::new(&client).await; let producer = Some(Address::zeroed()); let items = publish_blocks(stream.stream(), producer.clone(), None) @@ -143,9 +134,9 @@ async fn multiple_subscribers_same_subject() { let stream = stream.clone(); let items = items.clone(); handles.push(tokio::spawn(async move { - let mut sub = stream.subscribe().await.unwrap().enumerate(); + let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes.unwrap()).await; + let decoded_msg = Block::decode_raw(bytes).unwrap(); let (subject, block) = items[i].to_owned(); let height = decoded_msg.payload.height; @@ -178,8 +169,7 @@ async fn multiple_subscribers_same_subject() { #[tokio::test] async fn multiple_subscribers_different_subjects() { - let (conn, _) = server_setup().await.unwrap(); - let client = Client::with_opts(&conn.opts).await.unwrap(); + let (_, _, client) = server_setup().await.unwrap(); let producer = Some(Address::zeroed()); let block_stream = fuel_streams::Stream::::new(&client).await; let block_items = @@ -202,9 +192,9 @@ async fn multiple_subscribers_different_subjects() { let stream = block_stream.clone(); let items = block_items.clone(); handles.push(tokio::spawn(async move { - let mut sub = stream.subscribe().await.unwrap().enumerate(); + let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes.unwrap()).await; + let decoded_msg = Block::decode_raw(bytes).unwrap(); let (subject, block) = items[i].to_owned(); let height = decoded_msg.payload.height; @@ -222,10 +212,9 @@ async fn multiple_subscribers_different_subjects() { let stream = txs_stream.clone(); let items = txs_items.clone(); handles.push(tokio::spawn(async move { - let mut sub = stream.subscribe().await.unwrap().enumerate(); + let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); while let Some((i, bytes)) = sub.next().await { - let decoded_msg = - Transaction::decode_raw(bytes.unwrap().to_vec()).await; + let decoded_msg = Transaction::decode_raw(bytes).unwrap(); let (_, transaction) = items[i].to_owned(); assert_eq!(decoded_msg.payload, transaction); if i == 9 { From da2bf5166de8717afb75eabff2f882b9c1ccd896 Mon Sep 17 00:00:00 2001 From: AJ <34186192+Jurshsmith@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:54:03 +0100 Subject: [PATCH 02/15] fix(publisher): Remove unused cli arg (#353) --- .github/CODEOWNERS | 2 +- cluster/docker/fuel-streams-publisher.Dockerfile | 1 - scripts/run_publisher.sh | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5e47a289..b0ae8132 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @luizstacio @pedronauck @jurshsmith @0xterminator +* @luizstacio @pedronauck @jurshsmith @0xterminator @cue8it diff --git a/cluster/docker/fuel-streams-publisher.Dockerfile b/cluster/docker/fuel-streams-publisher.Dockerfile index a93014e2..23c01ed1 100644 --- a/cluster/docker/fuel-streams-publisher.Dockerfile +++ b/cluster/docker/fuel-streams-publisher.Dockerfile @@ -109,7 +109,6 @@ EXPOSE ${TELEMETRY_PORT} # hadolint ignore=DL3025 CMD exec ./fuel-streams-publisher \ --service-name "${SERVICE_NAME}" \ - --nats-url $NATS_URL \ --keypair $KEYPAIR \ --relayer $RELAYER \ --ip $IP \ diff --git a/scripts/run_publisher.sh b/scripts/run_publisher.sh index b566ea34..c2170dd8 100755 --- a/scripts/run_publisher.sh +++ b/scripts/run_publisher.sh @@ -100,7 +100,6 @@ COMMON_ARGS=( "--service-name" "fuel-${NETWORK}-node" "--db-path" "./cluster/docker/db-${NETWORK}" "--snapshot" "./cluster/chain-config/${NETWORK}" - "--nats-url" "nats://localhost:4222" "--port" "${PORT}" "--telemetry-port" "${TELEMETRY_PORT}" "--peering-port" "30333" From c165b19ff2b2658a4e14e7f96658c6eab28ed75a Mon Sep 17 00:00:00 2001 From: AJ <34186192+Jurshsmith@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:52:50 +0100 Subject: [PATCH 03/15] fix(publisher): Remove trailing hyphen in bucket name (#354) --- .../src/s3/s3_client_opts.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/crates/fuel-streams-storage/src/s3/s3_client_opts.rs b/crates/fuel-streams-storage/src/s3/s3_client_opts.rs index c937a032..a0d377a8 100644 --- a/crates/fuel-streams-storage/src/s3/s3_client_opts.rs +++ b/crates/fuel-streams-storage/src/s3/s3_client_opts.rs @@ -56,16 +56,14 @@ impl S3ClientOpts { } pub fn bucket(&self) -> String { - let bucket = match self.role { + match self.role { FuelNetworkUserRole::Admin => dotenvy::var("AWS_S3_BUCKET_NAME") .expect("AWS_S3_BUCKET_NAME must be set for admin role"), - FuelNetworkUserRole::Default => self.fuel_network.to_s3_bucket(), - }; - - format!( - "{}-{}", - bucket, - self.namespace.to_owned().unwrap_or_default() - ) + FuelNetworkUserRole::Default => format!( + "{}-{}", + self.fuel_network.to_s3_bucket(), + self.namespace.to_owned().unwrap_or_default() + ), + } } } From 6776d17f35194b78c7f06387a674fcaaca6d824f Mon Sep 17 00:00:00 2001 From: AJ <34186192+Jurshsmith@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:51:20 +0100 Subject: [PATCH 04/15] fix(publisher): Don't defer error logging (#355) --- crates/fuel-streams-publisher/src/publisher/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fuel-streams-publisher/src/publisher/mod.rs b/crates/fuel-streams-publisher/src/publisher/mod.rs index 9666053a..14ec8aa8 100644 --- a/crates/fuel-streams-publisher/src/publisher/mod.rs +++ b/crates/fuel-streams-publisher/src/publisher/mod.rs @@ -281,6 +281,7 @@ pub fn publish( Ok(()) } Err(e) => { + tracing::error!("Failed to publish: {:?}", e); telemetry.log_error(&e.to_string()); telemetry.update_publisher_error_metrics( wildcard, From a700a60e665949fe591cf68e0b3cf876d0039f55 Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Thu, 12 Dec 2024 21:01:12 -0300 Subject: [PATCH 05/15] build(repo): Update Cargo.lock --- .github/workflows/update_deps.yaml | 119 --------------------------- Cargo.lock | 126 +++++++++++++++-------------- 2 files changed, 64 insertions(+), 181 deletions(-) delete mode 100644 .github/workflows/update_deps.yaml diff --git a/.github/workflows/update_deps.yaml b/.github/workflows/update_deps.yaml deleted file mode 100644 index 17f444b7..00000000 --- a/.github/workflows/update_deps.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: Bump dependencies in Cargo.lock - -on: - schedule: - # Run weekly - - cron: 0 0 * * Sun - workflow_dispatch: - # Allows manual triggering of the workflow - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - contents: write - pull-requests: write - -defaults: - run: - shell: bash - -env: - # Prevents cargo from complaining about unstable features - RUSTC_BOOTSTRAP: 1 - RUST_VERSION: 1.81.0 - PR_TITLE: "chore(deps): weekly `cargo update`" - PR_MESSAGE: | - Automation to keep dependencies in `Cargo.lock` current. - The following is the output from `cargo update`: - COMMIT_MESSAGE: "chore(deps): update dependencies\n\n" - -jobs: - update: - name: Update Dependencies - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Install Rust - uses: ./.github/actions/setup-rust - with: - toolchain: stable - - - name: Update Cargo Dependencies - run: | - cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log - if [ ! -s cargo_update.log ]; then - echo "No updates found" > cargo_update.log - fi - - - name: Cargo audit dependencies - uses: actions-rs/audit-check@v1.2.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload Cargo.lock Artifact - uses: actions/upload-artifact@v4 - with: - name: Cargo-lock - path: Cargo.lock - retention-days: 1 - - - name: Upload Cargo Update Log Artifact - uses: actions/upload-artifact@v4 - with: - name: cargo-updates - path: cargo_update.log - retention-days: 1 - - pr: - name: Create or Update Pull Request - needs: update - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Download Cargo.lock from Update Job - uses: actions/download-artifact@v4 - with: - name: Cargo-lock - - - name: Download Cargo Update Log from Update Job - uses: actions/download-artifact@v4 - with: - name: cargo-updates - - - name: Craft PR Body and Commit Message - run: | - echo "${{ env.COMMIT_MESSAGE }}" > commit.txt - cat cargo_update.log >> commit.txt - echo "${{ env.PR_MESSAGE }}" > body.md - echo '```txt' >> body.md - cat cargo_update.log >> body.md - echo '```' >> body.md - - - name: Commit Changes - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git switch --force-create cargo_update - git add ./Cargo.lock - git commit --no-verify --file=commit.txt - - - name: Create or Update Pull Request - uses: peter-evans/create-pull-request@v6 - with: - commit-message: ${{ env.COMMIT_MESSAGE }} - title: ${{ env.PR_TITLE }} - body: | - ${{ env.PR_MESSAGE }} - ```txt - $(cat cargo_update.log) - ``` - branch: cargo_update - labels: dependencies - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index d21eef11..1d69266f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,9 +505,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.12" +version = "7.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10db7e8b2042f8d7ebcfebc482622411c23f88f3e9cd7fac74465b78fdab65f0" +checksum = "0ba6d24703c5adc5ba9116901b92ee4e4c0643c01a56c4fd303f3818638d7449" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -524,6 +524,7 @@ dependencies = [ "mime", "multer", "num-traits", + "once_cell", "pin-project-lite", "regex", "serde", @@ -537,9 +538,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.12" +version = "7.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad560d871a344178c35568a15be1bbb40cbcaced57838bf2eb1f654802000df7" +checksum = "a94c2d176893486bd37cd1b6defadd999f7357bf5804e92f510c08bcf16c538f" dependencies = [ "Inflector", "async-graphql-parser", @@ -554,9 +555,9 @@ dependencies = [ [[package]] name = "async-graphql-parser" -version = "7.0.12" +version = "7.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df338e3e6469f86cce1e2b0226644e9fd82ec04790e199f8dd06416632d89ea" +checksum = "79272bdbf26af97866e149f05b2b546edb5c00e51b5f916289931ed233e208ad" dependencies = [ "async-graphql-value", "pest", @@ -566,9 +567,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.12" +version = "7.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4cffd8bb84bc7895672c4e9b71d21e35526ffd645a29aedeed165a3f4a7ba9b" +checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1" dependencies = [ "bytes", "indexmap 2.7.0", @@ -1678,9 +1679,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.3" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ "jobserver", "libc", @@ -1734,9 +1735,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1992,18 +1993,18 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const_format" -version = "0.2.34" +version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" +checksum = "50c655d81ff1114fb0dcdea9225ea9f0cc712a6f8d189378e82bdf62a473a64b" dependencies = [ "const_format_proc_macros", ] [[package]] name = "const_format_proc_macros" -version = "0.2.34" +version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +checksum = "eff1a44b93f47b1bac19a27932f5c591e43d1ba357ee4f61526c8a25603f0eb1" dependencies = [ "proc-macro2", "quote", @@ -3330,9 +3331,9 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastrand" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "ff" @@ -4061,7 +4062,7 @@ dependencies = [ "serde_json", "strum 0.26.3", "strum_macros 0.26.4", - "thiserror 2.0.6", + "thiserror 2.0.4", "tokio", ] @@ -4167,7 +4168,7 @@ dependencies = [ "displaydoc", "fuel-streams-core", "futures", - "thiserror 2.0.6", + "thiserror 2.0.4", "tokio", ] @@ -4192,7 +4193,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.8", - "thiserror 2.0.6", + "thiserror 2.0.4", "tokio", ] @@ -4255,7 +4256,7 @@ dependencies = [ "serde_prometheus", "sha2 0.10.8", "sysinfo", - "thiserror 2.0.6", + "thiserror 2.0.4", "tokio", "tokio-stream", "tracing", @@ -4278,7 +4279,7 @@ dependencies = [ "pretty_assertions", "rand", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.4", "tokio", "tracing", ] @@ -5459,9 +5460,9 @@ dependencies = [ [[package]] name = "impl-tools" -version = "0.10.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4739bc9af85c18969eba5e4db90dbf26be140ff2e5628593693f18559e9e5fe" +checksum = "8a84bc8d2baf8da56e93b4247067d918e1a44829bbbe3e4b875aaf8d7d3c7bc9" dependencies = [ "autocfg", "impl-tools-lib", @@ -5471,9 +5472,9 @@ dependencies = [ [[package]] name = "impl-tools-lib" -version = "0.11.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "798fe18a7e727001b30a029ab9cdd485afd325801d4df846f0bb5338b2986a2c" +checksum = "a795a1e201125947a063b967c79de6ae152143ab522f481d4f493c44835ba37a" dependencies = [ "proc-macro-error2", "proc-macro2", @@ -5636,9 +5637,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" dependencies = [ "once_cell", "wasm-bindgen", @@ -5713,9 +5714,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.168" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libflate" @@ -6635,9 +6636,9 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.3" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" +checksum = "cc41f430805af9d1cf4adae4ed2149c759b877b01d909a1f40256188d09345d2" dependencies = [ "core2", "unsigned-varint 0.8.0", @@ -7258,12 +7259,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.15" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", - "thiserror 2.0.6", + "thiserror 1.0.69", "ucd-trie", ] @@ -7891,7 +7892,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.19", "socket2", - "thiserror 2.0.6", + "thiserror 2.0.4", "tokio", "tracing", ] @@ -7910,7 +7911,7 @@ dependencies = [ "rustls 0.23.19", "rustls-pki-types", "slab", - "thiserror 2.0.6", + "thiserror 2.0.4", "tinyvec", "tracing", "web-time", @@ -7918,9 +7919,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ "cfg_aliases", "libc", @@ -8441,15 +8442,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -9659,11 +9660,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.4", ] [[package]] @@ -9679,9 +9680,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" dependencies = [ "proc-macro2", "quote", @@ -10472,9 +10473,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" dependencies = [ "cfg-if", "once_cell", @@ -10483,12 +10484,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" dependencies = [ "bumpalo", "log", + "once_cell", "proc-macro2", "quote", "syn 2.0.90", @@ -10497,9 +10499,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" dependencies = [ "cfg-if", "js-sys", @@ -10510,9 +10512,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10520,9 +10522,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" dependencies = [ "proc-macro2", "quote", @@ -10533,9 +10535,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" [[package]] name = "wasm-encoder" @@ -10767,9 +10769,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" dependencies = [ "js-sys", "wasm-bindgen", From 6d394c0e178731bcfffd28e1fb94ed421080adbc Mon Sep 17 00:00:00 2001 From: 0xterminator Date: Tue, 17 Dec 2024 14:18:23 +0200 Subject: [PATCH 06/15] feat(repo): Added ws streamer service (#345) * feat(repo): Added draft ws version * feat(repo): Fixes after AJs merge * feat(repo): Added docker-compose script for creating S3 bucket * fix(repo): Added better deserialization * fix(publisher): re-allow ctrl c shutdown publisher process * refactor(repo): Disallow nested Cargo.lock * refactor(publisher): Include default S3 bucket for local development * feat(repo): Improved serializations * feat(repo): Removed binary for ws-streamer * feat(repo): Pass any wildcard to stream filtering * feat(repo): Added deliver policy to ws pattern for historical data * feat(repo): Switched to tokio-tungestenite * feat(repo): Small change to websocket subscriptions --------- Co-authored-by: AJ --- .env.sample | 9 +- .github/workflows/ci.yaml | 2 + .github/workflows/docker_publish.yaml | 2 +- .gitignore | 3 + .typos.toml | 1 + Cargo.lock | 236 ++++++- Cargo.toml | 1 + Makefile | 30 +- Tiltfile | 21 + .../charts/fuel-streams-publisher/Chart.yaml | 2 +- .../tests/webserver/deployment_test.yaml | 8 +- cluster/charts/fuel-streams/values.yaml | 49 +- cluster/docker/docker-compose.yml | 7 +- .../docker/fuel-streams-publisher.Dockerfile | 2 +- cluster/docker/fuel-streams-ws.Dockerfile | 91 +++ cluster/docker/init-localstack.sh | 7 + cluster/scripts/build_streamer.sh | 16 + crates/fuel-data-parser/src/lib.rs | 8 +- crates/fuel-networks/Cargo.toml | 2 + crates/fuel-networks/src/lib.rs | 52 +- crates/fuel-streams-core/src/inputs/mod.rs | 1 + crates/fuel-streams-core/src/receipts/mod.rs | 18 +- .../src/stream/stream_impl.rs | 2 +- crates/fuel-streams-publisher/src/main.rs | 7 +- .../src/publisher/mod.rs | 5 +- .../src/publisher/shutdown.rs | 54 +- .../src/telemetry/mod.rs | 2 +- crates/fuel-streams-ws/Cargo.toml | 76 +++ crates/fuel-streams-ws/README.md | 46 ++ crates/fuel-streams-ws/config.toml | 14 + crates/fuel-streams-ws/src/cli.rs | 17 + crates/fuel-streams-ws/src/client/mod.rs | 350 ++++++++++ crates/fuel-streams-ws/src/config.rs | 193 ++++++ crates/fuel-streams-ws/src/lib.rs | 17 + crates/fuel-streams-ws/src/main.rs | 71 ++ crates/fuel-streams-ws/src/server/api.rs | 88 +++ crates/fuel-streams-ws/src/server/auth.rs | 357 ++++++++++ crates/fuel-streams-ws/src/server/context.rs | 101 +++ .../src/server/http/handlers.rs | 73 ++ crates/fuel-streams-ws/src/server/http/mod.rs | 2 + .../fuel-streams-ws/src/server/http/models.rs | 17 + .../src/server/middlewares/auth.rs | 109 +++ .../src/server/middlewares/mod.rs | 1 + crates/fuel-streams-ws/src/server/mod.rs | 7 + crates/fuel-streams-ws/src/server/state.rs | 110 +++ .../fuel-streams-ws/src/server/ws/errors.rs | 20 + .../src/server/ws/fuel_streams.rs | 198 ++++++ crates/fuel-streams-ws/src/server/ws/mod.rs | 5 + .../fuel-streams-ws/src/server/ws/models.rs | 106 +++ .../fuel-streams-ws/src/server/ws/socket.rs | 397 +++++++++++ crates/fuel-streams-ws/src/server/ws/state.rs | 55 ++ .../src/telemetry/elastic_search.rs | 323 +++++++++ .../fuel-streams-ws/src/telemetry/metrics.rs | 190 ++++++ crates/fuel-streams-ws/src/telemetry/mod.rs | 248 +++++++ .../fuel-streams-ws/src/telemetry/runtime.rs | 72 ++ .../fuel-streams-ws/src/telemetry/system.rs | 634 ++++++++++++++++++ examples/Cargo.toml | 5 + examples/websockets.rs | 55 ++ knope.toml | 42 ++ scripts/run_streamer.sh | 94 +++ scripts/set_env.sh | 20 +- tests/tests/publisher.rs | 8 +- 62 files changed, 4659 insertions(+), 100 deletions(-) create mode 100644 cluster/docker/fuel-streams-ws.Dockerfile create mode 100755 cluster/docker/init-localstack.sh create mode 100755 cluster/scripts/build_streamer.sh create mode 100644 crates/fuel-streams-ws/Cargo.toml create mode 100644 crates/fuel-streams-ws/README.md create mode 100644 crates/fuel-streams-ws/config.toml create mode 100644 crates/fuel-streams-ws/src/cli.rs create mode 100644 crates/fuel-streams-ws/src/client/mod.rs create mode 100644 crates/fuel-streams-ws/src/config.rs create mode 100644 crates/fuel-streams-ws/src/lib.rs create mode 100644 crates/fuel-streams-ws/src/main.rs create mode 100644 crates/fuel-streams-ws/src/server/api.rs create mode 100755 crates/fuel-streams-ws/src/server/auth.rs create mode 100644 crates/fuel-streams-ws/src/server/context.rs create mode 100644 crates/fuel-streams-ws/src/server/http/handlers.rs create mode 100644 crates/fuel-streams-ws/src/server/http/mod.rs create mode 100644 crates/fuel-streams-ws/src/server/http/models.rs create mode 100644 crates/fuel-streams-ws/src/server/middlewares/auth.rs create mode 100644 crates/fuel-streams-ws/src/server/middlewares/mod.rs create mode 100644 crates/fuel-streams-ws/src/server/mod.rs create mode 100644 crates/fuel-streams-ws/src/server/state.rs create mode 100644 crates/fuel-streams-ws/src/server/ws/errors.rs create mode 100644 crates/fuel-streams-ws/src/server/ws/fuel_streams.rs create mode 100644 crates/fuel-streams-ws/src/server/ws/mod.rs create mode 100644 crates/fuel-streams-ws/src/server/ws/models.rs create mode 100644 crates/fuel-streams-ws/src/server/ws/socket.rs create mode 100644 crates/fuel-streams-ws/src/server/ws/state.rs create mode 100755 crates/fuel-streams-ws/src/telemetry/elastic_search.rs create mode 100644 crates/fuel-streams-ws/src/telemetry/metrics.rs create mode 100644 crates/fuel-streams-ws/src/telemetry/mod.rs create mode 100644 crates/fuel-streams-ws/src/telemetry/runtime.rs create mode 100644 crates/fuel-streams-ws/src/telemetry/system.rs create mode 100755 examples/websockets.rs create mode 100755 scripts/run_streamer.sh diff --git a/.env.sample b/.env.sample index 65a2309a..5b6c89de 100644 --- a/.env.sample +++ b/.env.sample @@ -9,11 +9,18 @@ AWS_REGION=us-east-1 AWS_ENDPOINT_URL=http://localhost:4566 AWS_S3_BUCKET_NAME=fuel-streams-testnet USE_ELASTIC_LOGGING=false -USE_PUBLISHER_METRICS=true +USE_METRICS=true PUBLISHER_MAX_THREADS=16 ELASTICSEARCH_URL=http://127.0.0.1:9200 ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=generated-secret +AWS_S3_ENABLED=false +AWS_ACCESS_KEY_ID=s3-access-key-id +AWS_SECRET_ACCESS_KEY=s3-secret-access-key +AWS_REGION=s3-region +AWS_ENDPOINT_URL=s3-endpoint +AWS_S3_BUCKET_NAME=fuel-streams-local +JWT_AUTH_SECRET=generated-secret # Mainnet Configuration MAINNET_RELAYER=https://mainnet.infura.io/v3/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b1a241b..cb25b762 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -228,6 +228,7 @@ jobs: - fuel-streams-core - fuel-streams-macros - fuel-streams-publisher + - fuel-streams-ws steps: - uses: actions/checkout@v4 @@ -263,6 +264,7 @@ jobs: matrix: package: - fuel-streams-publisher + - fuel-streams-ws is_release: - ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} platform: diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index 47ba1bd8..74f5167e 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -34,7 +34,7 @@ jobs: - name: Build and push Docker (${{ steps.sha.outputs.short_sha }}) uses: ./.github/actions/docker-publish - id: publish + id: publish-fuel-streams-nats with: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index b46e0b73..9b22fe23 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ values-publisher-secrets.yaml values-publisher-env.yaml localstack-data .vscode + +**/Cargo.lock +!./Cargo.lock diff --git a/.typos.toml b/.typos.toml index e8e34136..97819a36 100644 --- a/.typos.toml +++ b/.typos.toml @@ -3,6 +3,7 @@ extend-exclude = [ "pnpm-lock.yaml", "crates/fuel-streams-publisher/README.md", "crates/fuel-streams-publisher/src/elastic.rs", + "crates/fuel-streams-ws/README.md", "docker/chain-config", "docker/monitoring", "cluster", diff --git a/Cargo.lock b/Cargo.lock index 1d69266f..aa6a87fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,6 +210,20 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "actix-ws" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a1fb4f9f2794b0aadaf2ba5f14a6f034c7e86957b458c506a8cb75953f2d99" +dependencies = [ + "actix-codec", + "actix-http", + "actix-web", + "bytestring", + "futures-core", + "tokio", +] + [[package]] name = "addr2line" version = "0.24.2" @@ -1231,7 +1245,7 @@ dependencies = [ "rustversion", "serde", "sync_wrapper 1.0.2", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", ] @@ -1959,6 +1973,18 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "confy" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0" +dependencies = [ + "directories", + "serde", + "thiserror 1.0.69", + "toml", +] + [[package]] name = "console" version = "0.15.8" @@ -2783,6 +2809,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys 0.4.1", +] + [[package]] name = "directories-next" version = "2.0.0" @@ -2799,7 +2834,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "dirs-sys", + "dirs-sys 0.3.7", ] [[package]] @@ -2813,6 +2848,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -3252,7 +3299,7 @@ dependencies = [ "hashers", "http 0.2.12", "instant", - "jsonwebtoken", + "jsonwebtoken 8.3.0", "once_cell", "pin-project", "reqwest 0.11.27", @@ -3260,7 +3307,7 @@ dependencies = [ "serde_json", "thiserror 1.0.69", "tokio", - "tokio-tungstenite", + "tokio-tungstenite 0.20.1", "tracing", "tracing-futures", "url", @@ -4135,6 +4182,8 @@ name = "fuel-networks" version = "0.0.13" dependencies = [ "clap 4.5.23", + "serde", + "url", ] [[package]] @@ -4204,6 +4253,7 @@ dependencies = [ "anyhow", "fuel-core-types 0.40.2", "fuel-streams", + "fuel-streams-ws", "futures", "tokio", ] @@ -4284,6 +4334,57 @@ dependencies = [ "tracing", ] +[[package]] +name = "fuel-streams-ws" +version = "0.0.13" +dependencies = [ + "actix-cors", + "actix-server", + "actix-service", + "actix-web", + "actix-ws", + "anyhow", + "async-nats", + "async-trait", + "bytestring", + "chrono", + "clap 4.5.23", + "confy", + "derive_more 1.0.0", + "displaydoc", + "dotenvy", + "elasticsearch", + "fuel-streams", + "fuel-streams-core", + "fuel-streams-storage", + "futures", + "futures-util", + "jsonwebtoken 9.3.0", + "num_cpus", + "openssl", + "parking_lot", + "prometheus", + "rand", + "reqwest 0.12.9", + "rust_decimal", + "serde", + "serde_json", + "serde_prometheus", + "sysinfo", + "thiserror 2.0.4", + "time", + "tokio", + "tokio-tungstenite 0.24.0", + "toml", + "tracing", + "tracing-actix-web", + "tracing-subscriber", + "url", + "urlencoding", + "uuid", + "validator", +] + [[package]] name = "fuel-tx" version = "0.56.0" @@ -4865,9 +4966,9 @@ checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" [[package]] name = "hickory-proto" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" +checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" dependencies = [ "async-trait", "cfg-if", @@ -4876,7 +4977,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna 0.4.0", + "idna 1.0.3", "ipnet", "once_cell", "rand", @@ -4890,9 +4991,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" +checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" dependencies = [ "cfg-if", "futures-util", @@ -5342,16 +5443,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "1.0.3" @@ -5665,6 +5756,21 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" +dependencies = [ + "base64 0.21.7", + "js-sys", + "pem 3.0.4", + "ring 0.17.8", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "k256" version = "0.13.4" @@ -6781,9 +6887,9 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416060d346fbaf1f23f9512963e3e878f1a78e707cb699ba9215761754244307" +checksum = "16c903aa70590cb93691bf97a767c8d1d6122d2cc9070433deb3bbf36ce8bd23" dependencies = [ "bytes", "futures", @@ -7078,6 +7184,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -7601,6 +7713,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", + "syn 2.0.90", ] [[package]] @@ -8180,8 +8293,11 @@ dependencies = [ "bytes", "cookie 0.18.1", "cookie_store 0.21.1", + "encoding_rs", + "futures-channel", "futures-core", "futures-util", + "h2 0.4.7", "http 1.2.0", "http-body 1.0.1", "http-body-util", @@ -8205,6 +8321,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.2", + "system-configuration 0.6.1", "tokio", "tokio-native-tls", "tokio-rustls 0.26.1", @@ -8801,9 +8918,9 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] @@ -8819,9 +8936,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", @@ -9896,10 +10013,22 @@ dependencies = [ "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", - "tungstenite", + "tungstenite 0.20.1", "webpki-roots 0.25.4", ] +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.24.0", +] + [[package]] name = "tokio-util" version = "0.7.13" @@ -10047,14 +10176,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", ] @@ -10253,6 +10382,24 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.2.0", + "httparse", + "log", + "rand", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" @@ -10414,6 +10561,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", + "serde", +] + +[[package]] +name = "validator" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b4a29d8709210980a09379f27ee31549b73292c87ab9899beee1c0d3be6303" +dependencies = [ + "idna 1.0.3", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac855a2ce6f843beb229757e6e570a42e837bcb15e5f449dd48d5747d41bf77" +dependencies = [ + "darling 0.20.10", + "once_cell", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1cf2d1c6..1b17bb8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ fuel-networks = { path = "crates/fuel-networks" } fuel-data-parser = { version = "0.0.13", path = "crates/fuel-data-parser" } fuel-streams-core = { version = "0.0.13", path = "crates/fuel-streams-core" } fuel-streams-publisher = { version = "0.0.13", path = "crates/fuel-streams-publisher" } +fuel-streams-ws = { version = "0.0.13", path = "crates/fuel-streams-ws" } fuel-streams-macros = { version = "0.0.13", path = "crates/fuel-streams-macros" } fuel-streams-storage = { version = "0.0.13", path = "crates/fuel-streams-storage" } subject-derive = { version = "0.0.13", path = "crates/fuel-streams-macros/subject-derive" } diff --git a/Makefile b/Makefile index 937e3fb6..b8bdb8af 100644 --- a/Makefile +++ b/Makefile @@ -209,18 +209,40 @@ run-publisher: check-network $(if $(TELEMETRY_PORT),--telemetry-port $(TELEMETRY_PORT),) \ $(if $(extra_args),--extra-args "$(extra_args)",) -run-mainnet-dev: +run-publisher-mainnet-dev: $(MAKE) run-publisher NETWORK=mainnet MODE=dev -run-mainnet-profiling: +run-publisher-mainnet-profiling: $(MAKE) run-publisher NETWORK=mainnet MODE=profiling -run-testnet-dev: +run-publisher-testnet-dev: $(MAKE) run-publisher NETWORK=testnet MODE=dev -run-testnet-profiling: +run-publisher-testnet-profiling: $(MAKE) run-publisher NETWORK=testnet MODE=profiling +# ------------------------------------------------------------ +# Streamer Run Commands +# ------------------------------------------------------------ + +run-streamer: check-network + @./scripts/run_streamer.sh \ + --mode $(MODE) \ + $(if $(CONFIG_PATH),--config-path $(CONFIG_PATH),) \ + $(if $(extra_args),--extra-args "$(extra_args)",) + +run-streamer-mainnet-dev: + $(MAKE) run-streamer NETWORK=mainnet MODE=dev CONFIG_PATH=crates/fuel-streams-ws/config.toml + +run-streamer-mainnet-profiling: + $(MAKE) run-streamer NETWORK=mainnet MODE=profiling CONFIG_PATH=crates/fuel-streams-ws/config.toml + +run-streamer-testnet-dev: + $(MAKE) run-streamer NETWORK=testnet MODE=dev CONFIG_PATH=crates/fuel-streams-ws/config.toml + +run-streamer-testnet-profiling: + $(MAKE) run-streamer NETWORK=testnet MODE=profiling CONFIG_PATH=crates/fuel-streams-ws/config.toml + # ------------------------------------------------------------ # Docker Compose # ------------------------------------------------------------ diff --git a/Tiltfile b/Tiltfile index 254afdb9..2618c336 100755 --- a/Tiltfile +++ b/Tiltfile @@ -30,6 +30,27 @@ custom_build( ignore=['./target'] ) +# Build streamer ws image with proper configuration for Minikube +custom_build( + ref='fuel-streams-ws:latest', + command=['./cluster/scripts/build_streamer.sh'], + deps=[ + './src', + './Cargo.toml', + './Cargo.lock', + './docker/fuel-streams-ws.Dockerfile' + ], + live_update=[ + sync('./src', '/usr/src'), + sync('./Cargo.toml', '/usr/src/Cargo.toml'), + sync('./Cargo.lock', '/usr/src/Cargo.lock'), + run('cargo build', trigger=['./src', './Cargo.toml', './Cargo.lock']) + ], + skips_local_docker=True, + ignore=['./target'] +) + +# Deploy the Helm chart with values from .env # Get deployment mode from environment variable, default to 'full' config_mode = os.getenv('CLUSTER_MODE', 'full') diff --git a/cluster/charts/fuel-streams-publisher/Chart.yaml b/cluster/charts/fuel-streams-publisher/Chart.yaml index c3ed7724..1628fc80 100644 --- a/cluster/charts/fuel-streams-publisher/Chart.yaml +++ b/cluster/charts/fuel-streams-publisher/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: fuel-streams-publisher -description: A Helm chart for Kubernetes deployment of Fuel Core NATS +description: A Helm chart for Kubernetes deployment of Fuel streams publisher service # A chart can be either an 'application' or a 'library' chart. # # Application charts are a collection of templates that can be packaged into versioned archives diff --git a/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml b/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml index 71aee4ac..84833c59 100644 --- a/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml +++ b/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml @@ -66,10 +66,10 @@ tests: - it: should configure ports correctly set: webserver.enabled: true - webserver.service.port: 8082 + webserver.service.port: 9003 webserver.ports: - name: metrics - containerPort: 9090 + containerPort: 9003 protocol: TCP asserts: - lengthEqual: @@ -79,13 +79,13 @@ tests: path: spec.template.spec.containers[0].ports content: name: webserver - containerPort: 8082 + containerPort: 9003 protocol: TCP - contains: path: spec.template.spec.containers[0].ports content: name: metrics - containerPort: 9090 + containerPort: 9003 protocol: TCP - it: should set replicas when autoscaling is disabled diff --git a/cluster/charts/fuel-streams/values.yaml b/cluster/charts/fuel-streams/values.yaml index e00d00d1..a148f4c9 100755 --- a/cluster/charts/fuel-streams/values.yaml +++ b/cluster/charts/fuel-streams/values.yaml @@ -189,11 +189,11 @@ publisher: # ------------------------------------------------------------------------------------------------- webserver: - enabled: false - port: 8082 + enabled: true + port: 9003 image: - repository: fuel-streams-webserver + repository: fuel-streams-ws pullPolicy: Never tag: "latest" @@ -227,6 +227,49 @@ webserver: targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: 80 + env: + STREAMER_MAX_WORKERS: "10" + STREAMER_API_PORT: 9003 + JWT_AUTH_SECRET: "secret" + USE_ELASTIC_LOGGING: false + USE_METRICS: true + AWS_S3_ENABLED: true + NATS_URL: "fuel-streams-nats-publisher:4222" + NETWORK: testnet + + # Additional environment variables with complex structures + # extraEnv: [] + # - name: AWS_ACCESS_KEY_ID + # valueFrom: + # secretKeyRef: + # name: fuel-streams-webserver + # key: AWS_ACCESS_KEY_ID + # - name: AWS_SECRET_ACCESS_KEY + # valueFrom: + # secretKeyRef: + # name: fuel-streams-webserver + # key: AWS_SECRET_ACCESS_KEY + # - name: AWS_REGION + # valueFrom: + # secretKeyRef: + # name: fuel-streams-webserver + # key: AWS_REGION + # - name: AWS_S3_BUCKET_NAME + # valueFrom: + # secretKeyRef: + # name: fuel-streams-webserver + # key: AWS_S3_BUCKET_NAME + # - name: AWS_ENDPOINT_URL + # valueFrom: + # secretKeyRef: + # name: fuel-streams-webserver + # key: AWS_ENDPOINT_URL + # Optional: Bulk environment references + # envFrom: {} + # - configMapRef: + # name: additional-config + # - secretRef: + # name: additional-secrets # ------------------------------------------------------------------------------------------------- # NATS Core configuration # ------------------------------------------------------------------------------------------------- diff --git a/cluster/docker/docker-compose.yml b/cluster/docker/docker-compose.yml index e007f202..1966f3c3 100644 --- a/cluster/docker/docker-compose.yml +++ b/cluster/docker/docker-compose.yml @@ -19,11 +19,6 @@ services: env_file: - ./../../.env localstack: - profiles: - - all - - s3 - - dev - - monitoring image: localstack/localstack:latest container_name: localstack restart: always @@ -36,6 +31,8 @@ services: - AWS_ACCESS_KEY_ID=test - AWS_SECRET_ACCESS_KEY=test - DEFAULT_REGION=us-east-1 + - DEFAULT_BUCKETS=fuel-streams-local volumes: - ./localstack-data:/var/lib/localstack - /var/run/docker.sock:/var/run/docker.sock + - ./init-localstack.sh:/etc/localstack/init/ready.d/init-localstack.sh diff --git a/cluster/docker/fuel-streams-publisher.Dockerfile b/cluster/docker/fuel-streams-publisher.Dockerfile index 23c01ed1..a1b7f00f 100644 --- a/cluster/docker/fuel-streams-publisher.Dockerfile +++ b/cluster/docker/fuel-streams-publisher.Dockerfile @@ -84,7 +84,7 @@ ENV RELAYER_V2_LISTENING_CONTRACTS= ENV RELAYER_DA_DEPLOY_HEIGHT= ENV CHAIN_CONFIG= ENV NATS_URL= -ENV USE_PUBLISHER_METRICS= +ENV USE_METRICS= ENV USE_ELASTIC_LOGGING= WORKDIR /usr/src diff --git a/cluster/docker/fuel-streams-ws.Dockerfile b/cluster/docker/fuel-streams-ws.Dockerfile new file mode 100644 index 00000000..435eb1f8 --- /dev/null +++ b/cluster/docker/fuel-streams-ws.Dockerfile @@ -0,0 +1,91 @@ +# Stage 1: Build +FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx +FROM --platform=$BUILDPLATFORM rust:1.81.0 AS chef + +ARG TARGETPLATFORM +RUN cargo install cargo-chef && rustup target add wasm32-unknown-unknown +WORKDIR /build/ + +COPY --from=xx / / + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + lld \ + clang \ + libclang-dev \ + && xx-apt-get update \ + && xx-apt-get install -y libc6-dev g++ binutils \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +FROM chef AS planner +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + + +FROM chef AS builder +ARG DEBUG_SYMBOLS=false +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_PROFILE_RELEASE_DEBUG=$DEBUG_SYMBOLS +COPY --from=planner /build/recipe.json recipe.json +RUN echo $CARGO_PROFILE_RELEASE_DEBUG +# Build our project dependencies, not our application! +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/build/target \ + xx-cargo chef cook --release --no-default-features -p fuel-streams-ws --recipe-path recipe.json +# Up to this point, if our dependency tree stays the same, +# all layers should be cached. +COPY . . +# build application +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/build/target \ + xx-cargo build --release --no-default-features -p fuel-streams-ws \ + && xx-verify ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-ws \ + && cp ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-ws /root/fuel-streams-ws \ + && cp ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-ws.d /root/fuel-streams-ws.d + +# Stage 2: Run +FROM ubuntu:22.04 AS run + +ARG STREAMER_API_PORT=9003 + +ENV STREAMER_API_PORT=$STREAMER_API_PORT +ENV NATS_URL= +ENV NETWORK= +ENV USE_METRICS= +ENV USE_ELASTIC_LOGGING= +ENV AWS_S3_ENABLED= +ENV AWS_ACCESS_KEY_ID= +ENV AWS_SECRET_ACCESS_KEY= +ENV AWS_REGION= +ENV AWS_ENDPOINT_URL= +ENV AWS_S3_BUCKET_NAME= +ENV JWT_AUTH_SECRET= + +WORKDIR /usr/src + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + # Clean up + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /root/fuel-streams-ws . +COPY --from=builder /root/fuel-streams-ws.d . + +EXPOSE ${STREAMER_API_PORT} + +# https://stackoverflow.com/a/44671685 +# https://stackoverflow.com/a/40454758 +# hadolint ignore=DL3025 +CMD exec ./fuel-streams-ws diff --git a/cluster/docker/init-localstack.sh b/cluster/docker/init-localstack.sh new file mode 100755 index 00000000..9ad8d0d7 --- /dev/null +++ b/cluster/docker/init-localstack.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +echo "Creating S3 bucket in LocalStack..." + +awslocal s3 mb s3://fuel-streams-test +echo "Bucket created: fuel-streams-test" diff --git a/cluster/scripts/build_streamer.sh b/cluster/scripts/build_streamer.sh new file mode 100755 index 00000000..48dd5375 --- /dev/null +++ b/cluster/scripts/build_streamer.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Use environment variables provided by Tilt if available +IMAGE_NAME=${EXPECTED_IMAGE:-"fuel-streams-ws"} +TAG=${EXPECTED_TAG:-"latest"} +DOCKERFILE="docker/fuel-streams-ws.Dockerfile" + +# Ensure we're using minikube's docker daemon if not already set +if [ -z "${DOCKER_HOST:-}" ]; then + eval $(minikube docker-env) +fi + +# Build the docker image +docker build -t ${IMAGE_NAME}:${TAG} -f ${DOCKERFILE} . diff --git a/crates/fuel-data-parser/src/lib.rs b/crates/fuel-data-parser/src/lib.rs index 74b634f0..78bc227f 100644 --- a/crates/fuel-data-parser/src/lib.rs +++ b/crates/fuel-data-parser/src/lib.rs @@ -25,7 +25,13 @@ pub enum SerializationType { /// Traits required for a data type to be parseable pub trait DataParseable: - serde::Serialize + serde::de::DeserializeOwned + Clone + Send + Sync + Debug + serde::Serialize + + serde::de::DeserializeOwned + + Clone + + Send + + Sync + + Debug + + std::marker::Sized { } diff --git a/crates/fuel-networks/Cargo.toml b/crates/fuel-networks/Cargo.toml index 29a165a5..cc8000ad 100644 --- a/crates/fuel-networks/Cargo.toml +++ b/crates/fuel-networks/Cargo.toml @@ -12,3 +12,5 @@ rust-version = { workspace = true } [dependencies] clap = { workspace = true } +serde = { workspace = true } +url = "2.5" diff --git a/crates/fuel-networks/src/lib.rs b/crates/fuel-networks/src/lib.rs index 4a3ac3d0..78c3c963 100644 --- a/crates/fuel-networks/src/lib.rs +++ b/crates/fuel-networks/src/lib.rs @@ -1,4 +1,8 @@ +use std::str::FromStr; + /// FuelStreamsNetworks; shortened to FuelNetworks for brievity and public familiarity +use serde::{Deserialize, Serialize}; +use url::Url; #[derive(Debug, Clone, Default)] pub enum FuelNetworkUserRole { @@ -7,7 +11,10 @@ pub enum FuelNetworkUserRole { Default, } -#[derive(Debug, Copy, Clone, Default, clap::ValueEnum)] +#[derive( + Debug, Copy, Clone, Default, clap::ValueEnum, Deserialize, Serialize, +)] +#[serde(rename_all = "lowercase")] pub enum FuelNetwork { #[default] Local, @@ -15,6 +22,19 @@ pub enum FuelNetwork { Mainnet, } +impl FromStr for FuelNetwork { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "local" => Ok(FuelNetwork::Local), + "testnet" => Ok(FuelNetwork::Testnet), + "mainnet" => Ok(FuelNetwork::Mainnet), + _ => Err(format!("unknown network: {}", s)), + } + } +} + impl std::fmt::Display for FuelNetwork { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -43,6 +63,36 @@ impl FuelNetwork { .to_string() } + pub fn to_web_url(&self) -> Url { + match self { + FuelNetwork::Local => { + Url::parse("http://0.0.0.0:9003").expect("working url") + } + FuelNetwork::Testnet => { + Url::parse("http://stream-testnet.fuel.network:9003") + .expect("working url") + } + FuelNetwork::Mainnet => { + Url::parse("http://stream.fuel.network:9003") + .expect("working url") + } + } + } + + pub fn to_ws_url(&self) -> Url { + match self { + FuelNetwork::Local => { + Url::parse("ws://0.0.0.0:9003").expect("working url") + } + FuelNetwork::Testnet => { + Url::parse("ws://stream-testnet.fuel.network:9003") + .expect("working url") + } + FuelNetwork::Mainnet => Url::parse("ws://stream.fuel.network:9003") + .expect("working url"), + } + } + pub fn to_s3_url(&self) -> String { match self { FuelNetwork::Local => "http://localhost:4566".to_string(), diff --git a/crates/fuel-streams-core/src/inputs/mod.rs b/crates/fuel-streams-core/src/inputs/mod.rs index c18fb4e1..4e2e7db9 100644 --- a/crates/fuel-streams-core/src/inputs/mod.rs +++ b/crates/fuel-streams-core/src/inputs/mod.rs @@ -13,5 +13,6 @@ impl Streamable for Input { InputsCoinSubject::WILDCARD, InputsContractSubject::WILDCARD, InputsMessageSubject::WILDCARD, + InputsByIdSubject::WILDCARD, ]; } diff --git a/crates/fuel-streams-core/src/receipts/mod.rs b/crates/fuel-streams-core/src/receipts/mod.rs index 1da02fd6..c2ba4f4a 100644 --- a/crates/fuel-streams-core/src/receipts/mod.rs +++ b/crates/fuel-streams-core/src/receipts/mod.rs @@ -9,6 +9,20 @@ use crate::{StreamEncoder, Streamable}; impl StreamEncoder for Receipt {} impl Streamable for Receipt { const NAME: &'static str = "receipts"; - const WILDCARD_LIST: &'static [&'static str] = - &[ReceiptsCallSubject::WILDCARD, ReceiptsByIdSubject::WILDCARD]; + const WILDCARD_LIST: &'static [&'static str] = &[ + ReceiptsCallSubject::WILDCARD, + ReceiptsByIdSubject::WILDCARD, + ReceiptsBurnSubject::WILDCARD, + ReceiptsLogSubject::WILDCARD, + ReceiptsMintSubject::WILDCARD, + ReceiptsPanicSubject::WILDCARD, + ReceiptsReturnSubject::WILDCARD, + ReceiptsRevertSubject::WILDCARD, + ReceiptsLogDataSubject::WILDCARD, + ReceiptsTransferSubject::WILDCARD, + ReceiptsMessageOutSubject::WILDCARD, + ReceiptsReturnDataSubject::WILDCARD, + ReceiptsTransferOutSubject::WILDCARD, + ReceiptsScriptResultSubject::WILDCARD, + ]; } diff --git a/crates/fuel-streams-core/src/stream/stream_impl.rs b/crates/fuel-streams-core/src/stream/stream_impl.rs index 984e05e4..186f0896 100644 --- a/crates/fuel-streams-core/src/stream/stream_impl.rs +++ b/crates/fuel-streams-core/src/stream/stream_impl.rs @@ -58,7 +58,7 @@ impl PublishPacket { /// } /// ``` #[async_trait] -pub trait Streamable: StreamEncoder { +pub trait Streamable: StreamEncoder + std::marker::Sized { const NAME: &'static str; const WILDCARD_LIST: &'static [&'static str]; diff --git a/crates/fuel-streams-publisher/src/main.rs b/crates/fuel-streams-publisher/src/main.rs index 2d2a1602..cc667404 100644 --- a/crates/fuel-streams-publisher/src/main.rs +++ b/crates/fuel-streams-publisher/src/main.rs @@ -8,6 +8,7 @@ use fuel_streams_publisher::{ cli::Cli, publisher::shutdown::ShutdownController, server::{http::create_web_server, state::ServerState}, + shutdown, telemetry::Telemetry, FuelCore, FuelCoreLike, @@ -48,9 +49,9 @@ async fn main() -> anyhow::Result<()> { }); tracing::info!("Publisher started."); - let shutdown_controller = ShutdownController::new().arc(); - let shutdown_token = shutdown_controller.get_token(); - ShutdownController::spawn_signal_listener(shutdown_controller); + let (shutdown_controller, shutdown_token) = + shutdown::get_controller_and_token(); + ShutdownController::spawn_signal_listener(&shutdown_controller); // run publisher until shutdown signal intercepted if let Err(err) = publisher.run(shutdown_token, historical).await { diff --git a/crates/fuel-streams-publisher/src/publisher/mod.rs b/crates/fuel-streams-publisher/src/publisher/mod.rs index 14ec8aa8..2873857c 100644 --- a/crates/fuel-streams-publisher/src/publisher/mod.rs +++ b/crates/fuel-streams-publisher/src/publisher/mod.rs @@ -42,6 +42,9 @@ impl Publisher { let s3_client_opts = S3ClientOpts::admin_opts(); let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); + if let Err(e) = s3_client.create_bucket().await { + tracing::error!("Failed to create S3 bucket: {:?}", e); + } let fuel_streams = Arc::new(FuelStreams::new(&nats_client, &s3_client).await); @@ -112,7 +115,7 @@ impl Publisher { const MAX_RETAINED_BLOCKS: u64 = 100; pub async fn run( &self, - shutdown_token: ShutdownToken, + mut shutdown_token: ShutdownToken, historical: bool, ) -> anyhow::Result<()> { tracing::info!("Awaiting FuelCore Sync..."); diff --git a/crates/fuel-streams-publisher/src/publisher/shutdown.rs b/crates/fuel-streams-publisher/src/publisher/shutdown.rs index 606fe97d..89963fff 100644 --- a/crates/fuel-streams-publisher/src/publisher/shutdown.rs +++ b/crates/fuel-streams-publisher/src/publisher/shutdown.rs @@ -1,4 +1,4 @@ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use tokio::{ signal::unix::{signal, SignalKind}, @@ -10,53 +10,25 @@ use tokio::{ pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(90); // First, let's create a ShutdownToken that can be shared -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct ShutdownToken { - receiver: Arc>, + receiver: broadcast::Receiver<()>, } impl ShutdownToken { - pub async fn wait_for_shutdown(&self) -> bool { - let mut rx = self.receiver.resubscribe(); - rx.recv().await.is_ok() + pub async fn wait_for_shutdown(&mut self) -> bool { + self.receiver.recv().await.is_ok() } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ShutdownController { sender: broadcast::Sender<()>, - token: ShutdownToken, shutdown_initiated: OnceCell<()>, } -impl Default for ShutdownController { - fn default() -> Self { - Self::new() - } -} - impl ShutdownController { - pub fn new() -> Self { - let (sender, receiver) = broadcast::channel(1); - let token = ShutdownToken { - receiver: Arc::new(receiver), - }; - - Self { - sender, - token, - shutdown_initiated: OnceCell::new(), - } - } - pub fn arc(self) -> Arc { - Arc::new(self) - } - - pub fn get_token(&self) -> ShutdownToken { - self.token.clone() - } - - pub fn spawn_signal_listener(self: Arc) { + pub fn spawn_signal_listener(&self) { let sender = self.sender.clone(); tokio::spawn(async move { let mut sigint = @@ -87,3 +59,15 @@ impl ShutdownController { } } } + +pub fn get_controller_and_token() -> (ShutdownController, ShutdownToken) { + let (sender, receiver) = broadcast::channel(1); + + ( + ShutdownController { + sender, + shutdown_initiated: OnceCell::new(), + }, + ShutdownToken { receiver }, + ) +} diff --git a/crates/fuel-streams-publisher/src/telemetry/mod.rs b/crates/fuel-streams-publisher/src/telemetry/mod.rs index 13e1ebe1..17f26842 100644 --- a/crates/fuel-streams-publisher/src/telemetry/mod.rs +++ b/crates/fuel-streams-publisher/src/telemetry/mod.rs @@ -273,5 +273,5 @@ impl Telemetry { } pub fn should_use_publisher_metrics() -> bool { - dotenvy::var("USE_PUBLISHER_METRICS").is_ok_and(|val| val == "true") + dotenvy::var("USE_METRICS").is_ok_and(|val| val == "true") } diff --git a/crates/fuel-streams-ws/Cargo.toml b/crates/fuel-streams-ws/Cargo.toml new file mode 100644 index 00000000..86c66c2d --- /dev/null +++ b/crates/fuel-streams-ws/Cargo.toml @@ -0,0 +1,76 @@ +[package] +name = "fuel-streams-ws" +description = "Fuel library for streaming data from nats and storage" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } +publish = false + +[dependencies] +actix-cors = { workspace = true } +actix-server = { workspace = true } +actix-service = "2.0.2" +actix-web = { workspace = true } +actix-ws = "0.3.0" +anyhow = { workspace = true } +async-nats = { workspace = true } +async-trait = { workspace = true } +bytestring = "1.4.0" +chrono = { workspace = true } +clap = { workspace = true } +confy = "0.6" +derive_more = { version = "1.0", features = ["full"] } +displaydoc = { workspace = true } +dotenvy = { workspace = true } +elasticsearch = "8.15.0-alpha.1" +fuel-streams = { workspace = true, features = ["test-helpers"] } +fuel-streams-core = { workspace = true, features = ["test-helpers"] } +fuel-streams-storage = { workspace = true, features = ["test-helpers"] } +futures = { workspace = true } +futures-util = { workspace = true } +jsonwebtoken = "9.3.0" +num_cpus = "1.16" +parking_lot = { version = "0.12", features = ["serde"] } +prometheus = { version = "0.13", features = ["process"] } +rand = { workspace = true } +reqwest = { version = "^0.12.9", features = ["json", "blocking"] } +rust_decimal = { version = "1.13" } +serde = { workspace = true } +serde_json = { workspace = true } +serde_prometheus = { version = "0.2" } +sysinfo = { version = "0.29" } +thiserror = "2.0" +time = { version = "0.3", features = ["serde"] } +tokio = { workspace = true } +tokio-tungstenite = "0.24.0" +toml = "0.8.19" +tracing = { workspace = true } +tracing-actix-web = { workspace = true } +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +url = "2.5" +urlencoding = "2.1" +uuid = { version = "1.11.0", features = ["serde", "v4"] } +validator = { version = "0.19.0", features = ["derive"] } + +[dev-dependencies] + +[features] +default = [] +test-helpers = [] + +[target.x86_64-unknown-linux-gnu.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.x86_64-unknown-linux-musl.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.aarch64-unknown-linux-gnu.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.aarch64-unknown-linux-musl.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } diff --git a/crates/fuel-streams-ws/README.md b/crates/fuel-streams-ws/README.md new file mode 100644 index 00000000..5c60af56 --- /dev/null +++ b/crates/fuel-streams-ws/README.md @@ -0,0 +1,46 @@ +
+
+ + Logo + +

Fuel Streams Websockets

+

+ A binary webserver that consumes events published by fuel-streams and streams them further via websockets upon subscription +

+

+ + CI + + + Coverage + +

+

+ 📚 Documentation +   + 🐛 Report Bug +   + ✨ Request Feature +

+
+ +## 📝 About The Project + +A binary that consumes events published by fuel-streams and streams them further via websockets. + +## ⚡️ Getting Started + +### Prerequisites + +- [Rust toolchain](https://www.rust-lang.org/tools/install) +- [Docker](https://www.docker.com/get-started/) (optional) + +## 🤝 Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. + +For more information on contributing, please see the [CONTRIBUTING.md](../../CONTRIBUTING.md) file in the root of the repository. + +## 📜 License + +This project is licensed under the `Apache-2.0` license. See [`LICENSE`](../../LICENSE) for more information. diff --git a/crates/fuel-streams-ws/config.toml b/crates/fuel-streams-ws/config.toml new file mode 100644 index 00000000..1c71a7a4 --- /dev/null +++ b/crates/fuel-streams-ws/config.toml @@ -0,0 +1,14 @@ +[api] +port = 9003 + +[nats] +network = "local" + +[fuel] +network = "local" + +[s3] +enabled = true + +[auth] +jwt-secret = "secret" diff --git a/crates/fuel-streams-ws/src/cli.rs b/crates/fuel-streams-ws/src/cli.rs new file mode 100644 index 00000000..b27bb40a --- /dev/null +++ b/crates/fuel-streams-ws/src/cli.rs @@ -0,0 +1,17 @@ +use clap::Parser; + +/// CLI structure for parsing command-line arguments. +/// +/// - `config_path`: Path to the toml config file. +#[derive(Clone, Parser)] +pub struct Cli { + /// Config path + #[arg( + long, + value_name = "CONFIG", + env = "CONFIG_PATH", + default_value = "config.toml", + help = "Path to toml config file" + )] + pub config_path: Option, +} diff --git a/crates/fuel-streams-ws/src/client/mod.rs b/crates/fuel-streams-ws/src/client/mod.rs new file mode 100644 index 00000000..bbf7aa5c --- /dev/null +++ b/crates/fuel-streams-ws/src/client/mod.rs @@ -0,0 +1,350 @@ +use fuel_streams::{ + logs::Log, + subjects::IntoSubject, + types::{Block, FuelNetwork, Input, Output, Receipt, Transaction}, + utxos::Utxo, + Streamable, +}; +use futures_util::{ + stream::{SplitSink, SplitStream}, + SinkExt, + StreamExt, +}; +use reqwest::{ + header::{ + ACCEPT, + AUTHORIZATION, + CONNECTION, + CONTENT_TYPE, + HOST, + SEC_WEBSOCKET_KEY, + SEC_WEBSOCKET_VERSION, + UPGRADE, + }, + Client as HttpClient, +}; +use tokio::sync::{mpsc, RwLock}; +use tokio_tungstenite::{ + connect_async, + tungstenite::{ + client::IntoClientRequest, + handshake::client::generate_key, + protocol::Message, + }, + MaybeTlsStream, +}; +use url::Url; + +use crate::server::{ + http::models::{LoginRequest, LoginResponse}, + ws::{ + errors::WsSubscriptionError, + models::{ + ClientMessage, + DeliverPolicy, + ServerMessage, + SubscriptionPayload, + }, + socket::verify_and_extract_subject_name, + }, +}; + +#[derive(Debug)] +pub struct WebSocketClient { + read_stream: Option< + SplitStream< + tokio_tungstenite::WebSocketStream< + MaybeTlsStream, + >, + >, + >, + write_sink: Option< + RwLock< + SplitSink< + tokio_tungstenite::WebSocketStream< + MaybeTlsStream, + >, + Message, + >, + >, + >, + jwt_token: Option, + ws_url: Url, + network: FuelNetwork, + username: String, + password: String, +} + +impl WebSocketClient { + pub async fn new( + network: FuelNetwork, + username: &str, + password: &str, + ) -> anyhow::Result { + let jwt_token = Self::fetch_jwt(network, username, password).await?; + + let ws_url = network.to_ws_url().join("/api/v1/ws")?; + + Ok(Self { + read_stream: None, + write_sink: None, + jwt_token: Some(jwt_token), + ws_url, + network, + username: username.to_string(), + password: password.to_string(), + }) + } + + async fn fetch_jwt( + network: FuelNetwork, + username: &str, + password: &str, + ) -> anyhow::Result { + let client = HttpClient::new(); + let json_body = serde_json::to_string(&LoginRequest { + username: username.to_string(), + password: password.to_string(), + })?; + + let api_url = network.to_web_url().join("/api/v1/jwt")?; + + let response = client + .get(api_url) + .header(ACCEPT, "application/json") + .header(CONTENT_TYPE, "application/json") + .body(json_body) + .send() + .await?; + + if response.status().is_success() { + let json_body = response.json::().await?; + Ok(json_body.jwt_token) + } else { + Err(anyhow::anyhow!( + "Failed to fetch JWT: {}", + response.status() + )) + } + } + + pub async fn refresh_jwt(&mut self) -> anyhow::Result<()> { + let jwt_token = + Self::fetch_jwt(self.network, &self.username, &self.password) + .await?; + self.jwt_token = Some(jwt_token); + Ok(()) + } + + pub async fn connect(&mut self) -> anyhow::Result<()> { + let host = self + .ws_url + .host_str() + .ok_or(anyhow::anyhow!("Unparsable ws host url"))?; + + let jwt_token = self + .jwt_token + .clone() + .ok_or(anyhow::anyhow!("JWT token is missing"))?; + + let mut request = self.ws_url.as_str().into_client_request()?; + let headers_map = request.headers_mut(); + headers_map + .insert(AUTHORIZATION, format!("Bearer {}", jwt_token).parse()?); + headers_map.insert(HOST, host.parse()?); + headers_map.insert(UPGRADE, "websocket".parse()?); + headers_map.insert(CONNECTION, "Upgrade".parse().unwrap()); + headers_map.insert(SEC_WEBSOCKET_KEY, generate_key().parse()?); + headers_map.insert(SEC_WEBSOCKET_VERSION, "13".parse()?); + + let (socket, _response) = connect_async(request).await?; + let (write, read) = socket.split(); + + self.read_stream = Some(read); + self.write_sink = Some(RwLock::new(write)); + + Ok(()) + } + + async fn send_client_message( + &mut self, + message: ClientMessage, + ) -> anyhow::Result<()> { + let write_sink = self + .write_sink + .as_ref() + .ok_or_else(|| anyhow::anyhow!("Socket not connected"))?; + let mut write_guard = write_sink.write().await; + let serialized = serde_json::to_vec(&message)?; + write_guard.send(Message::Binary(serialized)).await?; + Ok(()) + } + + pub async fn subscribe( + &mut self, + wildcard: impl IntoSubject, + deliver_policy: DeliverPolicy, + ) -> anyhow::Result<()> { + let message = ClientMessage::Subscribe(SubscriptionPayload { + wildcard: wildcard.parse(), + deliver_policy, + }); + self.send_client_message(message).await?; + Ok(()) + } + + pub async fn unsubscribe( + &mut self, + wildcard: impl IntoSubject, + deliver_policy: DeliverPolicy, + ) -> anyhow::Result<()> { + let message = ClientMessage::Unsubscribe(SubscriptionPayload { + wildcard: wildcard.parse(), + deliver_policy, + }); + self.send_client_message(message).await?; + Ok(()) + } + + pub async fn listen( + &mut self, + ) -> anyhow::Result> { + let read_stream = self + .read_stream + .take() + .ok_or_else(|| anyhow::anyhow!("Socket not connected"))?; + let (tx, rx) = mpsc::unbounded_channel::(); + // TODO: the reason for using this type of channel is due to the fact that Streamable cannot be currently + // converted into a dynamic object trait, hence this approach of switching between types + tokio::spawn(async move { + let mut subscription_topic = String::new(); + let mut read_stream = read_stream; + while let Some(Ok(msg)) = read_stream.next().await { + match msg { + Message::Text(text) => { + println!("Received text: {:?} bytes", text.len()); + } + Message::Binary(bin) => { + let server_message = match serde_json::from_slice::< + ServerMessage, + >( + &bin + ) { + Ok(server_message) => server_message, + Err(e) => { + eprintln!("Unparsable server message: {:?}", e); + continue; + } + }; + + match &server_message { + ServerMessage::Subscribed(sub) => { + println!( + "Subscribed to wildcard: {:?}", + sub.wildcard + ); + subscription_topic = sub.wildcard.clone(); + } + ServerMessage::Unsubscribed(sub) => { + println!( + "Unsubscribed from wildcard: {:?}", + sub.wildcard + ); + } + ServerMessage::Update(update) => { + let _ = decode_print( + &subscription_topic, + update.clone(), + ) + .ok(); + // send server message over a channel to receivers + if tx.send(server_message).is_err() { + break; + } + } + ServerMessage::Error(err) => { + println!("Received error from ws: {:?}", err); + break; + } + } + } + Message::Ping(ping) => { + println!("Received ping: {:?} bytes", ping.len()); + } + Message::Pong(pong) => { + println!("Received pong: {:?} bytes", pong.len()); + } + Message::Close(close) => { + let close_code = close + .as_ref() + .map(|c| c.code.to_string()) + .unwrap_or_default(); + let close_reason = close + .as_ref() + .map(|c| c.reason.to_string()) + .unwrap_or_default(); + println!( + "Received close with code: {:?} and reason: {:?}", + close_code, close_reason + ); + break; + } + _ => { + eprintln!("Received unknown message type"); + break; + } + } + } + }); + + Ok(rx) + } +} + +pub fn decode_print( + subject_wildcard: &str, + s3_payload: Vec, +) -> Result<(), WsSubscriptionError> { + let subject = verify_and_extract_subject_name(subject_wildcard)?; + match subject.as_str() { + Transaction::NAME => { + let entity = serde_json::from_slice::(&s3_payload) + .map_err(WsSubscriptionError::UnparsablePayload)?; + println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); + } + Block::NAME => { + let entity = serde_json::from_slice::(&s3_payload) + .map_err(WsSubscriptionError::UnparsablePayload)?; + println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); + } + Input::NAME => { + let entity = serde_json::from_slice::(&s3_payload) + .map_err(WsSubscriptionError::UnparsablePayload)?; + println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); + } + Output::NAME => { + let entity = serde_json::from_slice::(&s3_payload) + .map_err(WsSubscriptionError::UnparsablePayload)?; + println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); + } + Receipt::NAME => { + let entity = serde_json::from_slice::(&s3_payload) + .map_err(WsSubscriptionError::UnparsablePayload)?; + println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); + } + Utxo::NAME => { + let entity = serde_json::from_slice::(&s3_payload) + .map_err(WsSubscriptionError::UnparsablePayload)?; + println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); + } + Log::NAME => { + let entity = serde_json::from_slice::(&s3_payload) + .map_err(WsSubscriptionError::UnparsablePayload)?; + println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); + } + _ => { + eprintln!("Unknown entity {:?}", subject.to_string()); + } + } + Ok(()) +} diff --git a/crates/fuel-streams-ws/src/config.rs b/crates/fuel-streams-ws/src/config.rs new file mode 100644 index 00000000..e90e294c --- /dev/null +++ b/crates/fuel-streams-ws/src/config.rs @@ -0,0 +1,193 @@ +use std::{ + num::ParseIntError, + path::{Path, PathBuf}, + str::{FromStr, ParseBoolError}, + time::Duration, +}; + +use confy::ConfyError; +use displaydoc::Display as DisplayDoc; +use fuel_streams::types::FuelNetwork; +use serde::{Deserialize, Deserializer}; +use thiserror::Error; +use tokio::{fs::File, io::AsyncReadExt}; + +#[derive(Debug, DisplayDoc, Error)] +pub enum Error { + /// Open config file: {0} + OpenConfig(std::io::Error), + /// Failed to parse config: {0} + ParseConfig(toml::de::Error), + /// Failed to parse config as utf-8: {0} + ParseUtf8(std::string::FromUtf8Error), + /// Failed to read config file: {0} + ReadConfig(std::io::Error), + /// Failed to read config metadata: {0} + ReadMeta(std::io::Error), + /// Failed to read env config: {0} + Confy(ConfyError), + /// Undecodable config element: {0} + UndecodableConfigElement(&'static str), + /// Parse int error: {0} + ParseInt(ParseIntError), + /// Parse bool error: {0} + ParseBool(ParseBoolError), +} + +#[derive(Debug, Default, Deserialize, Clone)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct S3Config { + pub enabled: bool, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct TlsConfig { + pub private_key: PathBuf, + pub certificate: PathBuf, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct ApiConfig { + pub port: u16, + pub tls: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct AuthConfig { + pub jwt_secret: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct FuelConfig { + pub network: FuelNetwork, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct NatsConfig { + pub network: FuelNetwork, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Config { + pub api: ApiConfig, + pub auth: AuthConfig, + pub s3: S3Config, + pub nats: NatsConfig, + pub fuel: FuelConfig, +} + +impl Default for Config { + fn default() -> Self { + Config { + api: ApiConfig { + port: 9003, + tls: None, + }, + auth: AuthConfig { + jwt_secret: String::new(), + }, + nats: NatsConfig { + network: FuelNetwork::Local, + }, + s3: S3Config { enabled: false }, + fuel: FuelConfig { + network: FuelNetwork::Local, + }, + } + } +} + +#[allow(dead_code)] +fn deserialize_duration_from_usize<'de, D>( + deserializer: D, +) -> Result +where + D: Deserializer<'de>, +{ + let seconds = u64::deserialize(deserializer)?; + Ok(Duration::from_secs(seconds)) +} + +#[allow(dead_code)] +fn deserialize_duration_option<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let seconds: Option = Option::deserialize(deserializer)?; + if seconds.is_none() { + return Ok(None); + } + Ok(seconds.map(Duration::from_secs)) +} + +impl Config { + pub async fn from_path( + path: impl AsRef + Send, + ) -> Result { + read_to_string(path).await?.parse() + } + + pub fn from_envs() -> Result { + let mut config = Self::default(); + + // ----------------------API-------------------------------- + if let Ok(app_port) = dotenvy::var("STREAMER_API_PORT") { + config.api.port = + app_port.parse::().map_err(Error::ParseInt)?; + } + + // ----------------------NATS-------------------------------- + if let Ok(nats_network) = dotenvy::var("NETWORK") { + config.nats.network = FuelNetwork::from_str(&nats_network) + .map_err(|_| Error::UndecodableConfigElement("NETWORK"))?; + } + + // ----------------------S3-------------------------------- + if let Ok(s3_enabled) = dotenvy::var("AWS_S3_ENABLED") { + config.s3.enabled = + s3_enabled.parse::().map_err(Error::ParseBool)?; + } + + // ----------------------AUTH-------------------------------- + if let Ok(jwt_secret) = dotenvy::var("JWT_AUTH_SECRET") { + config.auth.jwt_secret = jwt_secret; + } + + // ----------------------FUEL-------------------------------- + if let Ok(network) = dotenvy::var("NETWORK") { + config.fuel.network = FuelNetwork::from_str(&network) + .map_err(|_| Error::UndecodableConfigElement("NETWORK"))?; + } + + Ok(config) + } +} + +impl FromStr for Config { + type Err = Error; + + fn from_str(s: &str) -> Result { + toml::from_str(s).map_err(Error::ParseConfig) + } +} + +async fn read_to_string( + path: impl AsRef + Send, +) -> Result { + let mut file = File::open(path).await.map_err(Error::OpenConfig)?; + let meta = file.metadata().await.map_err(Error::ReadMeta)?; + let mut contents = + Vec::with_capacity(usize::try_from(meta.len()).unwrap_or(0)); + file.read_to_end(&mut contents) + .await + .map_err(Error::ReadConfig)?; + String::from_utf8(contents).map_err(Error::ParseUtf8) +} diff --git a/crates/fuel-streams-ws/src/lib.rs b/crates/fuel-streams-ws/src/lib.rs new file mode 100644 index 00000000..d60b1e6c --- /dev/null +++ b/crates/fuel-streams-ws/src/lib.rs @@ -0,0 +1,17 @@ +pub mod cli; +pub mod client; +pub mod config; +pub mod server; +pub mod telemetry; + +use std::{env, sync::LazyLock}; + +pub static STREAMER_MAX_WORKERS: LazyLock = LazyLock::new(|| { + let available_cpus = num_cpus::get(); + let default_threads = 2 * available_cpus; + + env::var("STREAMER_MAX_WORKERS") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(default_threads) +}); diff --git a/crates/fuel-streams-ws/src/main.rs b/crates/fuel-streams-ws/src/main.rs new file mode 100644 index 00000000..f8d6fe90 --- /dev/null +++ b/crates/fuel-streams-ws/src/main.rs @@ -0,0 +1,71 @@ +use anyhow::Context as _; +use clap::Parser; +use fuel_streams_ws::{ + cli::Cli, + config::Config, + server::{api::create_api, context::Context, state::ServerState}, +}; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // init tracing + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .with_span_events(FmtSpan::CLOSE) + .init(); + + // load envs + dotenvy::dotenv().context("Failed to env values")?; + + // read cli args + let cli = Cli::parse(); + + // load config + let config = match cli.config_path { + Some(path) => { + tracing::info!("Using config file: {}", path); + Config::from_path(path) + .await + .context("Failed to load toml config")? + } + None => { + tracing::info!("Using envs to load config"); + Config::from_envs().context("Failed to load toml config")? + } + }; + + // init context + let context = Context::new(&config).await?; + + // init server shared state + let state = ServerState::new(context).await; + + // create the actix webserver + let server = create_api(&config, state)?; + + // get server handle + let server_handle = server.handle(); + + // spawn the server in the background + let jh = tokio::spawn(async move { + tracing::info!("Starting actix server ..."); + if let Err(err) = server.await { + tracing::error!("Actix Web server error: {:?}", err); + } + }); + + let _ = tokio::join!(jh); + + // Await the Actix server shutdown + tracing::info!("Stopping actix server ..."); + server_handle.stop(true).await; + tracing::info!("Actix server stopped. Goodbye!"); + + Ok(()) +} diff --git a/crates/fuel-streams-ws/src/server/api.rs b/crates/fuel-streams-ws/src/server/api.rs new file mode 100644 index 00000000..4e06e39d --- /dev/null +++ b/crates/fuel-streams-ws/src/server/api.rs @@ -0,0 +1,88 @@ +use std::net::{Ipv4Addr, SocketAddrV4}; + +use actix_cors::Cors; +use actix_server::Server; +use actix_web::{ + http::{self, Method}, + middleware::{Compress, Logger as ActixLogger}, + web, + App, + HttpServer, +}; +use tracing_actix_web::TracingLogger; + +use super::{ + http::handlers::{get_health, get_metrics, request_jwt}, + middlewares::auth::JwtAuth, + state::ServerState, + ws::socket::get_ws, +}; +use crate::{config::Config, STREAMER_MAX_WORKERS}; + +const API_VERSION: &str = "v1"; + +fn with_prefixed_route(route: &str) -> String { + format!("/api/{}/{}", API_VERSION, route) +} + +pub fn create_api( + config: &Config, + state: ServerState, +) -> anyhow::Result { + let server_addr = std::net::SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::UNSPECIFIED, + config.api.port, + )); + let jwt_secret = config.auth.jwt_secret.clone(); + let server = HttpServer::new(move || { + let jwt_secret = jwt_secret.clone(); + // create cors + let cors = Cors::default() + .allow_any_origin() + .allowed_methods(vec![ + Method::GET, + Method::POST, + Method::PUT, + Method::OPTIONS, + Method::DELETE, + Method::PATCH, + Method::TRACE, + ]) + .allowed_headers(vec![ + http::header::AUTHORIZATION, + http::header::ACCEPT, + ]) + .allowed_header(http::header::CONTENT_TYPE) + .max_age(3600); + + App::new() + .app_data(web::Data::new(state.clone())) + .wrap(ActixLogger::default()) + .wrap(TracingLogger::default()) + .wrap(Compress::default()) + .wrap(cors) + .service( + web::resource(with_prefixed_route("health")) + .route(web::get().to(get_health)), + ) + .service( + web::resource(with_prefixed_route("metrics")) + .route(web::get().to(get_metrics)), + ) + .service( + web::resource(with_prefixed_route("jwt")) + .route(web::get().to(request_jwt)), + ) + .service( + web::resource(with_prefixed_route("ws")) + .wrap(JwtAuth::new(jwt_secret)) + .route(web::get().to(get_ws)), + ) + }) + .bind(server_addr)? + .workers(*STREAMER_MAX_WORKERS) + .shutdown_timeout(20) + .run(); + + Ok(server) +} diff --git a/crates/fuel-streams-ws/src/server/auth.rs b/crates/fuel-streams-ws/src/server/auth.rs new file mode 100755 index 00000000..fc11fe2c --- /dev/null +++ b/crates/fuel-streams-ws/src/server/auth.rs @@ -0,0 +1,357 @@ +use std::{collections::HashMap, convert::TryFrom, fmt}; + +use actix_web::{ + http::header::{HeaderMap, HeaderValue, AUTHORIZATION}, + HttpResponse, + ResponseError, +}; +use chrono::Utc; +use displaydoc::Display as DisplayDoc; +use jsonwebtoken::{ + decode, + encode, + Algorithm, + DecodingKey, + EncodingKey, + Header, + Validation, +}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +const BEARER: &str = "Bearer"; + +#[derive( + Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, +)] +pub enum UserType { + ADMIN, + CLIENT, +} + +/// User-related errors +#[derive(Clone, Debug, DisplayDoc, Error, PartialEq)] +pub enum UserError { + /// User not found + UserNotFound, + /// Unknown User Role: `{0}` + UnknownUserRole(String), + /// Unknown User Status: `{0}` + UnknownUserStatus(String), + /// Unallowed User Role: `{0}` + UnallowedUserRole(String), + /// Missing password + MissingPassword, + /// Missing username + MissingUsername, + /// Wrong password + WrongPassword, + /// User is not verified + UnverifiedUser, +} + +impl ResponseError for UserError { + fn error_response(&self) -> HttpResponse { + match self { + UserError::UserNotFound => { + HttpResponse::NotFound().body(self.to_string()) + } + UserError::UnknownUserRole(_) => { + HttpResponse::Unauthorized().body(self.to_string()) + } + UserError::UnknownUserStatus(_) => { + HttpResponse::NotFound().body(self.to_string()) + } + UserError::UnallowedUserRole(_) => { + HttpResponse::Unauthorized().body(self.to_string()) + } + UserError::MissingPassword => { + HttpResponse::Unauthorized().body(self.to_string()) + } + UserError::MissingUsername => { + HttpResponse::Unauthorized().body(self.to_string()) + } + UserError::WrongPassword => { + HttpResponse::Unauthorized().body(self.to_string()) + } + UserError::UnverifiedUser => { + HttpResponse::Unauthorized().body(self.to_string()) + } + } + } +} + +/// Auth errors +#[derive(Clone, Debug, DisplayDoc, Error, PartialEq)] +pub enum AuthError { + /// Wrong Credentials + WrongCredentialsError, + /// JWT Token not valid + JWTTokenError, + /// JWT Token Creation Error + JWTTokenCreationError, + /// No Auth Header + NoAuthHeaderError, + /// Invalid Auth Header + InvalidAuthHeaderError, + /// No Permission + NoPermissionError, + /// Expired Token + ExpiredToken, + /// Bad Encoded User Role: `{0}` + BadEncodedUserRole(String), + /// Unparsable UUID error: `{0}` + UnparsableUuid(String), +} + +impl ResponseError for AuthError { + fn error_response(&self) -> HttpResponse { + match self { + AuthError::WrongCredentialsError => { + HttpResponse::Unauthorized().body(self.to_string()) + } + AuthError::JWTTokenError => { + HttpResponse::Unauthorized().body(self.to_string()) + } + AuthError::JWTTokenCreationError => { + HttpResponse::InternalServerError().body(self.to_string()) + } + AuthError::NoAuthHeaderError => { + HttpResponse::Unauthorized().body(self.to_string()) + } + AuthError::InvalidAuthHeaderError => { + HttpResponse::Unauthorized().body(self.to_string()) + } + AuthError::NoPermissionError => { + HttpResponse::Unauthorized().body(self.to_string()) + } + AuthError::ExpiredToken => { + HttpResponse::Unauthorized().body(self.to_string()) + } + AuthError::BadEncodedUserRole(_) => { + HttpResponse::Unauthorized().body(self.to_string()) + } + AuthError::UnparsableUuid(_) => { + HttpResponse::Unauthorized().body(self.to_string()) + } + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct Claims { + sub: String, + role: String, + exp: usize, +} + +/// A user role +#[repr(i16)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum UserRole { + Admin = 0, + Client = 1, +} + +impl From for i16 { + fn from(role: UserRole) -> i16 { + role as i16 + } +} + +impl TryFrom for UserRole { + type Error = UserError; + + fn try_from(n: i16) -> Result { + match n { + 0 => Ok(UserRole::Admin), + 1 => Ok(UserRole::Client), + _ => Err(UserError::UnknownUserRole(n.to_string())), + } + } +} + +/// Maps a string to a Role +impl TryFrom<&str> for UserRole { + type Error = UserError; + + fn try_from(role: &str) -> Result { + match role.to_lowercase().as_str() { + "admin" => Ok(UserRole::Admin), + "client" => Ok(UserRole::Client), + _ => Err(UserError::UnknownUserRole(role.to_string())), + } + } +} + +impl fmt::Display for UserRole { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + UserRole::Admin => write!(f, "admin"), + UserRole::Client => write!(f, "client"), + } + } +} + +impl From for UserRole { + fn from(value: UserType) -> Self { + match value { + UserType::ADMIN => UserRole::Admin, + UserType::CLIENT => UserRole::Client, + } + } +} + +impl From for UserType { + fn from(value: UserRole) -> Self { + match value { + UserRole::Admin => UserType::ADMIN, + UserRole::Client => UserType::CLIENT, + } + } +} + +/// A user status +#[repr(i16)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum UserStatus { + Unverified = 0, + Verified = 1, +} + +impl From for i16 { + fn from(user_status: UserStatus) -> i16 { + user_status as i16 + } +} + +impl TryFrom for UserStatus { + type Error = UserError; + + fn try_from(n: i16) -> Result { + match n { + 0 => Ok(UserStatus::Unverified), + 1 => Ok(UserStatus::Verified), + _ => Err(UserError::UnknownUserStatus(n.to_string())), + } + } +} + +/// Maps a string to a status +impl TryFrom<&str> for UserStatus { + type Error = UserError; + + fn try_from(user_status: &str) -> Result { + match user_status.to_lowercase().as_str() { + "unverified" => Ok(UserStatus::Unverified), + "verified" => Ok(UserStatus::Verified), + _ => Err(UserError::UnknownUserStatus(user_status.to_string())), + } + } +} + +impl fmt::Display for UserStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + UserStatus::Unverified => write!(f, "unverified"), + UserStatus::Verified => write!(f, "verified"), + } + } +} + +pub fn create_jwt( + uid: &str, + role: &UserRole, + jwt_secret: &[u8], +) -> Result { + let expiration = Utc::now() + .checked_add_signed(chrono::Duration::minutes(60)) + .expect("valid timestamp") + .timestamp(); + + let claims = Claims { + sub: uid.to_owned(), + role: role.to_string(), + exp: expiration as usize, + }; + let header = Header::new(Algorithm::HS512); + encode(&header, &claims, &EncodingKey::from_secret(jwt_secret)) + .map_err(|_| AuthError::JWTTokenCreationError) +} + +fn jwt_from_header(headers: &HeaderMap) -> Result { + let header = match headers.get(AUTHORIZATION) { + Some(v) => v, + None => return Err(AuthError::NoAuthHeaderError), + }; + let auth_header = match std::str::from_utf8(header.as_bytes()) { + Ok(v) => v, + Err(_) => return Err(AuthError::NoAuthHeaderError), + }; + if !auth_header.starts_with(BEARER) { + return Err(AuthError::InvalidAuthHeaderError); + } + let decoded_jwt = + urlencoding::decode(auth_header.trim_start_matches(BEARER)) + .unwrap() + .trim() + .to_string(); + Ok(decoded_jwt) +} + +fn authorize( + (jwt_secret, headers): (String, actix_web::http::header::HeaderMap), +) -> Result<(Uuid, String), AuthError> { + match jwt_from_header(&headers) { + Ok(jwt) => { + let decoded = decode::( + &jwt, + &DecodingKey::from_secret(jwt_secret.as_bytes()), + &Validation::new(Algorithm::HS512), + ) + .map_err(|_| AuthError::JWTTokenError)?; + + // check user id + let user_id = + Uuid::parse_str(&decoded.claims.sub).map_err(|_| { + AuthError::UnparsableUuid(decoded.claims.sub.to_string()) + })?; + // check token expiration + let now = Utc::now().timestamp(); + + if (decoded.claims.exp as i64).lt(&now) { + return Err(AuthError::ExpiredToken); + } + + // TODO: check for user in the db by user_id + + // get the user's role + let _token_role = UserRole::try_from(decoded.claims.role.as_str()) + .map_err(|_| { + AuthError::BadEncodedUserRole(decoded.claims.role) + })?; + + // TODO: verify db user's role vs token_role + Ok((user_id, jwt)) + } + Err(e) => Err(e), + } +} + +pub fn authorize_request( + (jwt_secret, mut headers, query_map): ( + String, + actix_web::http::header::HeaderMap, + HashMap, + ), +) -> Result<(Uuid, String), AuthError> { + // move all query values to the headers + for (key, value) in query_map.iter() { + if AUTHORIZATION.as_str().eq_ignore_ascii_case(key) { + headers + .insert(AUTHORIZATION, HeaderValue::from_str(value).unwrap()); + } + } + authorize((jwt_secret, headers)) +} diff --git a/crates/fuel-streams-ws/src/server/context.rs b/crates/fuel-streams-ws/src/server/context.rs new file mode 100644 index 00000000..59d0166b --- /dev/null +++ b/crates/fuel-streams-ws/src/server/context.rs @@ -0,0 +1,101 @@ +use std::{sync::Arc, time::Duration}; + +use fuel_streams::client::Client; +use fuel_streams_core::prelude::*; +use fuel_streams_storage::S3Client; + +use crate::{ + config::Config, + server::ws::fuel_streams::FuelStreams, + telemetry::Telemetry, +}; + +pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(90); + +#[allow(dead_code)] +#[derive(Clone)] +pub struct Context { + pub client: Client, + pub nats_client: NatsClient, + pub fuel_streams: Arc, + pub telemetry: Arc, + pub s3_client: Option>, + pub jwt_secret: String, +} + +impl Context { + pub async fn new(config: &Config) -> anyhow::Result { + let nats_client_opts = NatsClientOpts::new(config.fuel.network); + let nats_client = NatsClient::connect(&nats_client_opts).await?; + + let s3_client_opts = S3ClientOpts::admin_opts(); + let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); + + let fuel_streams = + Arc::new(FuelStreams::new(&nats_client, &s3_client).await); + + let client = Client::connect(config.fuel.network).await?; + let telemetry = Telemetry::new(None).await?; + telemetry.start().await?; + + Ok(Context { + fuel_streams, + nats_client, + client, + telemetry, + s3_client: if config.s3.enabled { + Some(s3_client) + } else { + None + }, + jwt_secret: config.auth.jwt_secret.clone(), + }) + } + + pub async fn new_for_testing( + fuel_network: FuelNetwork, + ) -> anyhow::Result { + let nats_client_opts = NatsClientOpts::new(fuel_network); + let nats_client = NatsClient::connect(&nats_client_opts).await?; + let client = Client::connect(fuel_network).await?; + let s3_client_opts = S3ClientOpts::admin_opts(); + let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); + Ok(Context { + fuel_streams: Arc::new( + FuelStreams::new(&nats_client, &s3_client).await, + ), + nats_client: nats_client.clone(), + telemetry: Telemetry::new(None).await?, + s3_client: None, + jwt_secret: String::new(), + client, + }) + } + + pub fn get_streams(&self) -> &FuelStreams { + &self.fuel_streams + } + + #[allow(dead_code)] + async fn shutdown_services_with_timeout(&self) -> anyhow::Result<()> { + tokio::time::timeout(GRACEFUL_SHUTDOWN_TIMEOUT, async { + Context::flush_await_all_streams(&self.nats_client).await; + }) + .await?; + + Ok(()) + } + + #[allow(dead_code)] + async fn flush_await_all_streams(nats_client: &NatsClient) { + tracing::info!("Flushing in-flight messages to nats ..."); + match nats_client.nats_client.flush().await { + Ok(_) => { + tracing::info!("Flushed all streams successfully!"); + } + Err(e) => { + tracing::error!("Failed to flush all streams: {:?}", e); + } + } + } +} diff --git a/crates/fuel-streams-ws/src/server/http/handlers.rs b/crates/fuel-streams-ws/src/server/http/handlers.rs new file mode 100644 index 00000000..45fd8aa8 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/http/handlers.rs @@ -0,0 +1,73 @@ +use std::{collections::HashMap, sync::LazyLock}; + +use actix_web::{web, HttpResponse, Result}; +use uuid::Uuid; + +use super::models::{LoginRequest, LoginResponse}; +use crate::server::{ + auth::{create_jwt, AuthError, UserError, UserRole}, + state::ServerState, +}; + +pub static AUTH_DATA: LazyLock> = + LazyLock::new(|| { + HashMap::from_iter(vec![ + ( + "client".to_string(), + ("client".to_string(), UserRole::Client, Uuid::new_v4()), + ), + ( + "admin".to_string(), + ("admin".to_string(), UserRole::Admin, Uuid::new_v4()), + ), + ]) + }); + +pub async fn get_metrics( + state: web::Data, +) -> Result { + Ok(HttpResponse::Ok() + .content_type( + "application/openmetrics-text; version=1.0.0; charset=utf-8", + ) + .body(state.context.telemetry.get_metrics().await)) +} + +pub async fn get_health(state: web::Data) -> Result { + if !state.is_healthy() { + return Ok( + HttpResponse::ServiceUnavailable().body("Service Unavailable") + ); + } + Ok(HttpResponse::Ok().json(state.get_health().await)) +} + +// request jwt +pub async fn request_jwt( + state: web::Data, + req_body: web::Json, +) -> actix_web::Result { + // get user by username + let (password, user_role, uuid) = (*AUTH_DATA) + .get(&req_body.username) + .ok_or(UserError::UserNotFound)?; + + // compare pwd with expected one + if !password.eq_ignore_ascii_case(&req_body.password) { + return Err(AuthError::WrongCredentialsError.into()); + } + + // if all good, generate a jwt with the user role encoded + let jwt_token = create_jwt( + &uuid.to_string(), + user_role, + state.context.jwt_secret.as_bytes(), + ) + .map_err(|_| AuthError::JWTTokenCreationError)?; + + Ok(HttpResponse::Ok().json(&LoginResponse { + id: uuid.to_owned(), + username: req_body.username.clone(), + jwt_token, + })) +} diff --git a/crates/fuel-streams-ws/src/server/http/mod.rs b/crates/fuel-streams-ws/src/server/http/mod.rs new file mode 100644 index 00000000..759a498a --- /dev/null +++ b/crates/fuel-streams-ws/src/server/http/mod.rs @@ -0,0 +1,2 @@ +pub mod handlers; +pub mod models; diff --git a/crates/fuel-streams-ws/src/server/http/models.rs b/crates/fuel-streams-ws/src/server/http/models.rs new file mode 100644 index 00000000..11d8ebb8 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/http/models.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; +use validator::Validate; + +#[derive(Debug, Serialize, Deserialize, Validate)] +#[serde(rename_all = "camelCase")] +pub struct LoginRequest { + pub username: String, + pub password: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LoginResponse { + pub id: uuid::Uuid, + pub username: String, + pub jwt_token: String, +} diff --git a/crates/fuel-streams-ws/src/server/middlewares/auth.rs b/crates/fuel-streams-ws/src/server/middlewares/auth.rs new file mode 100644 index 00000000..e57de0cd --- /dev/null +++ b/crates/fuel-streams-ws/src/server/middlewares/auth.rs @@ -0,0 +1,109 @@ +use std::{ + collections::HashMap, + task::{Context, Poll}, +}; + +use actix_service::Transform; +use actix_web::{ + dev::{ServiceRequest, ServiceResponse}, + Error, + HttpMessage, + Result, +}; +use futures_util::future::{ready, LocalBoxFuture, Ready}; + +use crate::server::auth::authorize_request; + +pub struct JwtAuth { + jwt_secret: String, +} + +impl JwtAuth { + pub fn new(jwt_secret: String) -> Self { + JwtAuth { jwt_secret } + } +} + +impl Transform for JwtAuth +where + S: actix_service::Service< + ServiceRequest, + Response = ServiceResponse, + Error = Error, + > + 'static, + S::Future: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type InitError = (); + type Transform = JwtAuthMiddleware; + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + ready(Ok(JwtAuthMiddleware { + service, + jwt_secret: self.jwt_secret.clone(), + })) + } +} + +pub struct JwtAuthMiddleware { + service: S, + jwt_secret: String, +} + +impl actix_service::Service for JwtAuthMiddleware +where + S: actix_service::Service< + ServiceRequest, + Response = ServiceResponse, + Error = Error, + > + 'static, + S::Future: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type Future = LocalBoxFuture<'static, Result>; + + fn poll_ready( + &self, + cx: &mut Context<'_>, + ) -> Poll> { + self.service.poll_ready(cx) + } + + fn call(&self, req: ServiceRequest) -> Self::Future { + let jwt_secret = self.jwt_secret.clone(); + let headers = req.headers().clone(); + let query_map: HashMap = req + .query_string() + .split('&') + .filter_map(|pair| { + let mut parts = pair.split('='); + if let (Some(key), Some(value)) = (parts.next(), parts.next()) { + Some(( + key.to_string(), + urlencoding::decode(value).unwrap().into_owned(), + )) + } else { + None + } + }) + .collect(); + + // Validate the JWT + match authorize_request((jwt_secret, headers, query_map)) { + Ok((user_id, _jwt)) => { + req.extensions_mut().insert(user_id); + Box::pin(self.service.call(req)) + } + Err(e) => { + let err = e.to_string(); + // If JWT is invalid or missing, reject the request + Box::pin(async { + Err(actix_web::error::ErrorUnauthorized(err)) + }) + } + } + } +} diff --git a/crates/fuel-streams-ws/src/server/middlewares/mod.rs b/crates/fuel-streams-ws/src/server/middlewares/mod.rs new file mode 100644 index 00000000..0e4a05d5 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/middlewares/mod.rs @@ -0,0 +1 @@ +pub mod auth; diff --git a/crates/fuel-streams-ws/src/server/mod.rs b/crates/fuel-streams-ws/src/server/mod.rs new file mode 100644 index 00000000..1ecd4c65 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/mod.rs @@ -0,0 +1,7 @@ +pub mod api; +pub mod auth; +pub mod context; +pub mod http; +pub mod middlewares; +pub mod state; +pub mod ws; diff --git a/crates/fuel-streams-ws/src/server/state.rs b/crates/fuel-streams-ws/src/server/state.rs new file mode 100644 index 00000000..8a2a3609 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/state.rs @@ -0,0 +1,110 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use async_nats::jetstream::stream::State; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; + +use super::context::Context; +use crate::server::ws::fuel_streams::FuelStreamsExt; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct StreamInfo { + consumers: Vec, + state: StreamState, + stream_name: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +pub struct StreamState { + /// The number of messages contained in this stream + pub messages: u64, + /// The number of bytes of all messages contained in this stream + pub bytes: u64, + /// The lowest sequence number still present in this stream + #[serde(rename = "first_seq")] + pub first_sequence: u64, + /// The time associated with the oldest message still present in this stream + #[serde(rename = "first_ts")] + pub first_timestamp: i64, + /// The last sequence number assigned to a message in this stream + #[serde(rename = "last_seq")] + pub last_sequence: u64, + /// The time that the last message was received by this stream + #[serde(rename = "last_ts")] + pub last_timestamp: i64, + /// The number of consumers configured to consume this stream + pub consumer_count: usize, +} + +impl From for StreamState { + fn from(state: State) -> Self { + StreamState { + messages: state.messages, + bytes: state.bytes, + first_sequence: state.first_sequence, + first_timestamp: state.first_timestamp.unix_timestamp(), + last_sequence: state.last_sequence, + last_timestamp: state.last_timestamp.unix_timestamp(), + consumer_count: state.consumer_count, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct HealthResponse { + pub uptime: u64, + pub streams_info: Vec, +} + +#[derive(Clone)] +pub struct ServerState { + pub context: Context, + pub start_time: Instant, + pub connection_count: Arc>, +} + +impl ServerState { + pub async fn new(context: Context) -> Self { + Self { + start_time: Instant::now(), + connection_count: Arc::new(RwLock::new(0)), + context, + } + } +} + +impl ServerState { + pub fn is_healthy(&self) -> bool { + if !self.context.nats_client.is_connected() { + return false; + } + true + } + + pub async fn get_health(&self) -> HealthResponse { + let streams_info = self + .context + .fuel_streams + .get_consumers_and_state() + .await + .unwrap_or_default() + .into_iter() + .map(|res| StreamInfo { + consumers: res.1, + state: res.2.into(), + stream_name: res.0, + }) + .collect::>(); + HealthResponse { + uptime: self.uptime().as_secs(), + streams_info, + } + } + + pub fn uptime(&self) -> Duration { + self.start_time.elapsed() + } +} diff --git a/crates/fuel-streams-ws/src/server/ws/errors.rs b/crates/fuel-streams-ws/src/server/ws/errors.rs new file mode 100644 index 00000000..db76ca56 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/ws/errors.rs @@ -0,0 +1,20 @@ +use displaydoc::Display as DisplayDoc; +use fuel_streams_core::StreamError; +use thiserror::Error; + +/// Ws Subscription-related errors +#[derive(Debug, DisplayDoc, Error)] +pub enum WsSubscriptionError { + /// Unparsable subscription payload: `{0}` + UnparsablePayload(serde_json::Error), + /// Unknown subject name: `{0}` + UnknownSubjectName(String), + /// Unsupported wildcard pattern: `{0}` + UnsupportedWildcardPattern(String), + /// Unserializable message payload: `{0}` + UnserializableMessagePayload(serde_json::Error), + /// Stream Error: `{0}` + Stream(#[from] StreamError), + /// Closed by client with reason: `{0}` + ClosedWithReason(String), +} diff --git a/crates/fuel-streams-ws/src/server/ws/fuel_streams.rs b/crates/fuel-streams-ws/src/server/ws/fuel_streams.rs new file mode 100644 index 00000000..526550cb --- /dev/null +++ b/crates/fuel-streams-ws/src/server/ws/fuel_streams.rs @@ -0,0 +1,198 @@ +use std::sync::Arc; + +use async_nats::{ + jetstream::{context::CreateStreamErrorKind, stream::State as StreamState}, + RequestErrorKind, +}; +use fuel_streams::types::Log; +use fuel_streams_core::{prelude::*, SubscriptionConfig}; +use futures::stream::BoxStream; + +#[derive(Clone)] +/// Streams we currently support publishing to. +pub struct FuelStreams { + pub transactions: Stream, + pub blocks: Stream, + pub inputs: Stream, + pub outputs: Stream, + pub receipts: Stream, + pub utxos: Stream, + pub logs: Stream, +} + +impl FuelStreams { + pub async fn new( + nats_client: &NatsClient, + s3_client: &Arc, + ) -> Self { + Self { + transactions: Stream::::new(nats_client, s3_client) + .await, + blocks: Stream::::new(nats_client, s3_client).await, + inputs: Stream::::new(nats_client, s3_client).await, + outputs: Stream::::new(nats_client, s3_client).await, + receipts: Stream::::new(nats_client, s3_client).await, + utxos: Stream::::new(nats_client, s3_client).await, + logs: Stream::::new(nats_client, s3_client).await, + } + } + + pub async fn subscribe( + &self, + sub_subject: &str, + subscription_config: Option, + ) -> Result>, StreamError> { + match sub_subject { + Transaction::NAME => { + self.transactions.subscribe_raw(subscription_config).await + } + Block::NAME => self.blocks.subscribe_raw(subscription_config).await, + Input::NAME => self.inputs.subscribe_raw(subscription_config).await, + Output::NAME => { + self.outputs.subscribe_raw(subscription_config).await + } + Receipt::NAME => { + self.receipts.subscribe_raw(subscription_config).await + } + Utxo::NAME => self.utxos.subscribe_raw(subscription_config).await, + Log::NAME => self.logs.subscribe_raw(subscription_config).await, + _ => Err(StreamError::StreamCreation( + CreateStreamErrorKind::InvalidStreamName.into(), + )), + } + } +} + +#[async_trait::async_trait] +pub trait FuelStreamsExt: Sync + Send + 'static { + fn blocks(&self) -> &Stream; + fn transactions(&self) -> &Stream; + fn inputs(&self) -> &Stream; + fn outputs(&self) -> &Stream; + fn receipts(&self) -> &Stream; + fn utxos(&self) -> &Stream; + fn logs(&self) -> &Stream; + + async fn get_last_published_block(&self) -> anyhow::Result>; + + fn subjects_names() -> &'static [&'static str] { + &[ + Transaction::NAME, + Block::NAME, + Input::NAME, + Receipt::NAME, + Utxo::NAME, + Log::NAME, + ] + } + + fn is_within_subject_names(subject_name: &str) -> bool { + let subject_names = Self::subjects_names(); + subject_names.contains(&subject_name) + } + + fn subjects_wildcards(&self) -> &[&'static str] { + &[ + TransactionsSubject::WILDCARD, + BlocksSubject::WILDCARD, + InputsByIdSubject::WILDCARD, + InputsCoinSubject::WILDCARD, + InputsMessageSubject::WILDCARD, + InputsContractSubject::WILDCARD, + ReceiptsLogSubject::WILDCARD, + ReceiptsBurnSubject::WILDCARD, + ReceiptsByIdSubject::WILDCARD, + ReceiptsCallSubject::WILDCARD, + ReceiptsMintSubject::WILDCARD, + ReceiptsPanicSubject::WILDCARD, + ReceiptsReturnSubject::WILDCARD, + ReceiptsRevertSubject::WILDCARD, + ReceiptsLogDataSubject::WILDCARD, + ReceiptsTransferSubject::WILDCARD, + ReceiptsMessageOutSubject::WILDCARD, + ReceiptsReturnDataSubject::WILDCARD, + ReceiptsTransferOutSubject::WILDCARD, + ReceiptsScriptResultSubject::WILDCARD, + UtxosSubject::WILDCARD, + LogsSubject::WILDCARD, + ] + } + + fn wildcards() -> Vec<&'static str> { + let nested_wildcards = [ + Transaction::WILDCARD_LIST, + Block::WILDCARD_LIST, + Input::WILDCARD_LIST, + Receipt::WILDCARD_LIST, + Utxo::WILDCARD_LIST, + Log::WILDCARD_LIST, + ]; + nested_wildcards + .into_iter() + .flatten() + .copied() + .collect::>() + } + + async fn get_consumers_and_state( + &self, + ) -> Result, StreamState)>, RequestErrorKind>; + + #[cfg(feature = "test-helpers")] + async fn is_empty(&self) -> bool; +} + +#[async_trait::async_trait] +impl FuelStreamsExt for FuelStreams { + fn blocks(&self) -> &Stream { + &self.blocks + } + fn transactions(&self) -> &Stream { + &self.transactions + } + fn inputs(&self) -> &Stream { + &self.inputs + } + fn outputs(&self) -> &Stream { + &self.outputs + } + fn receipts(&self) -> &Stream { + &self.receipts + } + fn utxos(&self) -> &Stream { + &self.utxos + } + fn logs(&self) -> &Stream { + &self.logs + } + + async fn get_last_published_block(&self) -> anyhow::Result> { + Ok(self + .blocks + .get_last_published(BlocksSubject::WILDCARD) + .await?) + } + + async fn get_consumers_and_state( + &self, + ) -> Result, StreamState)>, RequestErrorKind> { + Ok(vec![ + self.transactions.get_consumers_and_state().await?, + self.blocks.get_consumers_and_state().await?, + self.inputs.get_consumers_and_state().await?, + self.outputs.get_consumers_and_state().await?, + self.receipts.get_consumers_and_state().await?, + self.utxos.get_consumers_and_state().await?, + self.logs.get_consumers_and_state().await?, + ]) + } + + #[cfg(feature = "test-helpers")] + async fn is_empty(&self) -> bool { + self.blocks.is_empty(BlocksSubject::WILDCARD).await + && self + .transactions + .is_empty(TransactionsSubject::WILDCARD) + .await + } +} diff --git a/crates/fuel-streams-ws/src/server/ws/mod.rs b/crates/fuel-streams-ws/src/server/ws/mod.rs new file mode 100644 index 00000000..c1f05088 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/ws/mod.rs @@ -0,0 +1,5 @@ +pub mod errors; +pub mod fuel_streams; +pub mod models; +pub mod socket; +pub mod state; diff --git a/crates/fuel-streams-ws/src/server/ws/models.rs b/crates/fuel-streams-ws/src/server/ws/models.rs new file mode 100644 index 00000000..3033e5f8 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/ws/models.rs @@ -0,0 +1,106 @@ +use fuel_streams_storage::DeliverPolicy as NatsDeliverPolicy; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +#[serde(rename_all = "camelCase")] +pub enum DeliverPolicy { + All, + Last, + New, + ByStartSequence { + #[serde(rename = "optStartSeq")] + start_sequence: u64, + }, + ByStartTime { + #[serde(rename = "optStartTime")] + start_time: time::OffsetDateTime, + }, + LastPerSubject, +} + +impl From for NatsDeliverPolicy { + fn from(policy: DeliverPolicy) -> Self { + match policy { + DeliverPolicy::All => NatsDeliverPolicy::All, + DeliverPolicy::Last => NatsDeliverPolicy::Last, + DeliverPolicy::New => NatsDeliverPolicy::New, + DeliverPolicy::ByStartSequence { start_sequence } => { + NatsDeliverPolicy::ByStartSequence { start_sequence } + } + DeliverPolicy::ByStartTime { start_time } => { + NatsDeliverPolicy::ByStartTime { start_time } + } + DeliverPolicy::LastPerSubject => NatsDeliverPolicy::LastPerSubject, + } + } +} + +impl From for DeliverPolicy { + fn from(policy: NatsDeliverPolicy) -> Self { + match policy { + NatsDeliverPolicy::All => DeliverPolicy::All, + NatsDeliverPolicy::Last => DeliverPolicy::Last, + NatsDeliverPolicy::New => DeliverPolicy::New, + NatsDeliverPolicy::ByStartSequence { start_sequence } => { + DeliverPolicy::ByStartSequence { start_sequence } + } + NatsDeliverPolicy::ByStartTime { start_time } => { + DeliverPolicy::ByStartTime { start_time } + } + NatsDeliverPolicy::LastPerSubject => DeliverPolicy::LastPerSubject, + } + } +} + +#[derive(Eq, PartialEq, Debug, Deserialize, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionPayload { + pub wildcard: String, + pub deliver_policy: DeliverPolicy, +} + +#[derive(Eq, PartialEq, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ClientMessage { + Subscribe(SubscriptionPayload), + Unsubscribe(SubscriptionPayload), +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ServerMessage { + Subscribed(SubscriptionPayload), + Unsubscribed(SubscriptionPayload), + Update(Vec), + Error(String), +} + +#[cfg(test)] +mod tests { + use super::{ClientMessage, DeliverPolicy, SubscriptionPayload}; + + #[test] + fn test_sub_ser() { + let stream_topic_wildcard = "blocks.*.*".to_owned(); + let msg = ClientMessage::Subscribe(SubscriptionPayload { + wildcard: stream_topic_wildcard.clone(), + deliver_policy: DeliverPolicy::All, + }); + let ser_str_value = serde_json::to_string(&msg).unwrap(); + println!("Ser value {:?}", ser_str_value); + let expected_value = serde_json::json!({ + "subscribe": { + "wildcard": stream_topic_wildcard, + "deliverPolicy": "all" + } + }); + let deser_msg_val = + serde_json::from_value::(expected_value).unwrap(); + assert!(msg.eq(&deser_msg_val)); + + let deser_msg_str = + serde_json::from_str::(&ser_str_value).unwrap(); + assert!(msg.eq(&deser_msg_str)); + } +} diff --git a/crates/fuel-streams-ws/src/server/ws/socket.rs b/crates/fuel-streams-ws/src/server/ws/socket.rs new file mode 100644 index 00000000..a705b2cf --- /dev/null +++ b/crates/fuel-streams-ws/src/server/ws/socket.rs @@ -0,0 +1,397 @@ +use std::sync::{atomic::AtomicUsize, Arc}; + +use actix_web::{ + web::{self, Bytes}, + HttpMessage, + HttpRequest, + Responder, +}; +use actix_ws::{Message, Session}; +use fuel_streams::{ + logs::Log, + types::{Block, Input, Output, Receipt, Transaction}, + utxos::Utxo, + StreamEncoder, + Streamable, +}; +use fuel_streams_core::SubscriptionConfig; +use fuel_streams_storage::DeliverPolicy; +use futures::StreamExt; +use uuid::Uuid; + +use super::{ + errors::WsSubscriptionError, + fuel_streams::FuelStreams, + models::ClientMessage, +}; +use crate::{ + server::{ + state::ServerState, + ws::{ + fuel_streams::FuelStreamsExt, + models::{ServerMessage, SubscriptionPayload}, + }, + }, + telemetry::Telemetry, +}; + +static _NEXT_USER_ID: AtomicUsize = AtomicUsize::new(1); + +pub async fn get_ws( + req: HttpRequest, + body: web::Payload, + state: web::Data, +) -> actix_web::Result { + // extract user id + let user_id = match req.extensions().get::() { + Some(user_id) => { + tracing::info!( + "Authenticated WebSocket connection for user: {:?}", + user_id.to_string() + ); + user_id.to_owned() + } + None => { + tracing::info!("Unauthenticated WebSocket connection"); + return Err(actix_web::error::ErrorUnauthorized( + "Missing or invalid JWT", + )); + } + }; + + // split the request into response, session, and message stream + let (response, session, mut msg_stream) = actix_ws::handle(&req, body)?; + + // record the new subscription + state.context.telemetry.increment_subscriptions_count(); + + // spawm an actor handling the ws connection + let streams = state.context.fuel_streams.clone(); + let telemetry = state.context.telemetry.clone(); + actix_web::rt::spawn(async move { + tracing::info!("Ws opened for user id {:?}", user_id.to_string()); + while let Some(Ok(msg)) = msg_stream.recv().await { + let mut session = session.clone(); + match msg { + Message::Ping(bytes) => { + tracing::info!("Received ping, {:?}", bytes); + if session.pong(&bytes).await.is_err() { + tracing::error!("Error sending pong, {:?}", bytes); + } + } + Message::Pong(bytes) => { + tracing::info!("Received pong, {:?}", bytes); + } + Message::Text(string) => { + tracing::info!("Received text, {string}"); + } + Message::Binary(bytes) => { + tracing::info!("Received binary {:?}", bytes); + let client_message = match parse_client_message(bytes) { + Ok(msg) => msg, + Err(e) => { + close_socket_with_error( + e, user_id, session, None, telemetry, + ) + .await; + return; + } + }; + + // handle the client message + match client_message { + ClientMessage::Subscribe(payload) => { + tracing::info!( + "Received subscribe message: {:?}", + payload + ); + let subject_wildcard = payload.wildcard; + let deliver_policy = payload.deliver_policy; + + // verify the subject name + let sub_subject = + match verify_and_extract_subject_name( + &subject_wildcard, + ) { + Ok(res) => res, + Err(e) => { + close_socket_with_error( + e, + user_id, + session, + Some(subject_wildcard.clone()), + telemetry, + ) + .await; + return; + } + }; + + // start the streamer async + let mut stream_session = session.clone(); + + // reply to socket with subscription + send_message_to_socket( + &mut session, + ServerMessage::Subscribed( + SubscriptionPayload { + wildcard: subject_wildcard.clone(), + deliver_policy, + }, + ), + ) + .await; + + // receive streaming in a background thread + let streams = streams.clone(); + let telemetry = telemetry.clone(); + actix_web::rt::spawn(async move { + // update metrics + telemetry.update_user_subscription_metrics( + user_id, + &subject_wildcard, + ); + + // subscribe to the stream + let config = SubscriptionConfig { + deliver_policy: DeliverPolicy::All, + filter_subjects: vec![ + subject_wildcard.clone() + ], + }; + let mut sub = match streams + .subscribe(&sub_subject, Some(config)) + .await + { + Ok(sub) => sub, + Err(e) => { + close_socket_with_error( + WsSubscriptionError::Stream(e), + user_id, + session, + Some(subject_wildcard.clone()), + telemetry, + ) + .await; + return; + } + }; + + // consume and forward to the ws + while let Some(s3_serialized_payload) = + sub.next().await + { + // decode and serialize back to ws payload + let serialized_ws_payload = match decode( + &subject_wildcard, + s3_serialized_payload, + ) + .await + { + Ok(res) => res, + Err(e) => { + telemetry.update_error_metrics( + &subject_wildcard, + &e.to_string(), + ); + tracing::error!("Error serializing received stream message: {:?}", e); + continue; + } + }; + + // send the payload over the stream + let _ = stream_session + .binary(serialized_ws_payload) + .await; + } + }); + } + ClientMessage::Unsubscribe(payload) => { + tracing::info!( + "Received unsubscribe message: {:?}", + payload + ); + let subject_wildcard = payload.wildcard; + + let deliver_policy = payload.deliver_policy; + + if let Err(e) = verify_and_extract_subject_name( + &subject_wildcard, + ) { + close_socket_with_error( + e, + user_id, + session, + Some(subject_wildcard.clone()), + telemetry, + ) + .await; + return; + } + + // TODO: implement session management for the same user_id + + // send a message to the client to confirm unsubscribing + send_message_to_socket( + &mut session, + ServerMessage::Unsubscribed( + SubscriptionPayload { + wildcard: subject_wildcard, + deliver_policy, + }, + ), + ) + .await; + return; + } + } + } + Message::Close(reason) => { + tracing::info!( + "Got close event, terminating session with reason {:?}", + reason + ); + let reason_str = + reason.and_then(|r| r.description).unwrap_or_default(); + close_socket_with_error( + WsSubscriptionError::ClosedWithReason( + reason_str.to_string(), + ), + user_id, + session, + None, + telemetry, + ) + .await; + return; + } + _ => { + tracing::error!("Received unknown message type"); + close_socket_with_error( + WsSubscriptionError::ClosedWithReason( + "Unknown message type".to_string(), + ), + user_id, + session, + None, + telemetry, + ) + .await; + return; + } + }; + } + }); + + Ok(response) +} + +fn parse_client_message( + msg: Bytes, +) -> Result { + let msg = serde_json::from_slice::(&msg) + .map_err(WsSubscriptionError::UnparsablePayload)?; + Ok(msg) +} + +fn stream_to_server_message( + msg: Vec, +) -> Result, WsSubscriptionError> { + let server_message = serde_json::to_vec(&ServerMessage::Update(msg)) + .map_err(WsSubscriptionError::UnserializableMessagePayload)?; + Ok(server_message) +} + +pub fn verify_and_extract_subject_name( + subject_wildcard: &str, +) -> Result { + let mut subject_parts = subject_wildcard.split('.'); + // TODO: more advanced checks here with Regex + if subject_parts.clone().count() == 1 { + return Err(WsSubscriptionError::UnsupportedWildcardPattern( + subject_wildcard.to_string(), + )); + } + let subject_name = subject_parts.next().unwrap_or_default(); + if !FuelStreams::is_within_subject_names(subject_name) { + return Err(WsSubscriptionError::UnknownSubjectName( + subject_wildcard.to_string(), + )); + } + Ok(subject_name.to_string()) +} + +async fn close_socket_with_error( + e: WsSubscriptionError, + user_id: uuid::Uuid, + mut session: Session, + subject_wildcard: Option, + telemetry: Arc, +) { + tracing::error!("ws subscription error: {:?}", e.to_string()); + if let Some(subject_wildcard) = subject_wildcard { + telemetry.update_error_metrics(&subject_wildcard, &e.to_string()); + telemetry.update_unsubscribed(user_id, &subject_wildcard); + } + telemetry.decrement_subscriptions_count(); + send_message_to_socket(&mut session, ServerMessage::Error(e.to_string())) + .await; + let _ = session.close(None).await; +} + +async fn send_message_to_socket(session: &mut Session, message: ServerMessage) { + let data = serde_json::to_vec(&message).ok().unwrap_or_default(); + let _ = session.binary(data).await; +} + +async fn decode( + subject_wildcard: &str, + s3_payload: Vec, +) -> Result, WsSubscriptionError> { + let subject = verify_and_extract_subject_name(subject_wildcard)?; + match subject.as_str() { + Transaction::NAME => { + let entity = Transaction::decode_or_panic(s3_payload); + let serialized_data = serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload)?; + stream_to_server_message(serialized_data) + } + Block::NAME => { + let entity = Block::decode_or_panic(s3_payload); + let serialized_data = serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload)?; + stream_to_server_message(serialized_data) + } + Input::NAME => { + let entity = Input::decode_or_panic(s3_payload); + let serialized_data = serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload)?; + stream_to_server_message(serialized_data) + } + Output::NAME => { + let entity = Output::decode_or_panic(s3_payload); + let serialized_data = serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload)?; + stream_to_server_message(serialized_data) + } + Receipt::NAME => { + let entity = Receipt::decode_or_panic(s3_payload); + let serialized_data = serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload)?; + stream_to_server_message(serialized_data) + } + Utxo::NAME => { + let entity = Utxo::decode_or_panic(s3_payload); + let serialized_data = serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload)?; + stream_to_server_message(serialized_data) + } + Log::NAME => { + let entity = Log::decode_or_panic(s3_payload); + let serialized_data = serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload)?; + stream_to_server_message(serialized_data) + } + _ => Err(WsSubscriptionError::UnknownSubjectName(subject.to_string())), + } +} diff --git a/crates/fuel-streams-ws/src/server/ws/state.rs b/crates/fuel-streams-ws/src/server/ws/state.rs new file mode 100644 index 00000000..4ee4ced1 --- /dev/null +++ b/crates/fuel-streams-ws/src/server/ws/state.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; + +use actix_ws::Session; +use bytestring::ByteString; +use futures_util::{stream::FuturesUnordered, StreamExt as _}; +use tokio::sync::Mutex; + +#[allow(dead_code)] +#[derive(Clone)] +struct WsClient { + inner: Arc>, +} + +#[allow(dead_code)] +struct WsClientInner { + sessions: Vec, +} + +#[allow(dead_code)] +impl WsClient { + fn new() -> Self { + WsClient { + inner: Arc::new(Mutex::new(WsClientInner { + sessions: Vec::new(), + })), + } + } + + async fn insert(&self, session: Session) { + self.inner.lock().await.sessions.push(session); + } + + async fn broadcast(&self, msg: impl Into) { + let msg = msg.into(); + + let mut inner = self.inner.lock().await; + let mut unordered = FuturesUnordered::new(); + + for mut session in inner.sessions.drain(..) { + let msg = msg.clone(); + + unordered.push(async move { + let res = session.text(msg).await; + res.map(|_| session) + .map_err(|_| tracing::debug!("Dropping session")) + }); + } + + while let Some(res) = unordered.next().await { + if let Ok(session) = res { + inner.sessions.push(session); + } + } + } +} diff --git a/crates/fuel-streams-ws/src/telemetry/elastic_search.rs b/crates/fuel-streams-ws/src/telemetry/elastic_search.rs new file mode 100755 index 00000000..905ef088 --- /dev/null +++ b/crates/fuel-streams-ws/src/telemetry/elastic_search.rs @@ -0,0 +1,323 @@ +// TODO: Consider using external lib for elasticsearch +// TODO: Consider modularizing this module further + +use std::{fs, io, path::PathBuf, sync::Arc}; + +use anyhow::Context; +use chrono::Utc; +use displaydoc::Display; +pub use elasticsearch::params::Refresh; +use elasticsearch::{ + self, + auth::{ClientCertificate, Credentials}, + cert::{Certificate, CertificateValidation}, + http::transport::{SingleNodeConnectionPool, Transport, TransportBuilder}, + params, + Elasticsearch, + IndexParts, +}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::{self, Url}; + +pub const ELASTICSEARCH_PATH: &str = "fuel-data-systems"; + +/// LogEntry represents a log entry that will be stored in Elastic Search +/// for monitoring purposes. +/// TODO: Consider adding more useful optional fields to this struct +#[derive(Serialize, Deserialize)] +pub struct LogEntry { + timestamp: chrono::DateTime, + level: String, + message: String, +} + +impl LogEntry { + pub fn new(level: &str, message: &str) -> Self { + Self { + timestamp: Utc::now(), + level: level.to_string(), + message: message.to_string(), + } + } +} + +pub async fn log(elastic_search: Arc, log_entry: LogEntry) { + if let Err(err) = elastic_search + .get_conn() + .index( + ELASTICSEARCH_PATH, + Some("publisher-logs"), + &log_entry, + Some(Refresh::WaitFor), + ) + .await + { + tracing::error!("Failed to log to ElasticSearch: {}", err); + } +} + +pub fn should_use_elasticsearch() -> bool { + dotenvy::var("USE_ELASTIC_LOGGING").is_ok_and(|val| val == "true") +} + +pub async fn new_elastic_search() -> anyhow::Result { + let elasticsearch_url = dotenvy::var("ELASTICSEARCH_URL") + .expect("`ELASTICSEARCH_URL` env must be set"); + let elsaticsearch_username = dotenvy::var("ELASTICSEARCH_USERNAME") + .expect("`ELASTICSEARCH_USERNAME` env must be set"); + let elsaticsearch_password = dotenvy::var("ELASTICSEARCH_PASSWORD") + .expect("`ELASTICSEARCH_PASSWORD` env must be set"); + + let config = Config { + url: elasticsearch_url, + enabled: true, + pool_max_size: Some(2), + username: Some(elsaticsearch_username), + password: Some(elsaticsearch_password), + ..Default::default() + }; + let client = ElasticSearch::new(&config) + .await + .context("Failed to configure Elasticsearch connection")?; + Ok(client) +} + +/// Elasticsearch errors +#[derive(Debug, Display, Error)] +pub enum ElasticSearchError { + /// ElasticSearchConfigError: `{0}` + Config(#[from] elasticsearch::http::transport::BuildError), + /// ElasticSearchDisabled + Disabled, + /// ElasticSearchError: `{0}` + Generic(#[from] elasticsearch::Error), + /// IoError: `{0}` + Io(#[from] io::Error), + /// UrlParseError: `{0}` + UrlParse(#[from] url::ParseError), + /// CertificateError: `{0}`: `{0}` + Certificate(PathBuf, io::Error), + /// SerdeJsonError: `{0}` + SerdeJson(#[from] serde_json::Error), +} + +#[derive(Debug, Deserialize, Clone, Default)] +#[serde(rename_all = "kebab-case")] +#[serde(default)] +pub struct Config { + pub url: String, + pub enabled: bool, + pub username: Option, + pub password: Option, + pub api_key_id: Option, + pub api_key_value: Option, + pub pool_max_size: Option, + pub pool_min_size: Option, + pub tls: Option, +} + +/// TLS acceptor configuration. +#[derive(Debug, Deserialize, Clone, Default)] +#[serde(rename_all = "kebab-case")] +#[serde(default)] +pub struct TlsConfig { + /// Filename of CA certificates in PEM format. + pub ca: Option, + /// Filename of combined TLS client certificate and key in PKCS#12 format. + pub certificate: Option, + /// Optional passphrase to decode the TLS private key. + pub key_passphrase: Option, +} + +#[derive(Debug, Clone)] +pub struct ElasticSearch(ElasticConnection); + +impl ElasticSearch { + pub async fn new(config: &Config) -> Result { + if !config.enabled { + return Err(ElasticSearchError::Disabled); + } + let conn_info = ConnectionInfo::new(config)?; + let conn = conn_info + .get_connection() + .expect("connection must be created"); + Ok(Self(conn)) + } + + pub fn get_conn(&self) -> &ElasticConnection { + &self.0 + } +} + +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct BulkResults { + pub errors: bool, + #[serde(rename = "items")] + pub results: Vec>, +} + +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Operation { + Create(T), + Delete(T), + Index(T), + Update(T), +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +pub struct OperationParams { + #[serde(rename = "_id", skip_serializing_if = "Option::is_none")] + id: Option, + #[serde(rename = "_index", skip_serializing_if = "Option::is_none")] + index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + version_type: Option, +} + +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct OperationStatus { + #[serde(rename = "_id")] + pub id: Option, + #[serde(rename = "_index")] + pub index: Option, + #[serde(rename = "status")] + pub http_code: u32, + #[serde(flatten)] + pub result: OperationResult, +} + +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub enum OperationResult { + #[serde(rename = "result")] + Ok(String), + #[serde(rename = "error")] + Error { + #[serde(rename = "type")] + kind: String, + reason: String, + }, +} + +#[derive(Clone, Debug)] +pub struct ConnectionInfo(Transport); + +impl ConnectionInfo { + pub fn new(config: &Config) -> Result { + let url = Url::parse(&config.url)?; + let pool = SingleNodeConnectionPool::new(url); + let transport = TransportBuilder::new(pool); + let tls = config.tls.clone().unwrap_or_default(); + let credentials = match ( + config.api_key_id.as_ref(), + config.api_key_value.as_ref(), + tls.certificate, + ) { + (Some(api_key_id), Some(api_key_value), _) => Some( + Credentials::ApiKey(api_key_id.into(), api_key_value.into()), + ), + (_, _, Some(certificate)) => { + Some(Credentials::Certificate(ClientCertificate::Pkcs12( + fs::read(&certificate).map_err(|err| { + ElasticSearchError::Certificate(certificate, err) + })?, + tls.key_passphrase, + ))) + } + _ => config.username.as_ref().map(|username| { + Credentials::Basic( + username.into(), + config.password.clone().unwrap_or_default(), + ) + }), + }; + let transport = if let Some(ca) = tls.ca { + transport.cert_validation(CertificateValidation::Full( + Certificate::from_pem(&fs::read(&ca).map_err(|err| { + ElasticSearchError::Certificate(ca.clone(), err) + })?) + .map_err(|err| { + ElasticSearchError::Certificate( + ca, + io::Error::new(io::ErrorKind::Other, err), + ) + })?, + )) + } else { + transport + }; + let transport = if let Some(credentials) = credentials { + transport.auth(credentials) + } else { + transport + }; + let inner = transport.build()?; + Ok(Self(inner)) + } + + pub fn get_connection( + &self, + ) -> Result { + let conn = Elasticsearch::new(self.0.clone()); + Ok(ElasticConnection(Some(conn))) + } +} + +#[derive(Debug, Clone)] +pub struct ElasticConnection(Option); + +impl ElasticConnection { + pub fn check_alive(&self) -> Option { + Some(self.0.is_some()) + } + + pub async fn ping(&self) -> Result<(), ElasticSearchError> { + let conn = self.0.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::ConnectionAborted, + "Connection to Elasticsearch is already closed", + ) + })?; + + let response = conn.ping().send().await?; + let _ = response.error_for_status_code()?; + Ok(()) + } +} + +impl ElasticConnection { + pub async fn index( + &self, + path: &str, + id: Option<&str>, + doc: B, + refresh: Option, + ) -> Result<(), ElasticSearchError> + where + B: Serialize, + { + let conn = self.0.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::ConnectionAborted, + "Connection to Elasticsearch is already closed", + ) + })?; + let index_parts = id + .map(|id| IndexParts::IndexId(path, id)) + .unwrap_or(IndexParts::Index(path)); + + let response = conn + .index(index_parts) + .body(doc) + .refresh(refresh.unwrap_or(Refresh::False)) + .send() + .await?; + response + .error_for_status_code() + .map(|_| ()) + .map_err(Into::into) + } +} diff --git a/crates/fuel-streams-ws/src/telemetry/metrics.rs b/crates/fuel-streams-ws/src/telemetry/metrics.rs new file mode 100644 index 00000000..5a65f1a0 --- /dev/null +++ b/crates/fuel-streams-ws/src/telemetry/metrics.rs @@ -0,0 +1,190 @@ +use prometheus::{ + register_int_counter_vec, + register_int_gauge_vec, + IntCounterVec, + IntGaugeVec, + Registry, +}; +use rand::{distributions::Alphanumeric, Rng}; + +#[derive(Clone, Debug)] +pub struct Metrics { + pub registry: Registry, + pub total_ws_subs: IntGaugeVec, + pub user_subscribed_messages: IntGaugeVec, + pub subs_messages_throughput: IntCounterVec, + pub subs_messages_error_rates: IntCounterVec, +} + +impl Default for Metrics { + fn default() -> Self { + Metrics::new(None).expect("Failed to create default Metrics") + } +} + +impl Metrics { + pub fn generate_random_prefix() -> String { + rand::thread_rng() + .sample_iter(&Alphanumeric) + .filter(|c| c.is_ascii_alphabetic()) + .take(6) + .map(char::from) + .collect() + } + + pub fn new_with_random_prefix() -> anyhow::Result { + Metrics::new(Some(Metrics::generate_random_prefix())) + } + + pub fn new(prefix: Option) -> anyhow::Result { + let metric_prefix = prefix + .clone() + .map(|p| format!("{}_", p)) + .unwrap_or_default(); + + let total_ws_subs = register_int_gauge_vec!( + format!("{}ws_streamer_metrics_total_subscriptions", metric_prefix), + "A metric counting the number of active ws subscriptions", + &[], + ) + .expect("metric must be created"); + + let user_subscribed_messages = register_int_gauge_vec!( + format!( + "{}ws_streamer_metrics_user_subscribed_messages", + metric_prefix + ), + "A metric counting the number of published messages", + &["user_id", "subject_wildcard"], + ) + .expect("metric must be created"); + + let subs_messages_throughput = register_int_counter_vec!( + format!("{}ws_streamer_metrics_subs_messages_throughput", metric_prefix), + "A metric counting the number of subscription messages per subject wildcard", + &["subject_wildcard"], + ) + .expect("metric must be created"); + + let subs_messages_error_rates = + register_int_counter_vec!( + format!("{}ws_streamer_metrics_subs_messages_error_rates", metric_prefix), + "A metric counting errors or failures during subscription message processing", + &["subject_wildcard", "error_type"], + ) + .expect("metric must be created"); + + let registry = + Registry::new_custom(prefix, None).expect("registry to be created"); + registry.register(Box::new(total_ws_subs.clone()))?; + registry.register(Box::new(user_subscribed_messages.clone()))?; + registry.register(Box::new(subs_messages_throughput.clone()))?; + registry.register(Box::new(subs_messages_error_rates.clone()))?; + + Ok(Self { + registry, + total_ws_subs, + user_subscribed_messages, + subs_messages_throughput, + subs_messages_error_rates, + }) + } +} + +#[cfg(test)] +mod tests { + use prometheus::{gather, Encoder, TextEncoder}; + + use super::*; + + impl Metrics { + pub fn random() -> Self { + Metrics::new_with_random_prefix() + .expect("Failed to create random Metrics") + } + } + + #[test] + fn test_user_subscribed_messages_metric() { + let metrics = Metrics::random(); + + metrics + .user_subscribed_messages + .with_label_values(&["user_id_1", "subject_wildcard_1"]) + .set(5); + + let metric_families = gather(); + let mut buffer = Vec::new(); + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + let output = String::from_utf8(buffer.clone()).unwrap(); + + assert!(output.contains("ws_streamer_metrics_user_subscribed_messages")); + assert!(output.contains("user_id_1")); + assert!(output.contains("subject_wildcard_1")); + assert!(output.contains("5")); + } + + #[test] + fn test_subs_messages_total_metric() { + let metrics = Metrics::random(); + + metrics.total_ws_subs.with_label_values(&[]).set(10); + + let metric_families = gather(); + let mut buffer = Vec::new(); + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + let output = String::from_utf8(buffer.clone()).unwrap(); + + assert!(output.contains("ws_streamer_metrics_total_subscriptions")); + assert!(output.contains("10")); + } + + #[test] + fn test_subs_messages_throughput_metric() { + let metrics = Metrics::random(); + + metrics + .subs_messages_throughput + .with_label_values(&["wildcard_1"]) + .inc_by(10); + + let metric_families = gather(); + let mut buffer = Vec::new(); + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + let output = String::from_utf8(buffer.clone()).unwrap(); + + assert!(output.contains("ws_streamer_metrics_subs_messages_throughput")); + assert!(output.contains("wildcard_1")); + assert!(output.contains("10")); + } + + #[test] + fn test_subs_messages_error_rates_metric() { + let metrics = Metrics::random(); + + metrics + .subs_messages_error_rates + .with_label_values(&["wildcard_1", "timeout"]) + .inc_by(1); + + let metric_families = gather(); + let mut buffer = Vec::new(); + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + let output = String::from_utf8(buffer.clone()).unwrap(); + + assert!( + output.contains("ws_streamer_metrics_subs_messages_error_rates") + ); + assert!(output.contains("wildcard_1")); + assert!(output.contains("timeout")); + assert!(output.contains("1")); + } +} diff --git a/crates/fuel-streams-ws/src/telemetry/mod.rs b/crates/fuel-streams-ws/src/telemetry/mod.rs new file mode 100644 index 00000000..db2a82db --- /dev/null +++ b/crates/fuel-streams-ws/src/telemetry/mod.rs @@ -0,0 +1,248 @@ +mod elastic_search; +pub mod metrics; +mod runtime; +#[allow(clippy::needless_borrows_for_generic_args)] +mod system; + +use std::{sync::Arc, time::Duration}; + +use anyhow::Context; +use elastic_search::{ + new_elastic_search, + should_use_elasticsearch, + ElasticSearch, + LogEntry, +}; +use metrics::Metrics; +// TODO: Consider using tokio's Rwlock instead +use parking_lot::RwLock; +use runtime::Runtime; +use system::{System, SystemMetricsWrapper}; + +#[derive(Clone)] +pub struct Telemetry { + runtime: Arc, + system: Arc>, + metrics: Option>, + elastic_search: Option>, +} + +impl Telemetry { + const DEDICATED_THREADS: usize = 2; + + pub async fn new(prefix: Option) -> anyhow::Result> { + let runtime = + Runtime::new(Self::DEDICATED_THREADS, Duration::from_secs(20)); + let system = Arc::new(RwLock::new(System::new().await)); + + let metrics = if should_use_metrics() { + Some(Arc::new(Metrics::new(prefix)?)) + } else { + None + }; + + let elastic_search = if should_use_elasticsearch() { + Some(Arc::new(new_elastic_search().await?)) + } else { + None + }; + + Ok(Arc::new(Self { + runtime: Arc::new(runtime), + system, + metrics, + elastic_search, + })) + } + + pub async fn start(&self) -> anyhow::Result<()> { + let system = Arc::clone(&self.system); + + if let Some(elastic_search) = self.elastic_search.as_ref() { + tracing::info!( + "Elastic Search connection live? {:?}", + elastic_search.get_conn().check_alive().unwrap_or_default() + ); + elastic_search + .get_conn() + .ping() + .await + .context("Error pinging elastisearch connection")?; + tracing::info!("Elastic logger pinged successfully!"); + }; + + self.runtime.start(move || { + system.write().refresh(); + }); + + Ok(()) + } + + pub fn log_info(&self, message: &str) { + let entry = LogEntry::new("INFO", message); + self.maybe_elog(entry); + tracing::info!("{}", message); + } + + pub fn log_error(&self, message: &str) { + let entry = LogEntry::new("ERROR", message); + self.maybe_elog(entry); + tracing::error!("{}", message); + } + + fn maybe_elog(&self, entry: LogEntry) { + if let Some(elastic_search) = &self.elastic_search { + self.runtime + .spawn(elastic_search::log(elastic_search.clone(), entry)); + } + } + + pub fn update_user_subscription_metrics( + &self, + user_id: uuid::Uuid, + subject_wildcard: &str, + ) { + self.maybe_use_metrics(|metrics| { + // Increment total user subscribed messages + metrics + .user_subscribed_messages + .with_label_values(&[ + user_id.to_string().as_str(), + subject_wildcard, + ]) + .inc(); + + // Increment throughput for the subscribed messages + metrics + .subs_messages_throughput + .with_label_values(&[subject_wildcard]) + .inc(); + }); + } + + pub fn update_error_metrics( + &self, + subject_wildcard: &str, + error_type: &str, + ) { + self.maybe_use_metrics(|metrics| { + metrics + .subs_messages_error_rates + .with_label_values(&[subject_wildcard, error_type]) + .inc(); + }); + } + + pub fn increment_subscriptions_count(&self) { + self.maybe_use_metrics(|metrics| { + metrics.total_ws_subs.with_label_values(&[]).inc(); + }); + } + + pub fn decrement_subscriptions_count(&self) { + self.maybe_use_metrics(|metrics| { + metrics.total_ws_subs.with_label_values(&[]).inc(); + }); + } + + pub fn update_unsubscribed( + &self, + user_id: uuid::Uuid, + subject_wildcard: &str, + ) { + self.maybe_use_metrics(|metrics| { + metrics + .user_subscribed_messages + .with_label_values(&[&user_id.to_string(), subject_wildcard]) + .dec(); + }); + } + + pub fn maybe_use_metrics(&self, f: F) + where + F: Fn(&Metrics), + { + if let Some(metrics) = &self.metrics { + f(metrics); + } + } + + pub async fn get_metrics(&self) -> String { + use prometheus::Encoder; + let encoder = prometheus::TextEncoder::new(); + + if self.metrics.is_none() { + return "".to_string(); + } + + // fetch all measured metrics + let mut buffer = Vec::new(); + if let Err(e) = encoder.encode( + &self.metrics.as_ref().unwrap().registry.gather(), + &mut buffer, + ) { + tracing::error!("could not encode custom metrics: {}", e); + }; + let mut res = match String::from_utf8(buffer.clone()) { + Ok(v) => v, + Err(e) => { + tracing::error!( + "custom metrics could not be from_utf8'd: {}", + e + ); + String::default() + } + }; + buffer.clear(); + + let mut buffer = Vec::new(); + if let Err(e) = encoder.encode(&prometheus::gather(), &mut buffer) { + tracing::error!("could not encode prometheus metrics: {}", e); + }; + let res_custom = match String::from_utf8(buffer.clone()) { + Ok(v) => v, + Err(e) => { + tracing::error!( + "prometheus metrics could not be from_utf8'd: {}", + e + ); + String::default() + } + }; + buffer.clear(); + + res.push_str(&res_custom); + + // now fetch and add system metrics + let system_metrics = match self.system.read().metrics() { + Ok(m) => { + let metrics = SystemMetricsWrapper::from(m); + let labels: Vec<(&str, &str)> = vec![]; + match serde_prometheus::to_string(&metrics, None, labels) { + Ok(m) => m, + Err(err) => { + tracing::error!( + "could not encode system metrics: {:?}", + err + ); + String::default() + } + } + } + Err(err) => { + tracing::error!( + "prometheus system metrics could not be stringified: {:?}", + err + ); + String::default() + } + }; + res.push_str(&system_metrics); + + res + } +} + +pub fn should_use_metrics() -> bool { + dotenvy::var("USE_METRICS").is_ok_and(|val| val == "true") +} diff --git a/crates/fuel-streams-ws/src/telemetry/runtime.rs b/crates/fuel-streams-ws/src/telemetry/runtime.rs new file mode 100644 index 00000000..6c3fc637 --- /dev/null +++ b/crates/fuel-streams-ws/src/telemetry/runtime.rs @@ -0,0 +1,72 @@ +use std::{ + collections::VecDeque, + pin::Pin, + sync::{Arc, Mutex}, +}; + +use futures::Future; +use tokio::time::{self, Duration}; + +// Task type: Each task is represented by a Boxed, pinned Future +type Task = Pin + Send + 'static>>; + +#[derive(Clone)] +pub struct Runtime { + task_queue: Arc>>, + max_capacity: usize, + interval: Duration, +} + +impl Runtime { + pub fn new(capacity: usize, interval: Duration) -> Self { + Self { + task_queue: Arc::new(Mutex::new(VecDeque::with_capacity(capacity))), + max_capacity: capacity, + interval, + } + } + + pub fn spawn(&self, task: F) + where + F: Future + Send + 'static, + { + let mut queue = self.task_queue.lock().unwrap(); + + // If the queue is at capacity, discard the oldest task + if queue.len() >= self.max_capacity { + queue.pop_front(); + } + + queue.push_back(Box::pin(task)); + } + + pub fn start(&self, blocking_task_executor: F) + where + F: FnOnce() + Send + 'static + Clone, + { + let interval = self.interval; + let task_queue = Arc::clone(&self.task_queue); + + tokio::spawn(async move { + let mut ticker = time::interval(interval); + + loop { + // Wait for the interval + ticker.tick().await; + + tokio::task::spawn_blocking(blocking_task_executor.clone()); + + // Lock the queue, drain tasks, and run them sequentially + let tasks: Vec<_> = { + let mut queue = task_queue.lock().unwrap(); + queue.drain(..).collect() + }; + + // Run each task sequentially + for task in tasks { + task.await; + } + } + }); + } +} diff --git a/crates/fuel-streams-ws/src/telemetry/system.rs b/crates/fuel-streams-ws/src/telemetry/system.rs new file mode 100644 index 00000000..ec0f14eb --- /dev/null +++ b/crates/fuel-streams-ws/src/telemetry/system.rs @@ -0,0 +1,634 @@ +use std::{ + collections::HashMap, + convert::TryFrom, + hash::Hash, + path::PathBuf, + time::Duration, +}; + +use derive_more::Deref; +use rust_decimal::{ + prelude::{FromPrimitive, ToPrimitive}, + Decimal, +}; +use serde::{ser::SerializeStruct, Serialize, Serializer}; +use sysinfo::{ + CpuExt, + CpuRefreshKind, + DiskExt, + Pid, + PidExt, + ProcessExt, + RefreshKind, + SystemExt, +}; +use thiserror::Error; +use tokio::time; + +// TODO: move this to web interface as `SystemsMetricsResponse` ? +#[derive(Serialize)] +pub struct SystemMetricsWrapper { + system: SystemMetrics, +} + +impl From for SystemMetricsWrapper { + fn from(system: SystemMetrics) -> Self { + Self { system } + } +} + +#[derive(Debug, Error)] +pub enum Error { + #[error("The process {0} could not be found")] + ProcessNotFound(Pid), +} + +#[derive(Debug, Deref)] +pub struct System { + /// System information from the `sysinfo` crate. + #[deref] + system: sysinfo::System, + /// Refresh settings. + specifics: RefreshKind, + /// Cached physical CPU core count. + cpu_physical_core_count: Option, + /// Process ID. + pid: Pid, +} + +impl System { + pub async fn new() -> Self { + let specifics = RefreshKind::new().with_disks_list().with_memory(); + // Gathering CPU information takes about 150ms+ extra. + let specifics = specifics.with_cpu(CpuRefreshKind::everything()); + + let mut system = sysinfo::System::new_with_specifics(specifics); + + // We're only interested in the current process. + // NOTE: This ::expect can never fail on Linux! + let pid = sysinfo::get_current_pid().expect("Unable to get PID"); + system.refresh_process(pid); + + // We have to refresh the CPU statistics once on startup. + time::sleep(Duration::from_millis(100)).await; + system.refresh_process(pid); + + // Only retrieve the physical CPU core count once (while + // hotplug CPUs exist on virtual and physical platforms, we + // just assume that it is usually not changing on runtime). + let cpu_physical_core_count = system.physical_core_count(); + + Self { + system, + specifics, + cpu_physical_core_count, + pid, + } + } + + pub fn refresh(&mut self) { + self.system.refresh_process(self.pid); + self.system.refresh_specifics(self.specifics); + } + + pub fn metrics(&self) -> Result { + SystemMetrics::try_from(self) + } + + fn pid(&self) -> Pid { + self.pid + } +} + +/// Accumulated system status information. +#[derive(Debug, Default, Serialize)] +pub struct SystemMetrics { + /// Parent process of the application. + pub application: Process, + /// System memory information. + pub memory: SystemMemory, + /// Load averages + pub load_average: LoadAverage, + /// Host and operation system information. + pub host: Host, + /// Disk information and usage. + pub disk: HashMap, + /// CPU physical core count. + #[serde(serialize_with = "format_value")] + pub cpu_physical_core_count: usize, + /// CPU count. + #[serde(serialize_with = "format_value")] + pub cpu_count: usize, + /// CPU information. + pub cpu: HashMap, +} + +impl TryFrom<&System> for SystemMetrics { + type Error = Error; + + fn try_from(system: &System) -> Result { + // Get current pid. + let pid = system.pid(); + + let disk = system + .disks() + .iter() + .map(|v| { + let path = v.mount_point().to_path_buf(); + let disk = Disk::from(v); + (path, disk) + }) + .collect(); + + let cpu = system + .cpus() + .iter() + .enumerate() + .map(|(i, v)| (i, v.into())) + .collect::>(); + // Total number of CPUs (including CPU threads). + let cpu_count = cpu.len(); + + // Use cached number of CPU physical cores, if set. + let cpu_physical_core_count = system + .cpu_physical_core_count + .unwrap_or_else(|| system.physical_core_count().unwrap_or(1)); + + Ok(Self { + application: TryFrom::try_from((system.deref(), pid))?, + memory: system.deref().into(), + load_average: system.deref().into(), + host: system.deref().into(), + disk, + cpu_count, + cpu_physical_core_count, + cpu, + }) + } +} + +/// System memory usage information. +#[derive(Debug, Clone, Default)] +pub struct Memory { + /// Total memory. + size: u64, + /// Used memory. + free: Option, + /// Memory usage in percent. + usage: Decimal, +} + +impl serde::Serialize for Memory { + fn serialize( + &self, + serializer: S, + ) -> Result { + if let Some(free) = self.free { + let mut s = serializer.serialize_struct("Memory", 3)?; + s.serialize_field("size", &Format::Memory(self.size))?; + s.serialize_field("free", &Format::Memory(free))?; + s.serialize_field("usage", &Format::Memory(AsF64(self.usage)))?; + s.end() + } else { + let mut s = serializer.serialize_struct("Memory", 2)?; + s.serialize_field("size", &Format::Memory2(self.size))?; + s.serialize_field("usage", &Format::Memory2(AsF64(self.usage)))?; + s.end() + } + } +} + +/// System memory usage information. +#[derive(Debug, Default, Serialize)] +pub struct SystemMemory { + /// System memory. + system: Memory, + /// Swap memory. + swap: Memory, +} + +impl From<&sysinfo::System> for SystemMemory { + fn from(system: &sysinfo::System) -> Self { + let size = system.total_memory(); + let used = system.used_memory(); + let free = Some(size.saturating_sub(used)); + let usage = percent_usage(used, size); + + let swap_size = system.total_swap(); + let swap_used = system.used_swap(); + let swap_free = Some(swap_size.saturating_sub(swap_used)); + let swap_usage = percent_usage(swap_used, swap_size); + + Self { + system: Memory { size, free, usage }, + swap: Memory { + size: swap_size, + free: swap_free, + usage: swap_usage, + }, + } + } +} + +/// Process information and metrics. +#[derive(Debug)] +pub struct Process { + pid: Pid, + name: String, + cpu_usage: Decimal, + memory: Memory, +} + +impl Default for Process { + fn default() -> Self { + Self { + pid: Pid::from(0), + name: Default::default(), + cpu_usage: Default::default(), + memory: Default::default(), + } + } +} + +impl serde::Serialize for Process { + fn serialize( + &self, + serializer: S, + ) -> Result { + let mut s = serializer.serialize_struct("Process", 4)?; + s.serialize_field( + "pid", + &Format::::Process(self.pid.as_u32() as i32), + )?; + s.serialize_field("name", &FormatKey(&self.name))?; + s.serialize_field( + "cpu_usage", + &Format::Process(AsF64(self.cpu_usage)), + )?; + s.serialize_field("memory", &self.memory)?; + s.end() + } +} + +impl TryFrom<(&sysinfo::System, Pid)> for Process { + type Error = Error; + + fn try_from( + (system, pid): (&sysinfo::System, Pid), + ) -> Result { + let process = system.process(pid).ok_or(Error::ProcessNotFound(pid))?; + + let total = system.total_memory(); + let size = process.memory(); + let usage = percent_usage(size, total); + + Ok(Self { + memory: Memory { + size, + free: None, + usage, + }, + ..Self::from(process) + }) + } +} + +impl From<&sysinfo::Process> for Process { + fn from(process: &sysinfo::Process) -> Self { + Self { + name: process.name().to_string(), + pid: process.pid(), + cpu_usage: decimal(process.cpu_usage()), + memory: Default::default(), + } + } +} + +/// Disk information and usage. +#[derive(Debug)] +pub struct Disk { + size: u64, + free: u64, + usage: Decimal, +} + +impl serde::Serialize for Disk { + fn serialize( + &self, + serializer: S, + ) -> Result { + let mut s = serializer.serialize_struct("Disk", 2)?; + s.serialize_field("size", &Format::Disk(self.size))?; + s.serialize_field("free", &Format::Disk(self.free))?; + s.serialize_field("usage", &Format::Disk(AsF64(self.usage)))?; + s.end() + } +} + +impl From<&sysinfo::Disk> for Disk { + fn from(disk: &sysinfo::Disk) -> Self { + let size = disk.total_space(); + let free = disk.available_space(); + let used = size.saturating_sub(free); + + // Calculate the disk usage in percent. + let usage = percent_usage(used, size); + + Self { size, free, usage } + } +} + +/// System memory usage information. +#[derive(Debug, Default)] +pub struct LoadAverage(f64, f64, f64); + +impl serde::Serialize for LoadAverage { + fn serialize( + &self, + serializer: S, + ) -> Result { + let mut s = serializer.serialize_struct("LoadAverage", 3)?; + s.serialize_field("1", &Format::LoadAverage(self.0))?; + s.serialize_field("5", &Format::LoadAverage(self.1))?; + s.serialize_field("15", &Format::LoadAverage(self.2))?; + s.end() + } +} + +impl From<&sysinfo::System> for LoadAverage { + fn from(system: &sysinfo::System) -> Self { + let load_avg = system.load_average(); + Self(load_avg.one, load_avg.five, load_avg.fifteen) + } +} + +/// System memory usage information. +#[derive(Debug, Default)] +pub struct Cpu { + #[allow(dead_code)] + name: String, + frequency: u64, + usage: Decimal, +} + +impl serde::Serialize for Cpu { + fn serialize( + &self, + serializer: S, + ) -> Result { + let mut s = serializer.serialize_struct("Cpu", 2)?; + s.serialize_field("frequency", &Format::Cpu(self.frequency))?; + s.serialize_field("usage", &Format::Cpu(AsF64(self.usage)))?; + s.end() + } +} + +impl From<&sysinfo::Cpu> for Cpu { + fn from(cpu: &sysinfo::Cpu) -> Self { + Self { + name: cpu.brand().to_string(), + frequency: cpu.frequency(), + usage: decimal(cpu.cpu_usage()), + } + } +} + +/// System memory usage information. +#[derive(Debug, Default)] +pub struct Host { + os_version: String, + kernel_version: String, + uptime: u64, +} + +impl serde::Serialize for Host { + fn serialize( + &self, + serializer: S, + ) -> Result { + let mut s = serializer.serialize_struct("Host", 3)?; + s.serialize_field("os_version", &FormatKey(&self.os_version))?; + s.serialize_field("kernel_version", &FormatKey(&self.kernel_version))?; + s.serialize_field("uptime", &Format::Host(self.uptime))?; + s.end() + } +} + +impl From<&sysinfo::System> for Host { + fn from(system: &sysinfo::System) -> Self { + Self { + os_version: system.long_os_version().unwrap_or_default(), + kernel_version: system.kernel_version().unwrap_or_default(), + uptime: system.uptime(), + } + } +} + +struct AsF64(Decimal); + +impl Serialize for AsF64 { + fn serialize( + &self, + serializer: S, + ) -> Result { + use serde::ser::Error; + let value = self.0.to_f64().ok_or_else(|| { + S::Error::custom(format!( + "Failed to convert a Decimal value into a f64: {:?}", + self.0 + )) + })?; + value.serialize(serializer) + } +} + +pub enum Format { + Cpu(T), + Disk(T), + Host(T), + LoadAverage(T), + Memory(T), + Memory2(T), + Process(T), +} + +impl serde::Serialize for Format { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + // https://en.wikipedia.org/wiki/Brainfuck light + let (code, v) = match &self { + Self::Host(v) => ("<<-|", v), + Self::Cpu(v) => (" (" ("<<-|", v), + Self::Memory(v) => (" ("< ("<<<|", v), + }; + + serializer.serialize_newtype_struct(code, v) + } +} + +pub struct FormatKey(T); + +impl serde::Serialize for FormatKey { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut hashmap = HashMap::new(); + hashmap.insert(&self.0, 1); + serializer.serialize_newtype_struct(".<<<|", &hashmap) + } +} + +fn format_value(value: &usize, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_newtype_struct("<<-|", value) +} + +const DECIMAL_PRECISION: u32 = 4; + +#[inline] +fn percent_usage(current: u64, max: u64) -> Decimal { + Decimal::from(current) + .checked_div(Decimal::from(max)) + .unwrap_or_default() + .checked_mul(100.into()) + .unwrap_or_default() + .round_dp(DECIMAL_PRECISION) +} + +#[inline] +fn decimal(current: f32) -> Decimal { + Decimal::from_f32(current) + .unwrap_or_default() + .round_dp(DECIMAL_PRECISION) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rust_decimal::Decimal; + use serde::Serialize; + + use super::*; + + #[derive(Serialize)] + pub struct Metrics { + system: super::SystemMetrics, + } + + impl From<&System> for Metrics { + fn from(system: &System) -> Self { + Self { + system: system.metrics().expect("metrics"), + } + } + } + + #[tokio::test] + async fn test_metrics_system_values() { + let system = System::new().await; + let metrics = Metrics::from(&system); + + // NOTE: This ::expect can never fail on Linux! + let pid = sysinfo::get_current_pid().expect("Unable to get PID"); + assert_eq!(metrics.system.application.pid, pid); + assert!(metrics.system.host.uptime > 0); + assert!(!metrics.system.cpu.is_empty()); + assert!(!metrics.system.disk.is_empty()); + } + + #[tokio::test] + async fn test_metrics_system_prometheus_full() { + let memory = Memory { + size: 1000, + free: Some(877), + usage: Decimal::new(1234, 2), + }; + + let metrics = Metrics { + system: SystemMetrics { + application: Process { + pid: Pid::from(0), + name: "process".to_string(), + cpu_usage: Decimal::new(1234, 2), + memory: memory.clone(), + }, + memory: SystemMemory { + system: memory.clone(), + swap: memory, + }, + load_average: LoadAverage(1.2, 2.3, 3.4), + host: Host { + os_version: "os-version".to_string(), + kernel_version: "kernel-version".to_string(), + uptime: 123456, + }, + disk: vec![( + PathBuf::from("disk1"), + Disk { + size: 1000, + free: 877, + usage: Decimal::new(1234, 2), + }, + )] + .into_iter() + .collect(), + cpu_physical_core_count: 1, + cpu_count: 1, + cpu: vec![( + 1, + Cpu { + name: "cpu1".to_string(), + frequency: 12345, + usage: Decimal::new(1234, 2), + }, + )] + .into_iter() + .collect(), + }, + }; + + let output = serde_prometheus::to_string(&metrics, None, &[]) + .expect("prometheus"); + + assert_eq!( + output.trim_end().split('\n').collect::>(), + vec![ + r#"system_application_pid 0"#, + r#"system_application_name{path = "process"} 1"#, + r#"system_application_cpu_usage 12.34"#, + r#"system_application_size{type = "memory"} 1000"#, + r#"system_application_free{type = "memory"} 877"#, + r#"system_application_usage{type = "memory"} 12.34"#, + r#"system_memory_size{type = "system"} 1000"#, + r#"system_memory_free{type = "system"} 877"#, + r#"system_memory_usage{type = "system"} 12.34"#, + r#"system_memory_size{type = "swap"} 1000"#, + r#"system_memory_free{type = "swap"} 877"#, + r#"system_memory_usage{type = "swap"} 12.34"#, + r#"system_load_average_1 1.2"#, + r#"system_load_average_5 2.3"#, + r#"system_load_average_15 3.4"#, + r#"system_host_os_version{path = "os-version"} 1"#, + r#"system_host_kernel_version{path = "kernel-version"} 1"#, + r#"system_host_uptime 123456"#, + r#"system_disk_size{path = "disk1"} 1000"#, + r#"system_disk_free{path = "disk1"} 877"#, + r#"system_disk_usage{path = "disk1"} 12.34"#, + r#"system_cpu_physical_core_count 1"#, + r#"system_cpu_count 1"#, + r#"system_cpu_frequency{id = "1"} 12345"#, + r#"system_cpu_usage{id = "1"} 12.34"#, + ] + ) + } +} diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 846fd568..cad5e612 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" anyhow = { workspace = true } fuel-core-types = { workspace = true } fuel-streams = { workspace = true, features = ["test-helpers"] } +fuel-streams-ws = { workspace = true } futures = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } @@ -42,3 +43,7 @@ path = "logs.rs" [[example]] name = "multiple-streams" path = "multiple-streams.rs" + +[[example]] +name = "websockets" +path = "websockets.rs" diff --git a/examples/websockets.rs b/examples/websockets.rs new file mode 100755 index 00000000..c05c8206 --- /dev/null +++ b/examples/websockets.rs @@ -0,0 +1,55 @@ +// Copyright 2024 Fuel Labs +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use fuel_streams::{ + blocks::BlocksSubject, + subjects::SubjectBuildable, + types::FuelNetwork, +}; +use fuel_streams_ws::{ + client::WebSocketClient, + server::ws::models::DeliverPolicy, +}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let mut client = + WebSocketClient::new(FuelNetwork::Local, "admin", "admin").await?; + + client.connect().await?; + + let subject = BlocksSubject::new(); + let deliver_policy = DeliverPolicy::New; + // .with_producer(Some(Address::zeroed())) + // .with_height(Some(183603.into())); + + println!("Subscribing to subject {:?} ...", subject); + client.subscribe(subject.clone(), deliver_policy).await?; + + let mut receiver = client.listen().await?; + + tokio::spawn(async move { + while let Some(_message) = receiver.recv().await { + // println!("Received: {:?}", message); + } + }); + + tokio::time::sleep(Duration::from_secs(15)).await; + + println!("Unsubscribing to subject {:?} ...", subject); + client.unsubscribe(subject, deliver_policy).await?; + + Ok(()) +} diff --git a/knope.toml b/knope.toml index 0f603216..6f7f5d2d 100644 --- a/knope.toml +++ b/knope.toml @@ -1,3 +1,6 @@ +# ------------------------------------------------------------ +# Fuel-streams-publisher package +# ------------------------------------------------------------ [packages.fuel-streams] versioned_files = ["crates/fuel-streams/Cargo.toml"] changelog = "CHANGELOG.md" @@ -34,6 +37,45 @@ path = "artifacts/fuel-streams-publisher-macOS-aarch64.tar.gz" [[packages.fuel-streams.assets]] path = "artifacts/fuel-streams-publisher-macOS-x86_64.tar.gz" +# ------------------------------------------------------------ +# Fuel-streams-ws package +# ------------------------------------------------------------ +[packages.fuel-streams-ws] +versioned_files = ["crates/fuel-streams-ws/Cargo.toml"] +changelog = "CHANGELOG.md" +extra_changelog_sections = [ + { types = [ + "major", + ], name = "⚠️ Breaking Change" }, + { types = [ + "minor", + ], name = "🚀 Features" }, + { types = [ + "patch", + ], name = "🐛 Fixes" }, + { footers = [ + "Changelog-Note", + ], name = "📝 Notes" }, +] + +[[packages.fuel-streams-ws.assets]] +path = "artifacts/fuel-streams-ws-Linux-aarch64-gnu.tar.gz" + +[[packages.fuel-streams-ws.assets]] +path = "artifacts/fuel-streams-ws-Linux-aarch64-musl.tar.gz" + +[[packages.fuel-streams-ws.assets]] +path = "artifacts/fuel-streams-ws-Linux-x86_64-gnu.tar.gz" + +[[packages.fuel-streams-ws.assets]] +path = "artifacts/fuel-streams-ws-Linux-x86_64-musl.tar.gz" + +[[packages.fuel-streams-ws.assets]] +path = "artifacts/fuel-streams-ws-macOS-aarch64.tar.gz" + +[[packages.fuel-streams-ws.assets]] +path = "artifacts/fuel-streams-ws-macOS-x86_64.tar.gz" + # ------------------------------------------------------------ # Workflow to get the current version # ------------------------------------------------------------ diff --git a/scripts/run_streamer.sh b/scripts/run_streamer.sh new file mode 100755 index 00000000..656ae57a --- /dev/null +++ b/scripts/run_streamer.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status +set -e + +# ------------------------------ +# Function to Display Usage +# ------------------------------ +usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " --mode : Specify the run mode (dev|profiling)" + echo " --config-path : Specify the toml config path" + echo " Default: config.toml" + echo " --extra-args : Optional additional arguments to append (in quotes)" + echo "" + echo "Examples:" + echo " $0 # Runs with all defaults" + echo " $0 --config-path # Runs with default config.toml" + echo " $0 --mode dev # Runs with dev mode" + echo " $0 --config-path ../config.toml --mode dev # Custom config toml path and mode" + exit 1 +} + +while [[ "$#" -gt 0 ]]; do + case $1 in + --mode) + MODE="$2" + shift 2 + ;; + --config-path) + CONFIG_PATH="$2" + shift 2 + ;; + --extra-args) + EXTRA_ARGS="$2" + shift 2 + ;; + --help) + usage + ;; + *) + echo "Error: Unknown parameter passed: $1" >&2 + usage + ;; + esac +done + +# ------------------------------ +# Load Environment +# ------------------------------ +source ./scripts/set_env.sh + +# Print the configuration being used +echo -e "\n==========================================" +echo "⚙️ Configuration" +echo -e "==========================================" + +# Runtime Configuration +echo "Runtime Settings:" +echo "→ Mode: $MODE" +if [ -n "$CONFIG_PATH" ]; then + echo "→ Config path: $CONFIG_PATH" +fi +if [ -n "$EXTRA_ARGS" ]; then + echo "→ Extra Arguments: $EXTRA_ARGS" +fi + +# Environment Variables +echo -e "\nEnvironment Variables:" +echo " → Use Metrics: ${USE_METRICS}" +echo " → Use Elastic Logging: $USE_ELASTIC_LOGGING" +echo " → AWS S3 Enabled: $AWS_S3_ENABLED" +echo " → AWS Access Key Id: $AWS_ACCESS_KEY_ID" +echo " → AWS Secret Access Key: $AWS_SECRET_ACCESS_KEY" +echo " → AWS Region: $AWS_REGION" +echo " → AWS Bucket: $AWS_S3_BUCKET_NAME" +echo " → AWS Endpoint: $AWS_ENDPOINT_URL" +echo " → Jwt Auth Secret: $JWT_AUTH_SECRET" +echo " → Nats Url: $NATS_URL" +echo -e "==========================================\n" + +# Define common arguments +COMMON_ARGS=( + "--config-path" "${CONFIG_PATH}" +) + +# Execute based on mode +if [ "$MODE" == "dev" ]; then + cargo run -p fuel-streams-ws -- "${COMMON_ARGS[@]}" ${EXTRA_ARGS} +else + cargo build --profile profiling --package fuel-streams-ws + samply record ./target/profiling/fuel-streams-ws "${COMMON_ARGS[@]}" ${EXTRA_ARGS} +fi diff --git a/scripts/set_env.sh b/scripts/set_env.sh index b2f56cdf..da21ea07 100755 --- a/scripts/set_env.sh +++ b/scripts/set_env.sh @@ -55,8 +55,16 @@ export SYNC_HEADER_BATCH_SIZE=$(eval echo "\$${NETWORK_UPPER}_SYNC_HEADER_BATCH_ export RELAYER_LOG_PAGE_SIZE=$(eval echo "\$${NETWORK_UPPER}_RELAYER_LOG_PAGE_SIZE") export CHAIN_CONFIG=$NETWORK export NETWORK=$NETWORK -export USE_PUBLISHER_METRICS="$(echo "$USE_PUBLISHER_METRICS")" +export USE_METRICS="$(echo "$USE_METRICS")" export USE_ELASTIC_LOGGING="$(echo "$USE_ELASTIC_LOGGING")" +export AWS_S3_ENABLED="$(echo "$AWS_S3_ENABLED")" +export AWS_ACCESS_KEY_ID="$(echo "$AWS_ACCESS_KEY_ID")" +export AWS_SECRET_ACCESS_KEY="$(echo "$AWS_SECRET_ACCESS_KEY")" +export AWS_REGION="$(echo "$AWS_REGION")" +export AWS_ENDPOINT_URL="$(echo "$AWS_ENDPOINT_URL")" +export AWS_S3_BUCKET_NAME="$(echo "$AWS_S3_BUCKET_NAME")" +export JWT_AUTH_SECRET="$(echo "$JWT_AUTH_SECRET")" +export NATS_URL="$(echo "$NATS_URL")" # Append network-specific variables to .env file { @@ -71,6 +79,14 @@ export USE_ELASTIC_LOGGING="$(echo "$USE_ELASTIC_LOGGING")" echo "SYNC_HEADER_BATCH_SIZE=$SYNC_HEADER_BATCH_SIZE" echo "RELAYER_LOG_PAGE_SIZE=$RELAYER_LOG_PAGE_SIZE" echo "CHAIN_CONFIG=$CHAIN_CONFIG" - echo "USE_PUBLISHER_METRICS=$USE_PUBLISHER_METRICS" + echo "USE_METRICS=$USE_METRICS" echo "USE_ELASTIC_LOGGING=$USE_ELASTIC_LOGGING" + echo "AWS_S3_ENABLED=$AWS_S3_ENABLED" + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" + echo "AWS_REGION=$AWS_REGION" + echo "AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL" + echo "AWS_S3_BUCKET_NAME=$AWS_S3_BUCKET_NAME" + echo "JWT_AUTH_SECRET=$JWT_AUTH_SECRET" + echo "NATS_URL=$NATS_URL" } >> .env diff --git a/tests/tests/publisher.rs b/tests/tests/publisher.rs index 52e0709c..ecd8bcf6 100644 --- a/tests/tests/publisher.rs +++ b/tests/tests/publisher.rs @@ -9,6 +9,7 @@ use fuel_core_types::blockchain::SealedBlock; use fuel_streams_core::prelude::*; use fuel_streams_publisher::{ publisher::shutdown::ShutdownController, + shutdown::get_controller_and_token, FuelCoreLike, Publisher, }; @@ -291,9 +292,8 @@ async fn publish_block( stop_publisher(shutdown_controller).await; } -async fn start_publisher(publisher: &Publisher) -> Arc { - let shutdown_controller = ShutdownController::new().arc(); - let shutdown_token = shutdown_controller.get_token(); +async fn start_publisher(publisher: &Publisher) -> ShutdownController { + let (shutdown_controller, shutdown_token) = get_controller_and_token(); tokio::spawn({ let publisher = publisher.clone(); async move { @@ -303,7 +303,7 @@ async fn start_publisher(publisher: &Publisher) -> Arc { wait_for_publisher_to_start().await; shutdown_controller } -async fn stop_publisher(shutdown_controller: Arc) { +async fn stop_publisher(shutdown_controller: ShutdownController) { wait_for_publisher_to_process_block().await; assert!(shutdown_controller.initiate_shutdown().is_ok()); From 71119fa6109633627df1c760d196889a365d820a Mon Sep 17 00:00:00 2001 From: 0xterminator Date: Wed, 18 Dec 2024 17:01:57 +0200 Subject: [PATCH 07/15] feat(repo): Added webserver service and added docker build step --- .github/workflows/docker_publish.yaml | 11 +++++++- cluster/charts/fuel-streams/Chart.yaml | 2 +- .../templates/webserver/deployment.yaml | 26 +++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index 74f5167e..43de8f3f 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -32,7 +32,7 @@ jobs: id: sha run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Build and push Docker (${{ steps.sha.outputs.short_sha }}) + - name: Build and push Docker (${{ steps.sha.outputs.short_sha }}) for publisher uses: ./.github/actions/docker-publish id: publish-fuel-streams-nats with: @@ -40,3 +40,12 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} image: ghcr.io/fuellabs/fuel-streams-publisher dockerfile: cluster/docker/fuel-streams-publisher.Dockerfile + + - name: Build and push Docker (${{ steps.sha.outputs.short_sha }}) for webserver + uses: ./.github/actions/docker-publish + id: publish-fuel-webserver-nats + with: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + image: ghcr.io/fuellabs/fuel-streams-ws + dockerfile: cluster/docker/fuel-streams-ws.Dockerfile diff --git a/cluster/charts/fuel-streams/Chart.yaml b/cluster/charts/fuel-streams/Chart.yaml index 588a1013..398418cc 100755 --- a/cluster/charts/fuel-streams/Chart.yaml +++ b/cluster/charts/fuel-streams/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: "1.0" description: A Helm chart for Kubernetes name: fuel-streams -version: 0.1.2 +version: 0.1.3 dependencies: - name: nats version: 1.2.6 diff --git a/cluster/charts/fuel-streams/templates/webserver/deployment.yaml b/cluster/charts/fuel-streams/templates/webserver/deployment.yaml index 759fc13a..73935fac 100644 --- a/cluster/charts/fuel-streams/templates/webserver/deployment.yaml +++ b/cluster/charts/fuel-streams/templates/webserver/deployment.yaml @@ -70,3 +70,29 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + {{- include "k8s.metadata" (dict "context" . "suffix" "-webserver") | nindent 2 }} + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + {{- include "set-value" (dict "context" $webserver "path" "config.annotations") | nindent 4 }} + labels: + {{- include "fuel-streams.labels" . | nindent 4 }} + {{- include "set-value" (dict "context" $webserver "path" "config.labels") | nindent 4 }} + app.kubernetes.io/component: webserver + app.kubernetes.io/service: external-service + +spec: + type: LoadBalancer + ports: + - name: webserver + port: {{ $webserver.port }} + targetPort: {{ $webserver.port }} + protocol: TCP + selector: + {{- include "fuel-streams.selectorLabels" . | nindent 4 }} + {{- include "set-value" (dict "context" $webserver "path" "config.selectorLabels") | nindent 4 }} + app.kubernetes.io/component: webserver + app.kubernetes.io/service: external-service From 95f388ca965466407f03bac23733fed143334fad Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Thu, 19 Dec 2024 19:46:31 -0300 Subject: [PATCH 08/15] fix(repo): use cli instead of config.toml --- .github/workflows/docker_publish.yaml | 18 ++- crates/fuel-streams-ws/config.toml | 14 --- crates/fuel-streams-ws/src/cli.rs | 63 ++++++++-- crates/fuel-streams-ws/src/config.rs | 161 ++++---------------------- crates/fuel-streams-ws/src/lib.rs | 4 +- crates/fuel-streams-ws/src/main.rs | 33 +----- 6 files changed, 102 insertions(+), 191 deletions(-) delete mode 100644 crates/fuel-streams-ws/config.toml diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index 43de8f3f..748ee112 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -2,6 +2,16 @@ name: Build and publish Docker image on: workflow_dispatch: + inputs: + image_type: + description: "Choose which image to build (publisher/webserver/both)" + required: true + type: choice + options: + - publisher + - webserver + - both + default: "both" push: branches: - main @@ -32,7 +42,9 @@ jobs: id: sha run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Build and push Docker (${{ steps.sha.outputs.short_sha }}) for publisher + - name: Build and push Docker for publisher + if: github.event.inputs.image_type == 'publisher' || github.event.inputs.image_type == 'both' + || github.event_name != 'workflow_dispatch' uses: ./.github/actions/docker-publish id: publish-fuel-streams-nats with: @@ -41,7 +53,9 @@ jobs: image: ghcr.io/fuellabs/fuel-streams-publisher dockerfile: cluster/docker/fuel-streams-publisher.Dockerfile - - name: Build and push Docker (${{ steps.sha.outputs.short_sha }}) for webserver + - name: Build and push Docker for webserver + if: github.event.inputs.image_type == 'webserver' || github.event.inputs.image_type == 'both' + || github.event_name != 'workflow_dispatch' uses: ./.github/actions/docker-publish id: publish-fuel-webserver-nats with: diff --git a/crates/fuel-streams-ws/config.toml b/crates/fuel-streams-ws/config.toml deleted file mode 100644 index 1c71a7a4..00000000 --- a/crates/fuel-streams-ws/config.toml +++ /dev/null @@ -1,14 +0,0 @@ -[api] -port = 9003 - -[nats] -network = "local" - -[fuel] -network = "local" - -[s3] -enabled = true - -[auth] -jwt-secret = "secret" diff --git a/crates/fuel-streams-ws/src/cli.rs b/crates/fuel-streams-ws/src/cli.rs index b27bb40a..29c5c25f 100644 --- a/crates/fuel-streams-ws/src/cli.rs +++ b/crates/fuel-streams-ws/src/cli.rs @@ -1,17 +1,64 @@ use clap::Parser; /// CLI structure for parsing command-line arguments. -/// -/// - `config_path`: Path to the toml config file. #[derive(Clone, Parser)] pub struct Cli { - /// Config path + /// API port number #[arg( long, - value_name = "CONFIG", - env = "CONFIG_PATH", - default_value = "config.toml", - help = "Path to toml config file" + value_name = "PORT", + env = "API_PORT", + default_value = "9003", + help = "Port number for the API server" )] - pub config_path: Option, + pub api_port: u16, + + /// NATS URL + #[arg( + long, + value_name = "NATS_URL", + env = "NATS_URL", + default_value = "nats://localhost:4222", + help = "NATS URL" + )] + pub nats_url: String, + + /// Fuel network configuration + #[arg( + long, + value_name = "NETWORK", + env = "NETWORK", + default_value = "local", + help = "Fuel network configuration (local, etc.)" + )] + pub network: String, + + /// Enable S3 + #[arg( + long, + value_name = "AWS_S3_ENABLED", + env = "AWS_S3_ENABLED", + default_value = "true", + help = "Enable S3 integration" + )] + pub s3_enabled: bool, + + /// JWT secret + #[arg( + long, + value_name = "JWT_AUTH_SECRET", + env = "JWT_AUTH_SECRET", + default_value = "secret", + help = "Secret key for JWT authentication" + )] + pub jwt_secret: String, + + /// Use metrics + #[arg( + long, + env = "USE_METRICS", + default_value = "false", + help = "Enable metrics" + )] + pub use_metrics: bool, } diff --git a/crates/fuel-streams-ws/src/config.rs b/crates/fuel-streams-ws/src/config.rs index e90e294c..910fa2eb 100644 --- a/crates/fuel-streams-ws/src/config.rs +++ b/crates/fuel-streams-ws/src/config.rs @@ -1,79 +1,49 @@ -use std::{ - num::ParseIntError, - path::{Path, PathBuf}, - str::{FromStr, ParseBoolError}, - time::Duration, -}; +use std::{path::PathBuf, str::FromStr}; -use confy::ConfyError; +use clap::Parser; use displaydoc::Display as DisplayDoc; use fuel_streams::types::FuelNetwork; -use serde::{Deserialize, Deserializer}; use thiserror::Error; -use tokio::{fs::File, io::AsyncReadExt}; #[derive(Debug, DisplayDoc, Error)] pub enum Error { - /// Open config file: {0} - OpenConfig(std::io::Error), - /// Failed to parse config: {0} - ParseConfig(toml::de::Error), - /// Failed to parse config as utf-8: {0} - ParseUtf8(std::string::FromUtf8Error), - /// Failed to read config file: {0} - ReadConfig(std::io::Error), - /// Failed to read config metadata: {0} - ReadMeta(std::io::Error), - /// Failed to read env config: {0} - Confy(ConfyError), /// Undecodable config element: {0} UndecodableConfigElement(&'static str), - /// Parse int error: {0} - ParseInt(ParseIntError), - /// Parse bool error: {0} - ParseBool(ParseBoolError), } -#[derive(Debug, Default, Deserialize, Clone)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[derive(Debug, Default, Clone)] pub struct S3Config { pub enabled: bool, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[derive(Clone, Debug)] pub struct TlsConfig { pub private_key: PathBuf, pub certificate: PathBuf, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[derive(Clone, Debug)] pub struct ApiConfig { pub port: u16, pub tls: Option, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[derive(Clone, Debug)] pub struct AuthConfig { pub jwt_secret: String, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[derive(Clone, Debug)] pub struct FuelConfig { pub network: FuelNetwork, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[derive(Clone, Debug)] pub struct NatsConfig { pub network: FuelNetwork, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] +#[derive(Clone, Debug)] pub struct Config { pub api: ApiConfig, pub auth: AuthConfig, @@ -82,112 +52,31 @@ pub struct Config { pub fuel: FuelConfig, } -impl Default for Config { - fn default() -> Self { - Config { +impl Config { + pub fn load() -> Result { + let cli = crate::cli::Cli::parse(); + Self::from_cli(&cli) + } + + fn from_cli(cli: &crate::cli::Cli) -> Result { + Ok(Config { api: ApiConfig { - port: 9003, + port: cli.api_port, tls: None, }, auth: AuthConfig { - jwt_secret: String::new(), + jwt_secret: cli.jwt_secret.clone(), }, nats: NatsConfig { network: FuelNetwork::Local, }, - s3: S3Config { enabled: false }, + s3: S3Config { + enabled: cli.s3_enabled, + }, fuel: FuelConfig { - network: FuelNetwork::Local, + network: FuelNetwork::from_str(&cli.network) + .map_err(|_| Error::UndecodableConfigElement("NETWORK"))?, }, - } - } -} - -#[allow(dead_code)] -fn deserialize_duration_from_usize<'de, D>( - deserializer: D, -) -> Result -where - D: Deserializer<'de>, -{ - let seconds = u64::deserialize(deserializer)?; - Ok(Duration::from_secs(seconds)) -} - -#[allow(dead_code)] -fn deserialize_duration_option<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let seconds: Option = Option::deserialize(deserializer)?; - if seconds.is_none() { - return Ok(None); - } - Ok(seconds.map(Duration::from_secs)) -} - -impl Config { - pub async fn from_path( - path: impl AsRef + Send, - ) -> Result { - read_to_string(path).await?.parse() + }) } - - pub fn from_envs() -> Result { - let mut config = Self::default(); - - // ----------------------API-------------------------------- - if let Ok(app_port) = dotenvy::var("STREAMER_API_PORT") { - config.api.port = - app_port.parse::().map_err(Error::ParseInt)?; - } - - // ----------------------NATS-------------------------------- - if let Ok(nats_network) = dotenvy::var("NETWORK") { - config.nats.network = FuelNetwork::from_str(&nats_network) - .map_err(|_| Error::UndecodableConfigElement("NETWORK"))?; - } - - // ----------------------S3-------------------------------- - if let Ok(s3_enabled) = dotenvy::var("AWS_S3_ENABLED") { - config.s3.enabled = - s3_enabled.parse::().map_err(Error::ParseBool)?; - } - - // ----------------------AUTH-------------------------------- - if let Ok(jwt_secret) = dotenvy::var("JWT_AUTH_SECRET") { - config.auth.jwt_secret = jwt_secret; - } - - // ----------------------FUEL-------------------------------- - if let Ok(network) = dotenvy::var("NETWORK") { - config.fuel.network = FuelNetwork::from_str(&network) - .map_err(|_| Error::UndecodableConfigElement("NETWORK"))?; - } - - Ok(config) - } -} - -impl FromStr for Config { - type Err = Error; - - fn from_str(s: &str) -> Result { - toml::from_str(s).map_err(Error::ParseConfig) - } -} - -async fn read_to_string( - path: impl AsRef + Send, -) -> Result { - let mut file = File::open(path).await.map_err(Error::OpenConfig)?; - let meta = file.metadata().await.map_err(Error::ReadMeta)?; - let mut contents = - Vec::with_capacity(usize::try_from(meta.len()).unwrap_or(0)); - file.read_to_end(&mut contents) - .await - .map_err(Error::ReadConfig)?; - String::from_utf8(contents).map_err(Error::ParseUtf8) } diff --git a/crates/fuel-streams-ws/src/lib.rs b/crates/fuel-streams-ws/src/lib.rs index d60b1e6c..e9c80f91 100644 --- a/crates/fuel-streams-ws/src/lib.rs +++ b/crates/fuel-streams-ws/src/lib.rs @@ -4,13 +4,13 @@ pub mod config; pub mod server; pub mod telemetry; -use std::{env, sync::LazyLock}; +use std::sync::LazyLock; pub static STREAMER_MAX_WORKERS: LazyLock = LazyLock::new(|| { let available_cpus = num_cpus::get(); let default_threads = 2 * available_cpus; - env::var("STREAMER_MAX_WORKERS") + dotenvy::var("STREAMER_MAX_WORKERS") .ok() .and_then(|val| val.parse().ok()) .unwrap_or(default_threads) diff --git a/crates/fuel-streams-ws/src/main.rs b/crates/fuel-streams-ws/src/main.rs index f8d6fe90..e629bd31 100644 --- a/crates/fuel-streams-ws/src/main.rs +++ b/crates/fuel-streams-ws/src/main.rs @@ -1,7 +1,4 @@ -use anyhow::Context as _; -use clap::Parser; use fuel_streams_ws::{ - cli::Cli, config::Config, server::{api::create_api, context::Context, state::ServerState}, }; @@ -20,36 +17,14 @@ async fn main() -> anyhow::Result<()> { .with_span_events(FmtSpan::CLOSE) .init(); - // load envs - dotenvy::dotenv().context("Failed to env values")?; + if let Err(err) = dotenvy::dotenv() { + tracing::error!("File .env not found: {:?}", err); + } - // read cli args - let cli = Cli::parse(); - - // load config - let config = match cli.config_path { - Some(path) => { - tracing::info!("Using config file: {}", path); - Config::from_path(path) - .await - .context("Failed to load toml config")? - } - None => { - tracing::info!("Using envs to load config"); - Config::from_envs().context("Failed to load toml config")? - } - }; - - // init context + let config = Config::load()?; let context = Context::new(&config).await?; - - // init server shared state let state = ServerState::new(context).await; - - // create the actix webserver let server = create_api(&config, state)?; - - // get server handle let server_handle = server.handle(); // spawn the server in the background From deeeffb8ef1febc9cfcdf40d0e55f95244c77176 Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Thu, 19 Dec 2024 20:48:01 -0300 Subject: [PATCH 09/15] fix(ws): remove duplicated nats clients --- .github/workflows/ci.yaml | 5 ++++- crates/fuel-streams-ws/src/cli.rs | 10 ---------- crates/fuel-streams-ws/src/config.rs | 12 +----------- crates/fuel-streams-ws/src/server/context.rs | 12 ++---------- 4 files changed, 7 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cb25b762..c6b9ac26 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,10 +49,13 @@ jobs: deps release core - publisher data-parser + networks fuel-streams macros + publisher + storage + ws lockfile: name: Validate Lockfile diff --git a/crates/fuel-streams-ws/src/cli.rs b/crates/fuel-streams-ws/src/cli.rs index 29c5c25f..4aed7400 100644 --- a/crates/fuel-streams-ws/src/cli.rs +++ b/crates/fuel-streams-ws/src/cli.rs @@ -23,16 +23,6 @@ pub struct Cli { )] pub nats_url: String, - /// Fuel network configuration - #[arg( - long, - value_name = "NETWORK", - env = "NETWORK", - default_value = "local", - help = "Fuel network configuration (local, etc.)" - )] - pub network: String, - /// Enable S3 #[arg( long, diff --git a/crates/fuel-streams-ws/src/config.rs b/crates/fuel-streams-ws/src/config.rs index 910fa2eb..73779f6b 100644 --- a/crates/fuel-streams-ws/src/config.rs +++ b/crates/fuel-streams-ws/src/config.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, str::FromStr}; +use std::path::PathBuf; use clap::Parser; use displaydoc::Display as DisplayDoc; @@ -33,11 +33,6 @@ pub struct AuthConfig { pub jwt_secret: String, } -#[derive(Clone, Debug)] -pub struct FuelConfig { - pub network: FuelNetwork, -} - #[derive(Clone, Debug)] pub struct NatsConfig { pub network: FuelNetwork, @@ -49,7 +44,6 @@ pub struct Config { pub auth: AuthConfig, pub s3: S3Config, pub nats: NatsConfig, - pub fuel: FuelConfig, } impl Config { @@ -73,10 +67,6 @@ impl Config { s3: S3Config { enabled: cli.s3_enabled, }, - fuel: FuelConfig { - network: FuelNetwork::from_str(&cli.network) - .map_err(|_| Error::UndecodableConfigElement("NETWORK"))?, - }, }) } } diff --git a/crates/fuel-streams-ws/src/server/context.rs b/crates/fuel-streams-ws/src/server/context.rs index 59d0166b..3dbf121b 100644 --- a/crates/fuel-streams-ws/src/server/context.rs +++ b/crates/fuel-streams-ws/src/server/context.rs @@ -1,6 +1,5 @@ use std::{sync::Arc, time::Duration}; -use fuel_streams::client::Client; use fuel_streams_core::prelude::*; use fuel_streams_storage::S3Client; @@ -15,7 +14,6 @@ pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(90); #[allow(dead_code)] #[derive(Clone)] pub struct Context { - pub client: Client, pub nats_client: NatsClient, pub fuel_streams: Arc, pub telemetry: Arc, @@ -25,23 +23,19 @@ pub struct Context { impl Context { pub async fn new(config: &Config) -> anyhow::Result { - let nats_client_opts = NatsClientOpts::new(config.fuel.network); + let nats_client_opts = NatsClientOpts::new(config.nats.network); let nats_client = NatsClient::connect(&nats_client_opts).await?; - let s3_client_opts = S3ClientOpts::admin_opts(); let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); - let fuel_streams = Arc::new(FuelStreams::new(&nats_client, &s3_client).await); - - let client = Client::connect(config.fuel.network).await?; let telemetry = Telemetry::new(None).await?; telemetry.start().await?; Ok(Context { fuel_streams, nats_client, - client, + // client, telemetry, s3_client: if config.s3.enabled { Some(s3_client) @@ -57,7 +51,6 @@ impl Context { ) -> anyhow::Result { let nats_client_opts = NatsClientOpts::new(fuel_network); let nats_client = NatsClient::connect(&nats_client_opts).await?; - let client = Client::connect(fuel_network).await?; let s3_client_opts = S3ClientOpts::admin_opts(); let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); Ok(Context { @@ -68,7 +61,6 @@ impl Context { telemetry: Telemetry::new(None).await?, s3_client: None, jwt_secret: String::new(), - client, }) } From cbc1f67fd1cb5a863f3317f08f275c4b62750ed9 Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Thu, 19 Dec 2024 21:02:30 -0300 Subject: [PATCH 10/15] fix(ws): use nats_url instead of FuelNetwork --- .../src/nats/nats_client_opts.rs | 21 +++++++++++++++---- crates/fuel-streams-ws/src/config.rs | 5 ++--- crates/fuel-streams-ws/src/server/context.rs | 3 ++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/fuel-streams-storage/src/nats/nats_client_opts.rs b/crates/fuel-streams-storage/src/nats/nats_client_opts.rs index 1c8cee04..c43833d1 100644 --- a/crates/fuel-streams-storage/src/nats/nats_client_opts.rs +++ b/crates/fuel-streams-storage/src/nats/nats_client_opts.rs @@ -46,6 +46,8 @@ pub struct NatsClientOpts { pub(crate) namespace: NatsNamespace, /// The timeout in seconds for NATS operations. pub(crate) timeout_secs: u64, + /// URL of the NATS server. + pub(crate) url: Option, } impl NatsClientOpts { @@ -55,6 +57,7 @@ impl NatsClientOpts { role: FuelNetworkUserRole::default(), namespace: NatsNamespace::default(), timeout_secs: 5, + url: None, } } @@ -67,11 +70,21 @@ impl NatsClientOpts { Self { role, ..self } } + pub fn with_url(self, url: String) -> Self { + Self { + url: Some(url), + ..self + } + } + pub fn get_url(&self) -> String { - match self.role { - FuelNetworkUserRole::Admin => dotenvy::var("NATS_URL") - .expect("NATS_URL must be set for admin role"), - FuelNetworkUserRole::Default => self.network.to_nats_url(), + match self.url.clone() { + Some(url) => url, + None => match self.role { + FuelNetworkUserRole::Admin => dotenvy::var("NATS_URL") + .expect("NATS_URL must be set for admin role"), + FuelNetworkUserRole::Default => self.network.to_nats_url(), + }, } } diff --git a/crates/fuel-streams-ws/src/config.rs b/crates/fuel-streams-ws/src/config.rs index 73779f6b..eeb01ce0 100644 --- a/crates/fuel-streams-ws/src/config.rs +++ b/crates/fuel-streams-ws/src/config.rs @@ -2,7 +2,6 @@ use std::path::PathBuf; use clap::Parser; use displaydoc::Display as DisplayDoc; -use fuel_streams::types::FuelNetwork; use thiserror::Error; #[derive(Debug, DisplayDoc, Error)] @@ -35,7 +34,7 @@ pub struct AuthConfig { #[derive(Clone, Debug)] pub struct NatsConfig { - pub network: FuelNetwork, + pub url: String, } #[derive(Clone, Debug)] @@ -62,7 +61,7 @@ impl Config { jwt_secret: cli.jwt_secret.clone(), }, nats: NatsConfig { - network: FuelNetwork::Local, + url: cli.nats_url.clone(), }, s3: S3Config { enabled: cli.s3_enabled, diff --git a/crates/fuel-streams-ws/src/server/context.rs b/crates/fuel-streams-ws/src/server/context.rs index 3dbf121b..7bda3220 100644 --- a/crates/fuel-streams-ws/src/server/context.rs +++ b/crates/fuel-streams-ws/src/server/context.rs @@ -23,7 +23,8 @@ pub struct Context { impl Context { pub async fn new(config: &Config) -> anyhow::Result { - let nats_client_opts = NatsClientOpts::new(config.nats.network); + let nats_client_opts = + NatsClientOpts::admin_opts().with_url(config.nats.url.clone()); let nats_client = NatsClient::connect(&nats_client_opts).await?; let s3_client_opts = S3ClientOpts::admin_opts(); let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); From 2e6f7c0de3f1d757cb155b85c55b592a0b44a267 Mon Sep 17 00:00:00 2001 From: 0xterminator Date: Fri, 20 Dec 2024 09:07:07 +0200 Subject: [PATCH 11/15] feat(repo): Removed unused args by infra for ws streamer --- Makefile | 9 ++++----- cluster/charts/fuel-streams/values.yaml | 2 +- cluster/docker/fuel-streams-ws.Dockerfile | 6 +++--- scripts/run_streamer.sh | 20 ++++---------------- 4 files changed, 12 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index b8bdb8af..feaa6c3f 100644 --- a/Makefile +++ b/Makefile @@ -228,20 +228,19 @@ run-publisher-testnet-profiling: run-streamer: check-network @./scripts/run_streamer.sh \ --mode $(MODE) \ - $(if $(CONFIG_PATH),--config-path $(CONFIG_PATH),) \ $(if $(extra_args),--extra-args "$(extra_args)",) run-streamer-mainnet-dev: - $(MAKE) run-streamer NETWORK=mainnet MODE=dev CONFIG_PATH=crates/fuel-streams-ws/config.toml + $(MAKE) run-streamer NETWORK=mainnet MODE=dev run-streamer-mainnet-profiling: - $(MAKE) run-streamer NETWORK=mainnet MODE=profiling CONFIG_PATH=crates/fuel-streams-ws/config.toml + $(MAKE) run-streamer NETWORK=mainnet MODE=profiling run-streamer-testnet-dev: - $(MAKE) run-streamer NETWORK=testnet MODE=dev CONFIG_PATH=crates/fuel-streams-ws/config.toml + $(MAKE) run-streamer NETWORK=testnet MODE=dev run-streamer-testnet-profiling: - $(MAKE) run-streamer NETWORK=testnet MODE=profiling CONFIG_PATH=crates/fuel-streams-ws/config.toml + $(MAKE) run-streamer NETWORK=testnet MODE=profiling # ------------------------------------------------------------ # Docker Compose diff --git a/cluster/charts/fuel-streams/values.yaml b/cluster/charts/fuel-streams/values.yaml index a148f4c9..912e3f0e 100755 --- a/cluster/charts/fuel-streams/values.yaml +++ b/cluster/charts/fuel-streams/values.yaml @@ -229,7 +229,7 @@ webserver: env: STREAMER_MAX_WORKERS: "10" - STREAMER_API_PORT: 9003 + API_PORT: 9003 JWT_AUTH_SECRET: "secret" USE_ELASTIC_LOGGING: false USE_METRICS: true diff --git a/cluster/docker/fuel-streams-ws.Dockerfile b/cluster/docker/fuel-streams-ws.Dockerfile index 435eb1f8..5e4a7a8a 100644 --- a/cluster/docker/fuel-streams-ws.Dockerfile +++ b/cluster/docker/fuel-streams-ws.Dockerfile @@ -56,9 +56,9 @@ RUN \ # Stage 2: Run FROM ubuntu:22.04 AS run -ARG STREAMER_API_PORT=9003 +ARG API_PORT=9003 -ENV STREAMER_API_PORT=$STREAMER_API_PORT +ENV API_PORT=$API_PORT ENV NATS_URL= ENV NETWORK= ENV USE_METRICS= @@ -83,7 +83,7 @@ RUN apt-get update -y \ COPY --from=builder /root/fuel-streams-ws . COPY --from=builder /root/fuel-streams-ws.d . -EXPOSE ${STREAMER_API_PORT} +EXPOSE ${API_PORT} # https://stackoverflow.com/a/44671685 # https://stackoverflow.com/a/40454758 diff --git a/scripts/run_streamer.sh b/scripts/run_streamer.sh index 656ae57a..c2a109a6 100755 --- a/scripts/run_streamer.sh +++ b/scripts/run_streamer.sh @@ -10,15 +10,12 @@ usage() { echo "Usage: $0 [options]" echo "Options:" echo " --mode : Specify the run mode (dev|profiling)" - echo " --config-path : Specify the toml config path" - echo " Default: config.toml" echo " --extra-args : Optional additional arguments to append (in quotes)" echo "" echo "Examples:" - echo " $0 # Runs with all defaults" - echo " $0 --config-path # Runs with default config.toml" - echo " $0 --mode dev # Runs with dev mode" - echo " $0 --config-path ../config.toml --mode dev # Custom config toml path and mode" + echo " $0 # Runs with all defaults" + echo " $0 --mode dev # Runs with dev mode" + echo " $0 --mode dev # Custom config toml path and mode" exit 1 } @@ -28,10 +25,6 @@ while [[ "$#" -gt 0 ]]; do MODE="$2" shift 2 ;; - --config-path) - CONFIG_PATH="$2" - shift 2 - ;; --extra-args) EXTRA_ARGS="$2" shift 2 @@ -59,9 +52,6 @@ echo -e "==========================================" # Runtime Configuration echo "Runtime Settings:" echo "→ Mode: $MODE" -if [ -n "$CONFIG_PATH" ]; then - echo "→ Config path: $CONFIG_PATH" -fi if [ -n "$EXTRA_ARGS" ]; then echo "→ Extra Arguments: $EXTRA_ARGS" fi @@ -81,9 +71,7 @@ echo " → Nats Url: $NATS_URL" echo -e "==========================================\n" # Define common arguments -COMMON_ARGS=( - "--config-path" "${CONFIG_PATH}" -) +COMMON_ARGS=() # Execute based on mode if [ "$MODE" == "dev" ]; then From b5bfbbfcb46bfb4e19bb7f65025cf729f7bfd52f Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Sun, 22 Dec 2024 21:35:31 -0300 Subject: [PATCH 12/15] feat: Adjusts to release S3 + WebServer (#361) * fix(repo): Solving publisher slowness (#352) * feat(repo): add sv-emitter package * build(repo): adjust run-publisher * feat(repo): first version of consumer * feat(repo): add the rest of payloads * refactor(repo): reuse executors * fix(repo): tests * build(repo): configure devops to emitter/consumer * build(repo): update chart version * fix(repo): consumer cli argument * build(repo): docker action * build(repo): fix consumer chart * build(repo): fix consumer chart * build(repo): fix consumer chart * build(repo): chart * fix(repo): general stuff * build(repo): docker action * ci(repo): fix * fix(repo): chain config * fix(repo): docker images * feat(repo): improve docker build * refactor(repo): a lot of improvements * fix(repo): fix helm chart nats config * feat(repo): worked using mirror * fix(repo): adjust nats cluster config * build(repo): adjust github action * build(repo): update chart version * build(repo): fix docker images * build(repo): lint warnings * ci(repo): adjust docker action * build(repo): fix docker file * fix(repo): stream creation * build(repo): fix tls for nats * build(repo): fix chart * build(repo): fix chart * build(repo): fix chart * build(repo): fix chart * fix(repo): use sts instead of deployment for consumer * fix(repo): fix chart * fix(repo): remove initcontainer * fix(repo): update chart * fix(repo): fix resources on consumer * fix(repo): bump chart * fix(repo): bump chart * fix(repo): bump chart * build(repo): fix makefile * build(repo): add publisher into one single Dockerfile * ci(repo): fix docker publish action * build(repo): fix build docker * ci(repo): fix docker action * ci(repo): fix tests * ci(repo): fix ci.yaml * build(repo): adjust fuel dependencies features * build(repo): Add DNS/TLS configuration for Nats (#357) * build(repo): Add DNS/TLS configuration for Nats * build(repo): bump chart version * build(repo): fix chart * build(repo): fix chart * build(repo): fix secretName on nats client websocket config * build(repo): Update NATS dependency on chart (#358) * build(repo): Update NATS dependency on chart * build(repo): Bump chart version * refactor(repo): remove FuelNetwork from nats * refactor(repo): remove unused benches * refactor(repo): general improvements * fix(repo): general fixes * build(repo): adjust chart for new architecture * build(repo): adjust chart * build(repo): fix chart * build(repo): fix chart * build(repo): fix chart * build(repo): fix chart * build(repo): docker image * build(repo): bump chart * build(repo): fix emitter Dockerimage * build(repo): adjust workflow dispatch for docker * build(repo): docker action * build(repo): fix publisher config * build(repo): bump chart * build(repo): general fixes and adjustments * build(repo): bump chart * fix(repo): s3 client workaround * refactor(repo): rename crates to be consistent * build(repo): bump chart * build(repo): some adjusts * fix(consumer): remove max_messages * refactor(repo): change s3 path * build(repo): bump chart * refactor(repo): perf improvements * refactor(webserver): send decoded payload direct on websocket --- .editorconfig | 4 + .env.sample | 34 +- .github/actions/setup-rust/action.yaml | 12 +- .github/workflows/ci.yaml | 130 +-- .github/workflows/docker_publish.yaml | 53 +- .github/workflows/helm_publish.yaml | 97 +-- .github/workflows/publish_release.yaml | 12 +- .gitignore | 4 +- .prettierignore | 3 +- .rustfmt.toml | 5 + .typos.toml | 4 +- Cargo.lock | 787 ++++++------------ Cargo.toml | 27 +- Makefile | 110 ++- Tiltfile | 108 ++- benches/bench-consumers/Cargo.toml | 30 - benches/bench-consumers/README.md | 13 - benches/bench-consumers/benches/consumers.rs | 43 - benches/bench-consumers/src/lib.rs | 1 - benches/bench-consumers/src/main.rs | 10 - .../src/runners/benchmark_results.rs | 103 --- benches/bench-consumers/src/runners/mod.rs | 5 - .../bench-consumers/src/runners/runner_all.rs | 25 - .../src/runners/runner_consumer.rs | 55 -- .../src/runners/runner_kv_watcher.rs | 40 - .../src/runners/runner_subscription.rs | 35 - benches/load-tester/Cargo.toml | 22 - benches/load-tester/README.md | 9 - benches/load-tester/src/lib.rs | 1 - benches/load-tester/src/main.rs | 16 - benches/load-tester/src/runners/cli.rs | 33 - benches/load-tester/src/runners/mod.rs | 4 - benches/load-tester/src/runners/results.rs | 118 --- benches/load-tester/src/runners/runner_all.rs | 247 ------ .../src/runners/runner_streamable.rs | 38 - benches/nats-publisher/README.md | 18 - benches/nats-publisher/config/nats.conf | 5 - benches/nats-publisher/src/lib.rs | 2 - benches/nats-publisher/src/main.rs | 78 -- benches/nats-publisher/src/utils/blocks.rs | 93 --- benches/nats-publisher/src/utils/mod.rs | 3 - benches/nats-publisher/src/utils/nats.rs | 130 --- benches/nats-publisher/src/utils/tx.rs | 129 --- cluster/README.md | 68 +- .../charts/fuel-streams-publisher/.helmignore | 23 - .../charts/fuel-streams-publisher/Chart.yaml | 21 - .../templates/_helpers.tpl | 62 -- .../fuel-streams-publisher/templates/hpa.yaml | 32 - .../templates/service.yaml | 17 - .../templates/serviceaccount.yaml | 13 - .../templates/statefulset.yaml | 147 ---- .../charts/fuel-streams-publisher/values.yaml | 124 --- cluster/charts/fuel-streams/Chart.lock | 10 +- cluster/charts/fuel-streams/Chart.yaml | 11 +- .../charts/fuel-streams/templates/_blocks.tpl | 54 +- .../fuel-streams/templates/_helpers.tpl | 61 +- .../charts/fuel-streams/templates/_hpa.yaml | 55 ++ .../fuel-streams/templates/certificate.yaml | 52 -- .../fuel-streams/templates/common-config.yaml | 21 + .../templates/consumer/statefulset.yaml | 84 ++ .../templates/external-service.yaml | 31 - .../publisher/network-configmap.yaml | 31 - .../templates/publisher/statefulset.yaml | 96 ++- .../templates/secret-creator.yaml | 4 +- .../templates/service-account.yaml | 2 +- .../templates/webserver/deployment.yaml | 67 +- .../templates/webserver/service.yaml | 37 + .../fuel-streams/tests/certificate_test.yaml | 2 +- .../tests/consumer/deployment_test.yaml | 305 +++++++ .../tests/external_service_test.yaml | 2 +- .../tests/publisher/network-configmap.yaml | 87 -- .../tests/publisher/statefulset.yaml | 14 +- .../tests/webserver/deployment_test.yaml | 12 +- cluster/charts/fuel-streams/values-local.yaml | 135 +++ cluster/charts/fuel-streams/values.yaml | 466 +++++------ cluster/docker/docker-compose.yml | 58 +- .../docker/fuel-streams-publisher.Dockerfile | 130 --- cluster/docker/init-localstack.sh | 5 +- cluster/docker/nats-config/accounts.conf | 15 + cluster/docker/nats-config/client.conf | 18 + cluster/docker/nats-config/core.conf | 13 + cluster/docker/nats-config/publisher.conf | 18 + cluster/docker/nats.conf | 52 -- ...s-ws.Dockerfile => sv-consumer.Dockerfile} | 39 +- cluster/docker/sv-publisher.Dockerfile | 81 ++ cluster/docker/sv-webserver.Dockerfile | 75 ++ cluster/scripts/build_docker.sh | 79 ++ cluster/scripts/build_publisher.sh | 16 - cluster/scripts/build_streamer.sh | 4 +- cluster/scripts/gen_env_secret.sh | 17 +- cluster/scripts/setup_k8s.sh | 39 - cluster/scripts/setup_minikube.sh | 70 +- crates/fuel-streams-core/Cargo.toml | 22 +- crates/fuel-streams-core/README.md | 4 +- .../src}/fuel_core_like.rs | 40 +- .../fuel-streams-core/src/fuel_core_types.rs | 14 +- crates/fuel-streams-core/src/inputs/types.rs | 23 + crates/fuel-streams-core/src/lib.rs | 13 +- crates/fuel-streams-core/src/logs/types.rs | 104 +-- crates/fuel-streams-core/src/outputs/types.rs | 28 +- .../fuel-streams-core/src/primitive_types.rs | 176 +++- .../fuel-streams-core/src/receipts/types.rs | 447 +++++++--- crates/fuel-streams-core/src/stream/error.rs | 20 +- .../src/stream}/fuel_streams.rs | 127 ++- crates/fuel-streams-core/src/stream/mod.rs | 2 + .../src/stream/stream_impl.rs | 56 +- .../src/transactions/types.rs | 28 +- crates/fuel-streams-core/src/types.rs | 3 +- .../fuel-streams-core/src/utxos/subjects.rs | 22 +- crates/fuel-streams-core/src/utxos/types.rs | 2 +- .../fuel-streams-executors}/Cargo.toml | 23 +- crates/fuel-streams-executors/src/blocks.rs | 70 ++ crates/fuel-streams-executors/src/inputs.rs | 129 +++ crates/fuel-streams-executors/src/lib.rs | 255 ++++++ crates/fuel-streams-executors/src/logs.rs | 38 + crates/fuel-streams-executors/src/outputs.rs | 155 ++++ crates/fuel-streams-executors/src/receipts.rs | 238 ++++++ .../src/transactions.rs | 82 ++ crates/fuel-streams-executors/src/utxos.rs | 85 ++ crates/fuel-streams-nats/Cargo.toml | 29 + .../nats => fuel-streams-nats/src}/error.rs | 0 crates/fuel-streams-nats/src/lib.rs | 15 + .../src}/nats_client.rs | 18 +- .../fuel-streams-nats/src/nats_client_opts.rs | 227 +++++ .../src}/nats_namespace.rs | 6 +- .../nats => fuel-streams-nats/src}/types.rs | 0 crates/fuel-streams-publisher/Cargo.toml | 74 -- crates/fuel-streams-publisher/README.md | 72 -- crates/fuel-streams-publisher/src/lib.rs | 22 - crates/fuel-streams-publisher/src/main.rs | 68 -- .../src/publisher/blocks_streams.rs | 394 --------- .../src/publisher/fuel_streams.rs | 137 --- .../src/publisher/mod.rs | 300 ------- .../src/publisher/payloads/blocks.rs | 29 - .../src/publisher/payloads/inputs.rs | 178 ---- .../src/publisher/payloads/logs.rs | 39 - .../src/publisher/payloads/mod.rs | 22 - .../src/publisher/payloads/outputs.rs | 161 ---- .../src/publisher/payloads/receipts.rs | 235 ------ .../src/publisher/payloads/transactions.rs | 177 ---- .../src/publisher/payloads/utxos.rs | 145 ---- .../src/publisher/shutdown.rs | 73 -- .../fuel-streams-publisher/src/server/http.rs | 117 --- .../fuel-streams-publisher/src/server/mod.rs | 2 - .../src/server/state.rs | 106 --- .../src/telemetry/mod.rs | 277 ------ .../src/telemetry/publisher.rs | 423 ---------- .../src/telemetry/system.rs | 634 -------------- crates/fuel-streams-storage/Cargo.toml | 5 +- crates/fuel-streams-storage/src/lib.rs | 4 - crates/fuel-streams-storage/src/nats/mod.rs | 16 - .../src/nats/nats_client_opts.rs | 147 ---- .../fuel-streams-storage/src/s3/s3_client.rs | 140 +++- .../src/s3/s3_client_opts.rs | 111 ++- .../fuel-streams-ws/src/server/ws/socket.rs | 397 --------- .../src/telemetry/elastic_search.rs | 323 ------- .../fuel-streams-ws/src/telemetry/runtime.rs | 72 -- crates/fuel-streams/src/client/client_impl.rs | 22 +- crates/sv-consumer/Cargo.toml | 50 ++ crates/sv-consumer/src/cli.rs | 22 + crates/sv-consumer/src/lib.rs | 33 + crates/sv-consumer/src/main.rs | 220 +++++ crates/sv-publisher/Cargo.toml | 51 ++ .../src/cli.rs | 20 +- crates/sv-publisher/src/lib.rs | 2 + crates/sv-publisher/src/main.rs | 213 +++++ crates/sv-publisher/src/shutdown.rs | 104 +++ .../Cargo.toml | 8 +- .../README.md | 2 +- .../src/cli.rs | 4 +- .../src/client/mod.rs | 0 .../src/config.rs | 2 +- .../src/lib.rs | 0 .../src/main.rs | 4 +- .../src/server/api.rs | 0 .../src/server/auth.rs | 0 .../src/server/context.rs | 33 +- .../src/server/http/handlers.rs | 0 .../src/server/http/mod.rs | 0 .../src/server/http/models.rs | 0 .../src/server/middlewares/auth.rs | 0 .../src/server/middlewares/mod.rs | 0 .../src/server/mod.rs | 0 .../src/server/state.rs | 2 +- .../src/server/ws/errors.rs | 0 .../src/server/ws/mod.rs | 1 - .../src/server/ws/models.rs | 2 +- crates/sv-webserver/src/server/ws/socket.rs | 377 +++++++++ .../src/server/ws/state.rs | 0 .../src/telemetry/elastic_search.rs | 0 .../src/telemetry/metrics.rs | 0 .../src/telemetry/mod.rs | 0 .../src/telemetry/runtime.rs | 0 .../src/telemetry/system.rs | 83 +- examples/Cargo.toml | 2 +- examples/multiple-streams.rs | 37 +- examples/websockets.rs | 2 +- knope.toml | 59 +- scripts/run_publisher.sh | 35 +- scripts/run_streamer.sh | 82 -- scripts/run_webserver.sh | 91 ++ scripts/set_env.sh | 28 +- tarpaulin.toml | 4 +- tests/Cargo.toml | 5 - tests/tests/client.rs | 43 +- tests/tests/publisher.rs | 762 +++++++++-------- 206 files changed, 6144 insertions(+), 9233 deletions(-) delete mode 100644 benches/bench-consumers/Cargo.toml delete mode 100644 benches/bench-consumers/README.md delete mode 100644 benches/bench-consumers/benches/consumers.rs delete mode 100644 benches/bench-consumers/src/lib.rs delete mode 100644 benches/bench-consumers/src/main.rs delete mode 100644 benches/bench-consumers/src/runners/benchmark_results.rs delete mode 100644 benches/bench-consumers/src/runners/mod.rs delete mode 100644 benches/bench-consumers/src/runners/runner_all.rs delete mode 100644 benches/bench-consumers/src/runners/runner_consumer.rs delete mode 100644 benches/bench-consumers/src/runners/runner_kv_watcher.rs delete mode 100644 benches/bench-consumers/src/runners/runner_subscription.rs delete mode 100644 benches/load-tester/Cargo.toml delete mode 100644 benches/load-tester/README.md delete mode 100644 benches/load-tester/src/lib.rs delete mode 100644 benches/load-tester/src/main.rs delete mode 100644 benches/load-tester/src/runners/cli.rs delete mode 100644 benches/load-tester/src/runners/mod.rs delete mode 100644 benches/load-tester/src/runners/results.rs delete mode 100644 benches/load-tester/src/runners/runner_all.rs delete mode 100644 benches/load-tester/src/runners/runner_streamable.rs delete mode 100644 benches/nats-publisher/README.md delete mode 100644 benches/nats-publisher/config/nats.conf delete mode 100644 benches/nats-publisher/src/lib.rs delete mode 100644 benches/nats-publisher/src/main.rs delete mode 100644 benches/nats-publisher/src/utils/blocks.rs delete mode 100644 benches/nats-publisher/src/utils/mod.rs delete mode 100644 benches/nats-publisher/src/utils/nats.rs delete mode 100644 benches/nats-publisher/src/utils/tx.rs delete mode 100644 cluster/charts/fuel-streams-publisher/.helmignore delete mode 100644 cluster/charts/fuel-streams-publisher/Chart.yaml delete mode 100644 cluster/charts/fuel-streams-publisher/templates/_helpers.tpl delete mode 100644 cluster/charts/fuel-streams-publisher/templates/hpa.yaml delete mode 100644 cluster/charts/fuel-streams-publisher/templates/service.yaml delete mode 100644 cluster/charts/fuel-streams-publisher/templates/serviceaccount.yaml delete mode 100644 cluster/charts/fuel-streams-publisher/templates/statefulset.yaml delete mode 100644 cluster/charts/fuel-streams-publisher/values.yaml create mode 100644 cluster/charts/fuel-streams/templates/_hpa.yaml delete mode 100644 cluster/charts/fuel-streams/templates/certificate.yaml create mode 100644 cluster/charts/fuel-streams/templates/common-config.yaml create mode 100644 cluster/charts/fuel-streams/templates/consumer/statefulset.yaml delete mode 100644 cluster/charts/fuel-streams/templates/external-service.yaml delete mode 100644 cluster/charts/fuel-streams/templates/publisher/network-configmap.yaml create mode 100644 cluster/charts/fuel-streams/templates/webserver/service.yaml create mode 100644 cluster/charts/fuel-streams/tests/consumer/deployment_test.yaml delete mode 100644 cluster/charts/fuel-streams/tests/publisher/network-configmap.yaml create mode 100644 cluster/charts/fuel-streams/values-local.yaml delete mode 100644 cluster/docker/fuel-streams-publisher.Dockerfile create mode 100644 cluster/docker/nats-config/accounts.conf create mode 100644 cluster/docker/nats-config/client.conf create mode 100644 cluster/docker/nats-config/core.conf create mode 100644 cluster/docker/nats-config/publisher.conf delete mode 100644 cluster/docker/nats.conf rename cluster/docker/{fuel-streams-ws.Dockerfile => sv-consumer.Dockerfile} (67%) create mode 100644 cluster/docker/sv-publisher.Dockerfile create mode 100644 cluster/docker/sv-webserver.Dockerfile create mode 100755 cluster/scripts/build_docker.sh delete mode 100755 cluster/scripts/build_publisher.sh delete mode 100755 cluster/scripts/setup_k8s.sh rename crates/{fuel-streams-publisher/src/publisher => fuel-streams-core/src}/fuel_core_like.rs (91%) rename crates/{fuel-streams-ws/src/server/ws => fuel-streams-core/src/stream}/fuel_streams.rs (76%) rename {benches/nats-publisher => crates/fuel-streams-executors}/Cargo.toml (52%) create mode 100644 crates/fuel-streams-executors/src/blocks.rs create mode 100644 crates/fuel-streams-executors/src/inputs.rs create mode 100644 crates/fuel-streams-executors/src/lib.rs create mode 100644 crates/fuel-streams-executors/src/logs.rs create mode 100644 crates/fuel-streams-executors/src/outputs.rs create mode 100644 crates/fuel-streams-executors/src/receipts.rs create mode 100644 crates/fuel-streams-executors/src/transactions.rs create mode 100644 crates/fuel-streams-executors/src/utxos.rs create mode 100644 crates/fuel-streams-nats/Cargo.toml rename crates/{fuel-streams-storage/src/nats => fuel-streams-nats/src}/error.rs (100%) create mode 100644 crates/fuel-streams-nats/src/lib.rs rename crates/{fuel-streams-storage/src/nats => fuel-streams-nats/src}/nats_client.rs (86%) create mode 100644 crates/fuel-streams-nats/src/nats_client_opts.rs rename crates/{fuel-streams-storage/src/nats => fuel-streams-nats/src}/nats_namespace.rs (92%) rename crates/{fuel-streams-storage/src/nats => fuel-streams-nats/src}/types.rs (100%) delete mode 100644 crates/fuel-streams-publisher/Cargo.toml delete mode 100644 crates/fuel-streams-publisher/README.md delete mode 100644 crates/fuel-streams-publisher/src/lib.rs delete mode 100644 crates/fuel-streams-publisher/src/main.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/blocks_streams.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/fuel_streams.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/mod.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/blocks.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/inputs.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/logs.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/mod.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/outputs.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/receipts.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/transactions.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/payloads/utxos.rs delete mode 100644 crates/fuel-streams-publisher/src/publisher/shutdown.rs delete mode 100644 crates/fuel-streams-publisher/src/server/http.rs delete mode 100644 crates/fuel-streams-publisher/src/server/mod.rs delete mode 100644 crates/fuel-streams-publisher/src/server/state.rs delete mode 100644 crates/fuel-streams-publisher/src/telemetry/mod.rs delete mode 100644 crates/fuel-streams-publisher/src/telemetry/publisher.rs delete mode 100644 crates/fuel-streams-publisher/src/telemetry/system.rs delete mode 100644 crates/fuel-streams-storage/src/nats/mod.rs delete mode 100644 crates/fuel-streams-storage/src/nats/nats_client_opts.rs delete mode 100644 crates/fuel-streams-ws/src/server/ws/socket.rs delete mode 100755 crates/fuel-streams-ws/src/telemetry/elastic_search.rs delete mode 100644 crates/fuel-streams-ws/src/telemetry/runtime.rs create mode 100644 crates/sv-consumer/Cargo.toml create mode 100644 crates/sv-consumer/src/cli.rs create mode 100644 crates/sv-consumer/src/lib.rs create mode 100644 crates/sv-consumer/src/main.rs create mode 100644 crates/sv-publisher/Cargo.toml rename crates/{fuel-streams-publisher => sv-publisher}/src/cli.rs (58%) create mode 100644 crates/sv-publisher/src/lib.rs create mode 100644 crates/sv-publisher/src/main.rs create mode 100644 crates/sv-publisher/src/shutdown.rs rename crates/{fuel-streams-ws => sv-webserver}/Cargo.toml (95%) rename crates/{fuel-streams-ws => sv-webserver}/README.md (97%) rename crates/{fuel-streams-ws => sv-webserver}/src/cli.rs (95%) rename crates/{fuel-streams-ws => sv-webserver}/src/client/mod.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/config.rs (97%) rename crates/{fuel-streams-ws => sv-webserver}/src/lib.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/main.rs (93%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/api.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/auth.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/context.rs (66%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/http/handlers.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/http/mod.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/http/models.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/middlewares/auth.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/middlewares/mod.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/mod.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/state.rs (98%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/ws/errors.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/ws/mod.rs (74%) rename crates/{fuel-streams-ws => sv-webserver}/src/server/ws/models.rs (98%) create mode 100644 crates/sv-webserver/src/server/ws/socket.rs rename crates/{fuel-streams-ws => sv-webserver}/src/server/ws/state.rs (100%) rename crates/{fuel-streams-publisher => sv-webserver}/src/telemetry/elastic_search.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/telemetry/metrics.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/telemetry/mod.rs (100%) rename crates/{fuel-streams-publisher => sv-webserver}/src/telemetry/runtime.rs (100%) rename crates/{fuel-streams-ws => sv-webserver}/src/telemetry/system.rs (88%) delete mode 100755 scripts/run_streamer.sh create mode 100755 scripts/run_webserver.sh diff --git a/.editorconfig b/.editorconfig index 72e61c1b..12808c50 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,3 +19,7 @@ trim_trailing_whitespace = false [*.{yml,yaml}] indent_size = 2 + +[{Makefile,**.mk}] +# Use tabs for indentation (Makefiles require tabs) +indent_style = tab diff --git a/.env.sample b/.env.sample index 5b6c89de..c111ace9 100644 --- a/.env.sample +++ b/.env.sample @@ -1,26 +1,34 @@ -# Common Configuration +# Authentication & Security KEYPAIR=generated-p2p-secret -NATS_URL=nats://localhost:4222 -NATS_ADMIN_PASS=generated-secret -NATS_SYSTEM_PASS=generated-secret +JWT_AUTH_SECRET=generated-secret + +# AWS S3 Configuration AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test -AWS_REGION=us-east-1 AWS_ENDPOINT_URL=http://localhost:4566 -AWS_S3_BUCKET_NAME=fuel-streams-testnet +AWS_REGION=us-east-1 +AWS_S3_ENABLED=false +AWS_S3_BUCKET_NAME=fuel-streams-local + +# NATS Configuration +NATS_URL=nats://localhost:4222 +NATS_PUBLISHER_URL=nats://localhost:4333 +NATS_SYSTEM_USER=sys +NATS_SYSTEM_PASS=sys +NATS_ADMIN_USER=admin +NATS_ADMIN_PASS=admin +NATS_PUBLIC_USER=default_user +NATS_PUBLIC_PASS="" + +# Monitoring & Logging USE_ELASTIC_LOGGING=false USE_METRICS=true PUBLISHER_MAX_THREADS=16 + +# Elasticsearch Configuration ELASTICSEARCH_URL=http://127.0.0.1:9200 ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=generated-secret -AWS_S3_ENABLED=false -AWS_ACCESS_KEY_ID=s3-access-key-id -AWS_SECRET_ACCESS_KEY=s3-secret-access-key -AWS_REGION=s3-region -AWS_ENDPOINT_URL=s3-endpoint -AWS_S3_BUCKET_NAME=fuel-streams-local -JWT_AUTH_SECRET=generated-secret # Mainnet Configuration MAINNET_RELAYER=https://mainnet.infura.io/v3/ diff --git a/.github/actions/setup-rust/action.yaml b/.github/actions/setup-rust/action.yaml index 11d83426..600f16ef 100644 --- a/.github/actions/setup-rust/action.yaml +++ b/.github/actions/setup-rust/action.yaml @@ -21,8 +21,16 @@ runs: - name: Create .env file with NATS environment variables shell: bash run: | - echo "NATS_ADMIN_PASS=${NATS_ADMIN_PASS:-default_pass}" >> .env - echo "NATS_PUBLIC_PASS=${NATS_PUBLIC_PASS:-temp-public-pass}" >> .env + set_env_var() { + echo "$1=${!1:-$2}" >> $GITHUB_ENV + echo "$1=${!1:-$2}" >> .env + } + set_env_var "NATS_SYSTEM_USER" "sys" + set_env_var "NATS_SYSTEM_PASS" "sys" + set_env_var "NATS_ADMIN_USER" "admin" + set_env_var "NATS_ADMIN_PASS" "admin" + set_env_var "NATS_PUBLIC_USER" "default_user" + set_env_var "NATS_PUBLIC_PASS" "" - name: Install Rust uses: dtolnay/rust-toolchain@master diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c6b9ac26..2055b492 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,6 @@ name: CI on: - workflow_dispatch: pull_request: types: - opened @@ -190,24 +189,24 @@ jobs: - name: Install dependencies run: cargo fetch - test-helm: - needs: install-deps - name: Test Helm - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Helm - uses: azure/setup-helm@v4 - with: - version: "latest" - - - name: Install helm unittest plugin - run: | - helm plugin install https://github.com/helm-unittest/helm-unittest.git - - - name: Run Helm unit tests - run: | - make helm-test + # test-helm: + # needs: install-deps + # name: Test Helm + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - name: Set up Helm + # uses: azure/setup-helm@v4 + # with: + # version: "latest" + + # - name: Install helm unittest plugin + # run: | + # helm plugin install https://github.com/helm-unittest/helm-unittest.git + + # - name: Run Helm unit tests + # run: | + # make helm-test test: needs: install-deps @@ -215,8 +214,13 @@ jobs: runs-on: ubuntu-latest env: NATS_URL: nats://127.0.0.1:4222 - NATS_ADMIN_PASS: secret - NATS_PUBLIC_PASS: secret + NATS_PUBLISHER_URL: nats://127.0.0.1:4333 + NATS_SYSTEM_USER: sys + NATS_SYSTEM_PASSWORD: sys + NATS_ADMIN_USER: admin + NATS_ADMIN_PASS: admin + NATS_PUBLIC_USER: default_user + NATS_PUBLIC_PASS: "" AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_REGION: us-east-1 @@ -226,13 +230,12 @@ jobs: fail-fast: false matrix: package: - # - fuel-data-parser + - fuel-data-parser - fuel-streams - fuel-streams-core - fuel-streams-macros - - fuel-streams-publisher - - fuel-streams-ws - + - sv-webserver + - sv-publisher steps: - uses: actions/checkout@v4 @@ -266,8 +269,9 @@ jobs: fail-fast: false matrix: package: - - fuel-streams-publisher - - fuel-streams-ws + - sv-consumer + - sv-publisher + - sv-webserver is_release: - ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} platform: @@ -336,34 +340,34 @@ jobs: rustup target add ${{ matrix.platform.target }} cargo build --release --locked --target ${{ matrix.platform.target }} --package ${{ matrix.package }} - - name: Strip binaries - run: ./scripts/strip-binary.sh "${{ matrix.platform.target }}" - - - name: Set Artifact Name - id: artifact-name - shell: bash - run: | - echo "value=${{ matrix.package }}-${{ matrix.platform.os_name }}" >> $GITHUB_OUTPUT - - - name: Package as archive - shell: bash - run: | - cd target/${{ matrix.platform.target }}/release - tar czvf ../../../${{ steps.artifact-name.outputs.value }}.tar.gz ${{ matrix.package }} - cd - - - - name: Publish release artifacts - uses: actions/upload-artifact@v4 - if: >- - (github.event_name == 'push' && - github.ref == 'refs/heads/main' && - contains(github.event.head_commit.message, 'ci(release): Preparing')) || - github.event_name == 'workflow_dispatch' - with: - name: ${{ steps.artifact-name.outputs.value }} - path: ${{ matrix.package }}-* - if-no-files-found: error - retention-days: 30 + # - name: Strip binaries + # run: ./scripts/strip-binary.sh "${{ matrix.platform.target }}" + + # - name: Set Artifact Name + # id: artifact-name + # shell: bash + # run: | + # echo "value=${{ matrix.package }}-${{ matrix.platform.os_name }}" >> $GITHUB_OUTPUT + + # - name: Package as archive + # shell: bash + # run: | + # cd target/${{ matrix.platform.target }}/release + # tar czvf ../../../${{ steps.artifact-name.outputs.value }}.tar.gz ${{ matrix.package }} + # cd - + + # - name: Publish release artifacts + # uses: actions/upload-artifact@v4 + # if: >- + # (github.event_name == 'push' && + # github.ref == 'refs/heads/main' && + # contains(github.event.head_commit.message, 'ci(release): Preparing')) || + # github.event_name == 'workflow_dispatch' + # with: + # name: ${{ steps.artifact-name.outputs.value }} + # path: ${{ matrix.package }}-* + # if-no-files-found: error + # retention-days: 30 release: name: Create Release with Knope @@ -374,7 +378,7 @@ jobs: github.event_name == 'workflow_dispatch' needs: - test - - test-helm + # - test-helm - build runs-on: ubuntu-latest permissions: @@ -384,14 +388,14 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - merge-multiple: true + # - name: Download Artifacts + # uses: actions/download-artifact@v4 + # with: + # path: artifacts + # merge-multiple: true - - name: List Artifacts - run: ls -R artifacts + # - name: List Artifacts + # run: ls -R artifacts - name: Run Knope Action uses: knope-dev/action@v2.1.0 diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index 748ee112..6aff171b 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -3,15 +3,16 @@ name: Build and publish Docker image on: workflow_dispatch: inputs: - image_type: - description: "Choose which image to build (publisher/webserver/both)" - required: true + package: type: choice + description: "Package to build and publish" + default: "all" + required: true options: - - publisher - - webserver - - both - default: "both" + - all + - sv-publisher + - sv-webserver + - sv-consumer push: branches: - main @@ -30,11 +31,15 @@ concurrency: jobs: build-and-publish-image: runs-on: ubuntu-latest - if: | - (github.event_name == 'release' && github.event.action == 'published') || - github.ref == 'refs/heads/main' || - github.event_name == 'workflow_dispatch' || - github.event_name == 'pull_request' + strategy: + matrix: + package: + - name: sv-webserver + image: cluster/docker/sv-webserver.Dockerfile + - name: sv-publisher + image: cluster/docker/sv-publisher.Dockerfile + - name: sv-consumer + image: cluster/docker/sv-consumer.Dockerfile steps: - uses: actions/checkout@v4 @@ -42,24 +47,14 @@ jobs: id: sha run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Build and push Docker for publisher - if: github.event.inputs.image_type == 'publisher' || github.event.inputs.image_type == 'both' - || github.event_name != 'workflow_dispatch' - uses: ./.github/actions/docker-publish - id: publish-fuel-streams-nats - with: - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - image: ghcr.io/fuellabs/fuel-streams-publisher - dockerfile: cluster/docker/fuel-streams-publisher.Dockerfile - - - name: Build and push Docker for webserver - if: github.event.inputs.image_type == 'webserver' || github.event.inputs.image_type == 'both' - || github.event_name != 'workflow_dispatch' + - name: Build and push Docker for ${matrix.package.name} (${{ steps.sha.outputs.short_sha }}) + if: | + (github.event_name == 'workflow_dispatch' && (github.event.inputs.package == 'all' || github.event.inputs.package == matrix.package.name)) || + github.event_name != 'workflow_dispatch' uses: ./.github/actions/docker-publish - id: publish-fuel-webserver-nats + id: publish with: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - image: ghcr.io/fuellabs/fuel-streams-ws - dockerfile: cluster/docker/fuel-streams-ws.Dockerfile + image: ghcr.io/fuellabs/${{ matrix.package.name }} + dockerfile: ${{ matrix.package.image }} diff --git a/.github/workflows/helm_publish.yaml b/.github/workflows/helm_publish.yaml index 932d54f6..8dc9de93 100644 --- a/.github/workflows/helm_publish.yaml +++ b/.github/workflows/helm_publish.yaml @@ -2,87 +2,41 @@ name: Build and Publish Helm Chart on: workflow_dispatch: - inputs: - chart: - description: "Select the Helm chart to deploy" - required: true - type: choice - options: - - fuel-streams-publisher - - fuel-streams push: branches: - main - paths: - - cluster/charts/fuel-streams-publisher/Chart.yaml - - cluster/charts/fuel-streams/Chart.yaml + release: + types: + - published + +env: + CHART_NAME: fuel-streams + CHART_PATH: cluster/charts/fuel-streams permissions: contents: read + packages: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: helm-release: - name: Build Helm Charts runs-on: ubuntu-latest - if: | - github.event_name == 'workflow_dispatch' || - (github.event_name == 'release' && github.event.action == 'published') || - github.ref == 'refs/heads/main' || - github.event_name == 'pull_request' - permissions: - contents: read - packages: write steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Determine charts to process - id: charts - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "charts=${{ github.event.inputs.chart }}" >> $GITHUB_OUTPUT - else - echo "charts=fuel-streams-publisher fuel-streams" >> $GITHUB_OUTPUT - fi - - - name: Validate Chart Names - run: | - for chart in ${{ steps.charts.outputs.charts }}; do - if [ ! -d "cluster/charts/$chart" ]; then - echo "Error: Chart '$chart' does not exist." - exit 1 - fi - done - + - uses: actions/checkout@v4 - name: Helm Dependencies Update run: | - set -e - for chart in ${{ steps.charts.outputs.charts }}; do - echo "Updating dependencies for $chart" - helm dependency update cluster/charts/$chart - done + cd ${{ env.CHART_PATH }} && helm dependency update - - name: Get chart versions - id: versions + - name: Get chart version + id: version run: | - publisher_version=$(awk '/^version:/ {print $2}' cluster/charts/fuel-streams-publisher/Chart.yaml) - streams_version=$(awk '/^version:/ {print $2}' cluster/charts/fuel-streams/Chart.yaml) - echo "publisher_version=$publisher_version" >> $GITHUB_OUTPUT - echo "streams_version=$streams_version" >> $GITHUB_OUTPUT - - - name: "Build chart: [fuel-streams-publisher v${{ steps.versions.outputs.publisher_version }}]" - if: contains(steps.charts.outputs.charts, 'fuel-streams-publisher') - uses: bsord/helm-push@v4.1.0 - with: - useOCIRegistry: true - registry-url: oci://ghcr.io/fuellabs/helmcharts - username: ${{ github.repository_owner }} - access-token: ${{ secrets.GITHUB_TOKEN }} - force: true - chart-folder: ./cluster/charts/fuel-streams-publisher + version=$(awk '/^version:/ {print $2}' ${{ env.CHART_PATH }}/Chart.yaml) + echo "version=$version" >> $GITHUB_OUTPUT - - name: "Build chart: [fuel-streams v${{ steps.versions.outputs.streams_version }}]" - if: contains(steps.charts.outputs.charts, 'fuel-streams') + - name: "Build chart: [${{ env.CHART_NAME }} v${{ steps.version.outputs.version }}]" uses: bsord/helm-push@v4.1.0 with: useOCIRegistry: true @@ -90,20 +44,11 @@ jobs: username: ${{ github.repository_owner }} access-token: ${{ secrets.GITHUB_TOKEN }} force: true - chart-folder: ./cluster/charts/fuel-streams + chart-folder: ${{ env.CHART_PATH }} - name: Build Summary run: |- echo "### Helm Charts Build Summary 📊" >> $GITHUB_STEP_SUMMARY echo "| Chart | Version | Status |" >> $GITHUB_STEP_SUMMARY echo "|-------|---------|--------|" >> $GITHUB_STEP_SUMMARY - - for chart in ${{ steps.charts.outputs.charts }}; do - version="" - if [ "$chart" = "fuel-streams-publisher" ]; then - version="${{ steps.versions.outputs.publisher_version }}" - elif [ "$chart" = "fuel-streams" ]; then - version="${{ steps.versions.outputs.streams_version }}" - fi - echo "| $chart | $version | ✅ Published |" >> $GITHUB_STEP_SUMMARY - done + echo "| ${{ env.CHART_NAME }} | ${{ steps.version.outputs.version }} | ✅ Published |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/publish_release.yaml b/.github/workflows/publish_release.yaml index 4dbc4f19..bd4edc6a 100644 --- a/.github/workflows/publish_release.yaml +++ b/.github/workflows/publish_release.yaml @@ -19,17 +19,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set NATS environment variables - shell: bash - run: | - set_env_var() { - echo "$1=${!1:-$2}" >> $GITHUB_ENV - echo "$1=${!1:-$2}" >> .env - } - set_env_var "NATS_ADMIN_PASS" "null" - - name: Install toolchain - uses: dtolnay/rust-toolchain@master + - name: Install Rust + uses: ./.github/actions/setup-rust with: toolchain: ${{ env.RUST_VERSION }} target: x86_64-unknown-linux-gnu,wasm32-unknown-unknown diff --git a/.gitignore b/.gitignore index 9b22fe23..8607f4ff 100644 --- a/.gitignore +++ b/.gitignore @@ -20,10 +20,8 @@ profile.json coverage/ docs/ **/**/charts/**.tgz -values-publisher-secrets.yaml +values-secrets.yaml values-publisher-env.yaml localstack-data .vscode - **/Cargo.lock -!./Cargo.lock diff --git a/.prettierignore b/.prettierignore index 8b7b5b96..0063613c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,2 @@ -helm -cluster +cluster/charts pnpm-lock.yaml diff --git a/.rustfmt.toml b/.rustfmt.toml index aaebb675..2d0df4eb 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -10,3 +10,8 @@ use_field_init_shorthand = true reorder_imports = true reorder_modules = true tab_spaces = 4 +# Add these new settings +format_macro_matchers = true +format_macro_bodies = true +# If you want macros to ignore the max_width setting +overflow_delimited_expr = true diff --git a/.typos.toml b/.typos.toml index 97819a36..87a65555 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,9 +1,7 @@ [files] extend-exclude = [ "pnpm-lock.yaml", - "crates/fuel-streams-publisher/README.md", - "crates/fuel-streams-publisher/src/elastic.rs", - "crates/fuel-streams-ws/README.md", + "crates/sv-webserver/README.md", "docker/chain-config", "docker/monitoring", "cluster", diff --git a/Cargo.lock b/Cargo.lock index aa6a87fc..a8c25d4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -410,15 +410,6 @@ version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" -[[package]] -name = "approx" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" -dependencies = [ - "num-traits", -] - [[package]] name = "arbitrary" version = "1.4.1" @@ -491,12 +482,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" -[[package]] -name = "assert_matches" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" - [[package]] name = "async-compression" version = "0.4.18" @@ -519,9 +504,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba6d24703c5adc5ba9116901b92ee4e4c0643c01a56c4fd303f3818638d7449" +checksum = "59fd6bd734afb8b6e4d0f84a3e77305ce0a7ccc60d70f6001cb5e1c3f38d8ff1" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -538,7 +523,6 @@ dependencies = [ "mime", "multer", "num-traits", - "once_cell", "pin-project-lite", "regex", "serde", @@ -552,9 +536,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94c2d176893486bd37cd1b6defadd999f7357bf5804e92f510c08bcf16c538f" +checksum = "ac38b4dd452d529d6c0248b51df23603f0a875770352e26ae8c346ce6c149b3e" dependencies = [ "Inflector", "async-graphql-parser", @@ -569,9 +553,9 @@ dependencies = [ [[package]] name = "async-graphql-parser" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79272bdbf26af97866e149f05b2b546edb5c00e51b5f916289931ed233e208ad" +checksum = "42d271ddda2f55b13970928abbcbc3423cfc18187c60e8769b48f21a93b7adaa" dependencies = [ "async-graphql-value", "pest", @@ -581,9 +565,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.11" +version = "7.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1" +checksum = "aefe909173a037eaf3281b046dc22580b59a38b765d7b8d5116f2ffef098048d" dependencies = [ "bytes", "indexmap 2.7.0", @@ -981,9 +965,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "8aa8ff1492fd9fb99ae28e8467af0dbbb7c31512b16fabf1a0f10d7bb6ef78bb" dependencies = [ "futures-util", "pin-project-lite", @@ -1088,7 +1072,7 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", @@ -1177,7 +1161,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "itoa", "matchit 0.5.0", "memchr", @@ -1208,7 +1192,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "itoa", "matchit 0.7.3", "memchr", @@ -1377,22 +1361,6 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" -[[package]] -name = "bench-consumers" -version = "0.0.13" -dependencies = [ - "anyhow", - "async-nats", - "chrono", - "criterion", - "fuel-core-types 0.40.2", - "fuel-streams-core", - "futures", - "nats-publisher", - "statrs", - "tokio", -] - [[package]] name = "bincode" version = "1.3.3" @@ -1592,12 +1560,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "bytemuck" -version = "1.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" - [[package]] name = "byteorder" version = "1.5.0" @@ -1693,9 +1655,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.2" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" dependencies = [ "jobserver", "libc", @@ -1749,9 +1711,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1973,29 +1935,17 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "confy" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0" -dependencies = [ - "directories", - "serde", - "thiserror 1.0.69", - "toml", -] - [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ "encode_unicode", - "lazy_static", "libc", - "unicode-width 0.1.14", - "windows-sys 0.52.0", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", ] [[package]] @@ -2019,18 +1969,18 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const_format" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c655d81ff1114fb0dcdea9225ea9f0cc712a6f8d189378e82bdf62a473a64b" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" dependencies = [ "const_format_proc_macros", ] [[package]] name = "const_format_proc_macros" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff1a44b93f47b1bac19a27932f5c591e43d1ba357ee4f61526c8a25603f0eb1" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" dependencies = [ "proc-macro2", "quote", @@ -2399,9 +2349,9 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2418,9 +2368,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -2809,15 +2759,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "directories" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" -dependencies = [ - "dirs-sys 0.4.1", -] - [[package]] name = "directories-next" version = "2.0.0" @@ -2834,7 +2775,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "dirs-sys 0.3.7", + "dirs-sys", ] [[package]] @@ -2848,18 +2789,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -3066,9 +2995,9 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" @@ -3344,22 +3273,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "eventsource-client" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ddc25e1ad2cc0106d5e2d967397b4fb2068a66677ee9b0eea4600e5cfe8fb4" -dependencies = [ - "futures", - "hyper 0.14.31", - "hyper-rustls 0.24.2", - "hyper-timeout 0.4.1", - "log", - "pin-project", - "rand", - "tokio", -] - [[package]] name = "eyre" version = "0.6.12" @@ -3378,9 +3291,9 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" @@ -3475,9 +3388,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" [[package]] name = "foreign-types" @@ -3579,7 +3492,7 @@ dependencies = [ "fuel-core-upgradable-executor", "futures", "hex", - "hyper 0.14.31", + "hyper 0.14.32", "indicatif", "itertools 0.12.1", "num_cpus", @@ -3613,7 +3526,6 @@ dependencies = [ "clap 4.5.23", "const_format", "dirs", - "dotenvy", "fuel-core", "fuel-core-chain-config", "fuel-core-compression", @@ -3658,14 +3570,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a10ccde16fd926137070d3baa77a1096c2ff6cdca26d14177139c16e59e697d" dependencies = [ "anyhow", - "base64 0.22.1", "cynic", "derive_more 0.99.18", - "eventsource-client", "fuel-core-types 0.40.2", - "futures", "hex", - "hyper-rustls 0.24.2", "itertools 0.12.1", "reqwest 0.11.27", "schemafy_lib", @@ -3939,7 +3847,7 @@ dependencies = [ "fuel-vm 0.58.2", "impl-tools", "itertools 0.12.1", - "mockall 0.11.4", + "mockall", "num_enum", "paste", "postcard", @@ -4109,7 +4017,7 @@ dependencies = [ "serde_json", "strum 0.26.3", "strum_macros 0.26.4", - "thiserror 2.0.4", + "thiserror 2.0.8", "tokio", ] @@ -4217,7 +4125,7 @@ dependencies = [ "displaydoc", "fuel-streams-core", "futures", - "thiserror 2.0.4", + "thiserror 2.0.8", "tokio", ] @@ -4225,25 +4133,31 @@ dependencies = [ name = "fuel-streams-core" version = "0.0.13" dependencies = [ + "anyhow", "async-nats", "async-trait", "chrono", "displaydoc", + "fuel-core", + "fuel-core-bin", "fuel-core-client", "fuel-core-importer", + "fuel-core-services", + "fuel-core-storage", "fuel-core-types 0.40.2", "fuel-data-parser", "fuel-networks", "fuel-streams-macros", + "fuel-streams-nats", "fuel-streams-storage", "futures", "hex", "pretty_assertions", "serde", "serde_json", - "sha2 0.10.8", - "thiserror 2.0.4", + "thiserror 2.0.8", "tokio", + "tracing", ] [[package]] @@ -4253,136 +4167,67 @@ dependencies = [ "anyhow", "fuel-core-types 0.40.2", "fuel-streams", - "fuel-streams-ws", "futures", + "sv-webserver", "tokio", ] [[package]] -name = "fuel-streams-macros" -version = "0.0.13" -dependencies = [ - "subject-derive", -] - -[[package]] -name = "fuel-streams-publisher" +name = "fuel-streams-executors" version = "0.0.13" dependencies = [ - "actix-cors", - "actix-server", - "actix-web", "anyhow", - "assert_matches", "async-nats", - "async-trait", - "chrono", - "clap 4.5.23", - "derive_more 1.0.0", - "displaydoc", - "dotenvy", - "elasticsearch", "fuel-core", - "fuel-core-bin", - "fuel-core-importer", - "fuel-core-services", - "fuel-core-storage", - "fuel-core-types 0.40.2", - "fuel-streams", "fuel-streams-core", - "fuel-streams-storage", "futures", - "mockall 0.13.1", - "mockall_double", "num_cpus", - "openssl", - "parking_lot", - "prometheus", - "rand", "rayon", - "rust_decimal", "serde", "serde_json", - "serde_prometheus", "sha2 0.10.8", - "sysinfo", - "thiserror 2.0.4", + "thiserror 2.0.8", "tokio", - "tokio-stream", "tracing", - "tracing-actix-web", - "url", ] [[package]] -name = "fuel-streams-storage" +name = "fuel-streams-macros" +version = "0.0.13" +dependencies = [ + "subject-derive", +] + +[[package]] +name = "fuel-streams-nats" version = "0.0.13" dependencies = [ "async-nats", - "aws-config", - "aws-sdk-s3", - "aws-smithy-runtime-api", - "aws-smithy-types", "displaydoc", "dotenvy", - "fuel-networks", "pretty_assertions", "rand", "serde_json", - "thiserror 2.0.4", + "thiserror 2.0.8", "tokio", "tracing", ] [[package]] -name = "fuel-streams-ws" +name = "fuel-streams-storage" version = "0.0.13" dependencies = [ - "actix-cors", - "actix-server", - "actix-service", - "actix-web", - "actix-ws", - "anyhow", - "async-nats", - "async-trait", - "bytestring", - "chrono", - "clap 4.5.23", - "confy", - "derive_more 1.0.0", - "displaydoc", + "aws-config", + "aws-sdk-s3", + "aws-smithy-runtime-api", + "aws-smithy-types", "dotenvy", - "elasticsearch", - "fuel-streams", - "fuel-streams-core", - "fuel-streams-storage", - "futures", - "futures-util", - "jsonwebtoken 9.3.0", - "num_cpus", - "openssl", - "parking_lot", - "prometheus", + "pretty_assertions", "rand", - "reqwest 0.12.9", - "rust_decimal", - "serde", "serde_json", - "serde_prometheus", - "sysinfo", - "thiserror 2.0.4", - "time", + "thiserror 2.0.8", "tokio", - "tokio-tungstenite 0.24.0", - "toml", "tracing", - "tracing-actix-web", - "tracing-subscriber", - "url", - "urlencoding", - "uuid", - "validator", ] [[package]] @@ -4612,7 +4457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", ] @@ -5142,9 +4987,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -5166,9 +5011,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -5193,26 +5038,25 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "rustls 0.21.12", "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", - "webpki-roots 0.25.4", ] [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "f6884a48c6826ec44f524c7456b163cebe9e55a18d7b5e307cb4f100371cc767" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "tokio", "tokio-rustls 0.26.1", @@ -5226,7 +5070,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.31", + "hyper 0.14.32", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -5238,7 +5082,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "pin-project-lite", "tokio", @@ -5253,7 +5097,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "native-tls", "tokio", @@ -5272,7 +5116,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -5508,7 +5352,7 @@ dependencies = [ "bytes", "futures", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "rand", "tokio", @@ -5551,9 +5395,9 @@ dependencies = [ [[package]] name = "impl-tools" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a84bc8d2baf8da56e93b4247067d918e1a44829bbbe3e4b875aaf8d7d3c7bc9" +checksum = "b4739bc9af85c18969eba5e4db90dbf26be140ff2e5628593693f18559e9e5fe" dependencies = [ "autocfg", "impl-tools-lib", @@ -5563,9 +5407,9 @@ dependencies = [ [[package]] name = "impl-tools-lib" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a795a1e201125947a063b967c79de6ae152143ab522f481d4f493c44835ba37a" +checksum = "798fe18a7e727001b30a029ab9cdd485afd325801d4df846f0bb5338b2986a2c" dependencies = [ "proc-macro-error2", "proc-macro2", @@ -5621,7 +5465,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width 0.2.0", + "unicode-width", "web-time", ] @@ -5728,9 +5572,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -5820,9 +5664,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.167" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libflate" @@ -6168,7 +6012,7 @@ dependencies = [ "quinn", "rand", "ring 0.17.8", - "rustls 0.23.19", + "rustls 0.23.20", "socket2", "thiserror 1.0.69", "tokio", @@ -6260,7 +6104,7 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.17.8", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-webpki 0.101.7", "thiserror 1.0.69", "x509-parser", @@ -6427,21 +6271,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" -[[package]] -name = "load-tester" -version = "0.0.13" -dependencies = [ - "anyhow", - "async-nats", - "chrono", - "clap 4.5.23", - "fuel-streams", - "fuel-streams-core", - "futures", - "statrs", - "tokio", -] - [[package]] name = "local-channel" version = "0.1.5" @@ -6551,16 +6380,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "matrixmultiply" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" -dependencies = [ - "autocfg", - "rawpointer", -] - [[package]] name = "md-5" version = "0.10.6" @@ -6609,9 +6428,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -6638,22 +6457,8 @@ dependencies = [ "downcast", "fragile", "lazy_static", - "mockall_derive 0.11.4", - "predicates 2.1.5", - "predicates-tree", -] - -[[package]] -name = "mockall" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" -dependencies = [ - "cfg-if", - "downcast", - "fragile", - "mockall_derive 0.13.1", - "predicates 3.1.2", + "mockall_derive", + "predicates", "predicates-tree", ] @@ -6669,30 +6474,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "mockall_derive" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 2.0.90", -] - -[[package]] -name = "mockall_double" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1ca96e5ac35256ae3e13536edd39b172b88f41615e1d7b653c8ad24524113e8" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 2.0.90", -] - [[package]] name = "multer" version = "3.1.0" @@ -6742,9 +6523,9 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.2" +version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc41f430805af9d1cf4adae4ed2149c759b877b01d909a1f40256188d09345d2" +checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" dependencies = [ "core2", "unsigned-varint 0.8.0", @@ -6770,23 +6551,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" -[[package]] -name = "nalgebra" -version = "0.33.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b" -dependencies = [ - "approx", - "matrixmultiply", - "num-complex", - "num-rational", - "num-traits", - "rand", - "rand_distr", - "simba", - "typenum", -] - [[package]] name = "names" version = "0.14.0" @@ -6814,25 +6578,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "nats-publisher" -version = "0.0.13" -dependencies = [ - "anyhow", - "async-nats", - "clap 4.5.23", - "criterion", - "fuel-core", - "fuel-core-bin", - "fuel-core-importer", - "fuel-core-storage", - "fuel-core-types 0.40.2", - "fuel-data-parser", - "fuel-streams-core", - "tokio", - "tracing", -] - [[package]] name = "netlink-packet-core" version = "0.7.0" @@ -6984,15 +6729,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - [[package]] name = "num-conv" version = "0.1.0" @@ -7026,7 +6762,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -7060,6 +6795,15 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -7184,12 +6928,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "os_str_bytes" version = "6.6.1" @@ -7371,12 +7109,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 1.0.69", + "thiserror 2.0.8", "ucd-trie", ] @@ -7592,27 +7330,17 @@ dependencies = [ "regex", ] -[[package]] -name = "predicates" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" -dependencies = [ - "anstyle", - "predicates-core", -] - [[package]] name = "predicates-core" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" [[package]] name = "predicates-tree" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" dependencies = [ "predicates-core", "termtree", @@ -7790,9 +7518,9 @@ dependencies = [ [[package]] name = "proptest" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bitflags 2.6.0", "lazy_static", @@ -8003,9 +7731,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.0", - "rustls 0.23.19", + "rustls 0.23.20", "socket2", - "thiserror 2.0.4", + "thiserror 2.0.8", "tokio", "tracing", ] @@ -8021,10 +7749,10 @@ dependencies = [ "rand", "ring 0.17.8", "rustc-hash 2.1.0", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.4", + "thiserror 2.0.8", "tinyvec", "tracing", "web-time", @@ -8032,9 +7760,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -8089,16 +7817,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand", -] - [[package]] name = "rand_xorshift" version = "0.3.0" @@ -8108,12 +7826,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - [[package]] name = "rayon" version = "1.10.0" @@ -8148,9 +7860,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags 2.6.0", ] @@ -8254,7 +7966,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "ipnet", "js-sys", @@ -8301,8 +8013,8 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", - "hyper-rustls 0.27.3", + "hyper 1.5.2", + "hyper-rustls 0.27.4", "hyper-tls", "hyper-util", "ipnet", @@ -8314,7 +8026,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.19", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -8559,15 +8271,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -8584,9 +8296,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "once_cell", "ring 0.17.8", @@ -8630,7 +8342,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.0.1", + "security-framework 3.1.0", ] [[package]] @@ -8653,9 +8365,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -8704,15 +8416,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -[[package]] -name = "safe_arch" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3460605018fdc9612bce72735cba0d27efbcd9904780d44c7e3a9948f96148a" -dependencies = [ - "bytemuck", -] - [[package]] name = "same-file" version = "1.0.6" @@ -8874,9 +8577,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" dependencies = [ "bitflags 2.6.0", "core-foundation 0.10.0", @@ -8887,9 +8590,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -8897,9 +8600,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" dependencies = [ "serde", ] @@ -9152,19 +8855,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "simba" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a386a501cd104797982c15ae17aafe8b9261315b5d07e3ec803f2ea26be0fa" -dependencies = [ - "approx", - "num-complex", - "num-traits", - "paste", - "wide", -] - [[package]] name = "simdutf8" version = "0.1.5" @@ -9308,30 +8998,13 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7beae5182595e9a8b683fa98c4317f956c9a2dec3b9716990d20023cc60c766" -[[package]] -name = "statrs" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" -dependencies = [ - "approx", - "nalgebra", - "num-traits", - "rand", -] - [[package]] name = "streams-tests" version = "0.0.13" dependencies = [ - "anyhow", - "async-trait", "fuel-core", - "fuel-core-importer", - "fuel-core-types 0.40.2", "fuel-streams", "fuel-streams-core", - "fuel-streams-publisher", "futures", "pretty_assertions", "rand", @@ -9446,11 +9119,103 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" +[[package]] +name = "sv-consumer" +version = "0.0.13" +dependencies = [ + "anyhow", + "async-nats", + "clap 4.5.23", + "dotenvy", + "fuel-core", + "fuel-streams-core", + "fuel-streams-executors", + "futures", + "num_cpus", + "openssl", + "serde_json", + "sv-publisher", + "thiserror 2.0.8", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "sv-publisher" +version = "0.0.13" +dependencies = [ + "anyhow", + "async-nats", + "clap 4.5.23", + "fuel-core", + "fuel-core-bin", + "fuel-core-types 0.40.2", + "fuel-streams-core", + "fuel-streams-executors", + "futures", + "openssl", + "thiserror 2.0.8", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "sv-webserver" +version = "0.0.13" +dependencies = [ + "actix-cors", + "actix-server", + "actix-service", + "actix-web", + "actix-ws", + "anyhow", + "async-nats", + "bytestring", + "chrono", + "clap 4.5.23", + "derive_more 1.0.0", + "displaydoc", + "dotenvy", + "elasticsearch", + "fuel-streams", + "fuel-streams-core", + "fuel-streams-nats", + "fuel-streams-storage", + "futures", + "futures-util", + "jsonwebtoken 9.3.0", + "num_cpus", + "openssl", + "parking_lot", + "prometheus", + "rand", + "reqwest 0.12.9", + "rust_decimal", + "serde", + "serde_json", + "serde_prometheus", + "sysinfo", + "thiserror 2.0.8", + "time", + "tokio", + "tokio-tungstenite 0.24.0", + "tracing", + "tracing-actix-web", + "tracing-subscriber", + "url", + "urlencoding", + "uuid", + "validator", +] + [[package]] name = "symbolic-common" -version = "12.12.3" +version = "12.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ba5365997a4e375660bed52f5b42766475d5bc8ceb1bb13fea09c469ea0f49" +checksum = "cd33e73f154e36ec223c18013f7064a2c120f1162fc086ac9933542def186b00" dependencies = [ "debugid", "memmap2", @@ -9460,9 +9225,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.12.3" +version = "12.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beff338b2788519120f38c59ff4bb15174f52a183e547bac3d6072c2c0aa48aa" +checksum = "89e51191290147f071777e37fe111800bb82a9059f9c95b19d2dd41bfeddf477" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -9756,9 +9521,9 @@ dependencies = [ [[package]] name = "termtree" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "textwrap" @@ -9777,11 +9542,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "08f5383f3e0071702bf93ab5ee99b52d26936be9dedd9413067cbdcddcb6141a" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.8", ] [[package]] @@ -9797,9 +9562,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "f2f357fcec90b3caef6623a099691be676d033b40a058ac95d2a6ade6fa0c943" dependencies = [ "proc-macro2", "quote", @@ -9844,7 +9609,9 @@ checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", @@ -9986,7 +9753,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.19", + "rustls 0.23.20", "tokio", ] @@ -10111,7 +9878,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-timeout 0.4.1", "percent-encoding", "pin-project", @@ -10139,7 +9906,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-timeout 0.5.2", "hyper-util", "percent-encoding", @@ -10333,6 +10100,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", + "time", "tracing", "tracing-core", "tracing-log", @@ -10432,9 +10200,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" @@ -10457,12 +10225,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.0" @@ -10651,9 +10413,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -10662,13 +10424,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn 2.0.90", @@ -10677,9 +10438,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -10690,9 +10451,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10700,9 +10461,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -10713,9 +10474,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-encoder" @@ -10947,9 +10708,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -10980,16 +10741,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "wide" -version = "0.7.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e6db2670d2be78525979e9a5f9c69d296fd7d670549fe9ebf70f8708cb5019" -dependencies = [ - "bytemuck", - "safe_arch", -] - [[package]] name = "widestring" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1b17bb8e..a2ca7839 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,21 +33,31 @@ clap = { version = "4.5", features = ["derive", "env"] } dotenvy = "0.15" displaydoc = "0.2" futures = "0.3" -fuel-core-bin = { version = "0.40.2", features = ["p2p", "relayer", "rocksdb"] } -fuel-core = { version = "0.40.2", features = ["p2p", "relayer", "rocksdb"] } -fuel-core-client = { version = "0.40.2" } +fuel-core-bin = { version = "0.40.2", default-features = false, features = [ + "p2p", + "relayer", + "rocksdb", +] } +fuel-core = { version = "0.40.2", default-features = false, features = [ + "p2p", + "relayer", + "rocksdb", +] } +fuel-core-client = { version = "0.40.2", default-features = false, features = ["std"] } fuel-core-importer = { version = "0.40.2" } fuel-core-storage = { version = "0.40.2" } -fuel-core-types = { version = "0.40.2", features = ["test-helpers", "serde"] } -fuel-core-services = { version = "0.40.2" } +fuel-core-types = { version = "0.40.2", default-features = false, features = ["std", "serde"] } +fuel-core-services = { version = "0.40.2", default-features = false, features = ["test-helpers"] } futures-util = "0.3" itertools = "0.13" mockall = "0.13" mockall_double = "0.3.1" hex = "0.4" pretty_assertions = "1.4" +num_cpus = "1.16" rand = "0.8" serde = { version = "1.0", features = ["derive"] } +rayon = "1.10.0" serde_json = "1.0" sha2 = "0.10" strum = "0.26" @@ -63,11 +73,14 @@ fuel-streams = { path = "crates/fuel-streams" } fuel-networks = { path = "crates/fuel-networks" } fuel-data-parser = { version = "0.0.13", path = "crates/fuel-data-parser" } fuel-streams-core = { version = "0.0.13", path = "crates/fuel-streams-core" } -fuel-streams-publisher = { version = "0.0.13", path = "crates/fuel-streams-publisher" } -fuel-streams-ws = { version = "0.0.13", path = "crates/fuel-streams-ws" } +sv-webserver = { version = "0.0.13", path = "crates/sv-webserver" } fuel-streams-macros = { version = "0.0.13", path = "crates/fuel-streams-macros" } +fuel-streams-nats = { version = "0.0.13", path = "crates/fuel-streams-nats" } fuel-streams-storage = { version = "0.0.13", path = "crates/fuel-streams-storage" } subject-derive = { version = "0.0.13", path = "crates/fuel-streams-macros/subject-derive" } +fuel-streams-executors = { version = "0.0.13", path = "crates/fuel-streams-executors" } +sv-publisher = { version = "0.0.13", path = "crates/sv-publisher" } +sv-consumer = { version = "0.0.13", path = "crates/sv-consumer" } # Workspace projects [workspace.metadata.cargo-machete] diff --git a/Makefile b/Makefile index feaa6c3f..877935b0 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ RUST_VERSION := 1.81.0 clean clean-build cleanup-artifacts test-watch test bench helm-test \ fmt fmt-cargo fmt-rust fmt-prettier fmt-markdown lint lint-cargo \ lint-rust lint-clippy lint-prettier lint-markdown lint-machete \ - audit audit-fix-test audit-fix load-test run-publisher \ + audit audit-fix-test audit-fix load-test run-publisher run-consumer \ run-mainnet-dev run-mainnet-profiling run-testnet-dev run-testnet-profiling \ start-nats stop-nats restart-nats clean-nats minikube-setup minikube-start \ minikube-delete k8s-setup helm-setup cluster-setup pre-cluster \ @@ -112,6 +112,9 @@ clean-build: rm -rf target/ rm -rf node_modules/ +cleanup-artifacts: REPO_OWNER="fuellabs" +cleanup-artifacts: REPO_NAME="data-systems" +cleanup-artifacts: DAYS_TO_KEEP=10 cleanup-artifacts: @echo "Running artifact cleanup..." @./scripts/cleanup_artifacts.sh $(REPO_OWNER) $(REPO_NAME) $(DAYS_TO_KEEP) @@ -120,20 +123,24 @@ cleanup-artifacts: # Testing # ------------------------------------------------------------ +test-watch: PROFILE="all" test-watch: cargo watch -x "test --profile $(PROFILE)" +test: PACKAGE="all" +test: PROFILE="dev" test: + @echo "Running tests for package $(PACKAGE) with profile $(PROFILE)" @if [ "$(PACKAGE)" = "all" ] || [ -z "$(PACKAGE)" ]; then \ - cargo nextest run --cargo-profile $(PROFILE) --workspace --color always --locked --no-tests=pass && \ - cargo test --profile $(PROFILE) --doc --workspace; \ + cargo nextest run --cargo-profile $(PROFILE) --workspace --color always --no-tests=pass --all-features && \ + cargo test --profile $(PROFILE) --doc --workspace --all-features; \ else \ - cargo nextest run --cargo-profile $(PROFILE) -p $(PACKAGE) --color always --locked --no-tests=pass && \ - cargo test --profile $(PROFILE) --doc -p $(PACKAGE); \ + cargo nextest run --cargo-profile $(PROFILE) -p $(PACKAGE) --color always --no-tests=pass --all-features && \ + cargo test --profile $(PROFILE) --doc -p $(PACKAGE) --all-features; \ fi bench: - cargo bench -p data-parser -p nats-publisher -p bench-consumers + cargo bench -p data-parser helm-test: helm unittest -f "tests/**/*.yaml" -f "tests/*.yaml" cluster/charts/fuel-streams @@ -201,13 +208,14 @@ load-test: # Publisher Run Commands # ------------------------------------------------------------ +run-publisher: NETWORK="testnet" +run-publisher: MODE="dev" +run-publisher: PORT="4000" +run-publisher: TELEMETRY_PORT="8080" +run-publisher: NATS_URL="localhost:4222" +run-publisher: EXTRA_ARGS="" run-publisher: check-network - @./scripts/run_publisher.sh \ - --network $(NETWORK) \ - --mode $(MODE) \ - $(if $(PORT),--port $(PORT),) \ - $(if $(TELEMETRY_PORT),--telemetry-port $(TELEMETRY_PORT),) \ - $(if $(extra_args),--extra-args "$(extra_args)",) + @./scripts/run_publisher.sh run-publisher-mainnet-dev: $(MAKE) run-publisher NETWORK=mainnet MODE=dev @@ -221,46 +229,72 @@ run-publisher-testnet-dev: run-publisher-testnet-profiling: $(MAKE) run-publisher NETWORK=testnet MODE=profiling +# ------------------------------------------------------------ +# Consumer Run Commands +# ------------------------------------------------------------ + +run-consumer: NATS_URL="localhost:4222" +run-consumer: NATS_PUBLISHER_URL="localhost:4333" +run-consumer: + cargo run --package sv-consumer --profile dev -- \ + --nats-url $(NATS_URL) \ + --nats-publisher-url $(NATS_PUBLISHER_URL) + # ------------------------------------------------------------ # Streamer Run Commands # ------------------------------------------------------------ -run-streamer: check-network - @./scripts/run_streamer.sh \ - --mode $(MODE) \ - $(if $(extra_args),--extra-args "$(extra_args)",) +run-webserver: NETWORK="testnet" +run-webserver: MODE="dev" +run-webserver: PORT="9003" +run-webserver: NATS_URL="nats://localhost:4222" +run-webserver: EXTRA_ARGS="" +run-webserver: check-network + @./scripts/run_webserver.sh --mode $(MODE) --port $(PORT) --nats-url $(NATS_URL) --extra-args $(EXTRA_ARGS) -run-streamer-mainnet-dev: - $(MAKE) run-streamer NETWORK=mainnet MODE=dev +run-webserver-mainnet-dev: + $(MAKE) run-webserver NETWORK=mainnet MODE=dev -run-streamer-mainnet-profiling: - $(MAKE) run-streamer NETWORK=mainnet MODE=profiling +run-webserver-mainnet-profiling: + $(MAKE) run-webserver NETWORK=mainnet MODE=profiling -run-streamer-testnet-dev: - $(MAKE) run-streamer NETWORK=testnet MODE=dev +run-webserver-testnet-dev: + $(MAKE) run-webserver NETWORK=testnet MODE=dev -run-streamer-testnet-profiling: - $(MAKE) run-streamer NETWORK=testnet MODE=profiling +run-webserver-testnet-profiling: + $(MAKE) run-webserver NETWORK=testnet MODE=profiling # ------------------------------------------------------------ # Docker Compose # ------------------------------------------------------------ +# Define service profiles +DOCKER_SERVICES := nats localstack docker + +run-docker-compose: PROFILE="all" run-docker-compose: @./scripts/set_env.sh - @docker compose -f cluster/docker/docker-compose.yml --env-file .env $(COMMAND) + @docker compose -f cluster/docker/docker-compose.yml --profile $(PROFILE) --env-file .env $(COMMAND) + +# Common docker-compose commands +define make-docker-commands +start-$(1): + $(MAKE) run-docker-compose PROFILE="$(if $(filter docker,$(1)),all,$(1))" COMMAND="up -d" -start-nats: - $(MAKE) run-docker-compose COMMAND="up -d" +stop-$(1): + $(MAKE) run-docker-compose PROFILE="$(if $(filter docker,$(1)),all,$(1))" COMMAND="down" -stop-nats: - $(MAKE) run-docker-compose COMMAND="down" +restart-$(1): + $(MAKE) run-docker-compose PROFILE="$(if $(filter docker,$(1)),all,$(1))" COMMAND="restart" -restart-nats: - $(MAKE) run-docker-compose COMMAND="restart" +clean-$(1): + $(MAKE) run-docker-compose PROFILE="$(if $(filter docker,$(1)),all,$(1))" COMMAND="down -v --remove-orphans" -clean-nats: - $(MAKE) run-docker-compose COMMAND="down -v --rmi all --remove-orphans" +reset-$(1): clean-$(1) start-$(1) +endef + +# Generate targets for each service +$(foreach service,$(DOCKER_SERVICES),$(eval $(call make-docker-commands,$(service)))) # ------------------------------------------------------------ # Local cluster (Minikube) @@ -290,15 +324,10 @@ minikube-delete: @echo "Deleting minikube..." @minikube delete -k8s-setup: - @echo "Setting up k8s..." - @./cluster/scripts/setup_k8s.sh $(NAMESPACE) - helm-setup: @cd cluster/charts/fuel-streams && helm dependency update - @cd cluster/charts/fuel-streams-publisher && helm dependency update -cluster-setup: minikube-setup k8s-setup helm-setup +cluster-setup: minikube-setup helm-setup pre-cluster: @./scripts/set_env.sh @@ -311,5 +340,4 @@ cluster-up: pre-cluster cluster-down: pre-cluster CLUSTER_MODE=$(MODE) tilt --file ./Tiltfile down -cluster-reset: pre-cluster - CLUSTER_MODE=$(MODE) tilt --file ./Tiltfile reset +cluster-reset: cluster-down cluster-up diff --git a/Tiltfile b/Tiltfile index 2618c336..f81a64c0 100755 --- a/Tiltfile +++ b/Tiltfile @@ -10,15 +10,43 @@ version_settings(True) # Enable 'new version' banner # Load environment variables from .env file dotenv() -# Build publisher image with proper configuration for Minikube +allow_k8s_contexts('minikube') + +# Build sv-publisher +custom_build( + ref='sv-publisher:latest', + command=[ + './cluster/scripts/build_docker.sh', + '--dockerfile', './cluster/docker/sv-publisher.Dockerfile' + ], + deps=[ + './src', + './Cargo.toml', + './Cargo.lock', + './cluster/docker/sv-publisher.Dockerfile' + ], + live_update=[ + sync('./src', '/usr/src'), + sync('./Cargo.toml', '/usr/src/Cargo.toml'), + sync('./Cargo.lock', '/usr/src/Cargo.lock'), + run('cargo build', trigger=['./src', './Cargo.toml', './Cargo.lock']) + ], + ignore=['./target'] +) + +# Build sv-consumer custom_build( - ref='fuel-streams-publisher:latest', - command=['./cluster/scripts/build_publisher.sh'], + ref='sv-consumer:latest', + image_deps=['sv-publisher:latest'], + command=[ + './cluster/scripts/build_docker.sh', + '--dockerfile', './cluster/docker/sv-consumer.Dockerfile' + ], deps=[ './src', './Cargo.toml', './Cargo.lock', - './cluster/docker/fuel-streams-publisher.Dockerfile' + './cluster/docker/sv-consumer.Dockerfile' ], live_update=[ sync('./src', '/usr/src'), @@ -26,19 +54,22 @@ custom_build( sync('./Cargo.lock', '/usr/src/Cargo.lock'), run('cargo build', trigger=['./src', './Cargo.toml', './Cargo.lock']) ], - skips_local_docker=True, ignore=['./target'] ) # Build streamer ws image with proper configuration for Minikube custom_build( - ref='fuel-streams-ws:latest', - command=['./cluster/scripts/build_streamer.sh'], + ref='sv-webserver:latest', + image_deps=['sv-consumer:latest', 'sv-publisher:latest'], + command=[ + './cluster/scripts/build_docker.sh', + '--dockerfile', './cluster/docker/sv-webserver.Dockerfile' + ], deps=[ './src', './Cargo.toml', './Cargo.lock', - './docker/fuel-streams-ws.Dockerfile' + './cluster/docker/sv-webserver.Dockerfile' ], live_update=[ sync('./src', '/usr/src'), @@ -46,7 +77,6 @@ custom_build( sync('./Cargo.lock', '/usr/src/Cargo.lock'), run('cargo build', trigger=['./src', './Cargo.toml', './Cargo.lock']) ], - skips_local_docker=True, ignore=['./target'] ) @@ -57,50 +87,39 @@ config_mode = os.getenv('CLUSTER_MODE', 'full') # Resource configurations RESOURCES = { 'publisher': { - 'name': 'fuel-streams-publisher', - 'ports': ['4000:4000', '8080:8080'], + 'name': 'fuel-streams-sv-publisher', + 'ports': ['8080:8080'], 'labels': 'publisher', - 'config_mode': ['minimal', 'full'] + 'config_mode': ['minimal', 'full'], + 'deps': ['fuel-streams-nats-core', 'fuel-streams-nats-publisher'] + }, + 'consumer': { + 'name': 'fuel-streams-sv-consumer', + 'ports': ['8081:8080'], + 'labels': 'consumer', + 'config_mode': ['minimal', 'full'], + 'deps': ['fuel-streams-nats-core', 'fuel-streams-nats-publisher', 'fuel-streams-sv-publisher'] + }, + 'sv-webserver': { + 'name': 'fuel-streams-sv-webserver', + 'ports': ['9003:9003'], + 'labels': 'ws', + 'config_mode': ['minimal', 'full'], + 'deps': ['fuel-streams-nats-core', 'fuel-streams-nats-publisher'] }, 'nats-core': { 'name': 'fuel-streams-nats-core', - 'ports': ['4222:4222', '8222:8222'], - 'labels': 'nats', - 'config_mode': ['minimal', 'full'] - }, - 'nats-client': { - 'name': 'fuel-streams-nats-client', - 'ports': ['4223:4222', '8443:8443'], + 'ports': ['4222:4222', '6222:6222', '7422:7422'], 'labels': 'nats', 'config_mode': ['minimal', 'full'] }, 'nats-publisher': { 'name': 'fuel-streams-nats-publisher', - 'ports': ['4224:4222'], + 'ports': ['4333:4222', '6222:6222', '7433:7422'], 'labels': 'nats', - 'config_mode': ['minimal', 'full'] + 'config_mode': ['minimal', 'full'], + 'deps': ['fuel-streams-nats-core'] }, - # 'grafana': { - # 'name': 'fuel-streams-grafana', - # 'ports': ['3000:3000'], - # 'labels': 'monitoring', - # 'config_mode': ['minimal', 'full'] - # }, - # 'prometheus-operator': { - # 'name': 'fuel-streams-prometheus-operator', - # 'labels': 'monitoring', - # 'config_mode': ['minimal', 'full'] - # }, - # 'kube-state-metrics': { - # 'name': 'fuel-streams-kube-state-metrics', - # 'labels': 'monitoring', - # 'config_mode': ['minimal', 'full'] - # }, - # 'node-exporter': { - # 'name': 'fuel-streams-prometheus-node-exporter', - # 'labels': 'monitoring', - # 'config_mode': ['minimal', 'full'] - # } } k8s_yaml(helm( @@ -108,8 +127,9 @@ k8s_yaml(helm( name='fuel-streams', namespace='fuel-streams', values=[ - 'cluster/charts/fuel-streams/values-publisher-secrets.yaml', - 'cluster/charts/fuel-streams/values.yaml' + 'cluster/charts/fuel-streams/values.yaml', + 'cluster/charts/fuel-streams/values-local.yaml', + 'cluster/charts/fuel-streams/values-secrets.yaml' ] )) diff --git a/benches/bench-consumers/Cargo.toml b/benches/bench-consumers/Cargo.toml deleted file mode 100644 index fb0dfb5b..00000000 --- a/benches/bench-consumers/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "bench-consumers" -authors = { workspace = true } -keywords = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -version = { workspace = true } -rust-version = { workspace = true } -publish = false - -[[bench]] -name = "consumers" -harness = false -path = "benches/consumers.rs" - -[dependencies] -anyhow = { workspace = true } -async-nats = { workspace = true } -chrono = { workspace = true } -fuel-core-types = { workspace = true } -fuel-streams-core = { workspace = true, features = ["bench-helpers"] } -futures = { workspace = true } -nats-publisher = { path = "../nats-publisher" } -statrs = "0.18" -tokio = { workspace = true } - -[dev-dependencies] -criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } diff --git a/benches/bench-consumers/README.md b/benches/bench-consumers/README.md deleted file mode 100644 index ad650983..00000000 --- a/benches/bench-consumers/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Running - -1. After running the [`nats-publisher`](../nats-publisher/README.md) locally, just execute the run on this project: - - ```sh - cargo run - ``` - -2. You can also run benchmarks with cargo: - - ```sh - cargo bench --bench consumers - ``` diff --git a/benches/bench-consumers/benches/consumers.rs b/benches/bench-consumers/benches/consumers.rs deleted file mode 100644 index 8b66907f..00000000 --- a/benches/bench-consumers/benches/consumers.rs +++ /dev/null @@ -1,43 +0,0 @@ -use bench_consumers::runners::{ - runner_consumer::run_blocks_consumer, - runner_kv_watcher::run_watch_kv_blocks, - runner_subscription::run_subscriptions, -}; -use criterion::{criterion_group, criterion_main, Criterion}; -use nats_publisher::utils::nats::NatsHelper; -use tokio::runtime::Runtime; - -static MSGS_LIMIT: usize = 10000; - -fn benchmark_all(c: &mut Criterion) { - let rt = Runtime::new().unwrap(); - let mut group = c.benchmark_group("NATS Benchmarks"); - let nats = rt.block_on(async { NatsHelper::connect(false).await.unwrap() }); - - group.bench_function("consume_blocks_ack_none", |b| { - b.to_async(&rt).iter(|| async { - run_blocks_consumer(&nats, MSGS_LIMIT).await.unwrap() - }); - }); - - group.bench_function("watch_kv_blocks", |b| { - b.to_async(&rt).iter(|| async { - run_watch_kv_blocks(&nats, MSGS_LIMIT).await.unwrap() - }); - }); - - group.bench_function("subscriptions", |b| { - b.to_async(&rt).iter(|| async { - run_subscriptions(&nats, MSGS_LIMIT).await.unwrap() - }); - }); - - group.finish(); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(10); // Adjust sample size as needed - targets = benchmark_all -); -criterion_main!(benches); diff --git a/benches/bench-consumers/src/lib.rs b/benches/bench-consumers/src/lib.rs deleted file mode 100644 index 0edfafa9..00000000 --- a/benches/bench-consumers/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod runners; diff --git a/benches/bench-consumers/src/main.rs b/benches/bench-consumers/src/main.rs deleted file mode 100644 index a4bc6b64..00000000 --- a/benches/bench-consumers/src/main.rs +++ /dev/null @@ -1,10 +0,0 @@ -use runners::runner_all::run_all_benchmarks; - -mod runners; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - println!("Running benchmarks"); - run_all_benchmarks().await?; - Ok(()) -} diff --git a/benches/bench-consumers/src/runners/benchmark_results.rs b/benches/bench-consumers/src/runners/benchmark_results.rs deleted file mode 100644 index e4aaaf42..00000000 --- a/benches/bench-consumers/src/runners/benchmark_results.rs +++ /dev/null @@ -1,103 +0,0 @@ -use core::fmt; -use std::time::{Duration, Instant}; - -use chrono::{DateTime, Utc}; -use statrs::statistics::{Data, Distribution}; - -#[derive(Debug, Clone)] -pub struct BenchmarkResult { - pub name: String, - pub message_count: usize, - pub error_count: usize, - start_time: Instant, - pub elapsed_time: Option, - pub messages_per_second: Option, - pub publish_times: Vec, - pub mean_publish_time: Option, - pub messages_limit: usize, -} - -impl fmt::Display for BenchmarkResult { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "\n{}\nBenchmark Results: {}\n{}\nTotal Messages: {}\nTotal Errors: {}\nElapsed Time: {:?}\nMessages per Second: {:.2}\nMean Publish Time: {:?}\n{}", - "=".repeat(50), - self.name, - "=".repeat(50), - self.message_count, - self.error_count, - self.elapsed_time.unwrap_or_default(), - self.messages_per_second.unwrap_or_default(), - self.mean_publish_time.unwrap_or_default(), - "=".repeat(50) - ) - } -} - -impl BenchmarkResult { - pub fn new(name: String, messages_limit: usize) -> Self { - Self { - name, - message_count: 0, - error_count: 0, - start_time: Instant::now(), - elapsed_time: None, - messages_per_second: None, - publish_times: vec![], - mean_publish_time: None, - messages_limit, - } - } - - pub fn increment_message_count(&mut self) { - self.message_count += 1; - } - - pub fn increment_error_count(&mut self) { - self.error_count += 1; - } - - pub fn finalize(&mut self) -> &mut Self { - self.calculate_mean_publish_time(); - let elapsed = self.start_time.elapsed(); - self.elapsed_time = Some(elapsed); - self.messages_per_second = - Some(self.message_count as f64 / elapsed.as_secs_f64()); - self - } - - pub fn is_complete(&self) -> bool { - self.message_count + self.error_count >= self.messages_limit - } - - pub fn add_publish_time(&mut self, timestamp: u128) -> &mut Self { - let current_time = Utc::now(); - let publish_time = - DateTime::::from_timestamp_millis(timestamp as i64) - .expect("Invalid timestamp"); - let duration = current_time - .signed_duration_since(publish_time) - .to_std() - .expect("Duration calculation failed"); - - self.publish_times.push(duration); - self - } - - pub fn calculate_mean_publish_time(&mut self) { - if self.publish_times.is_empty() { - return; - } - - let times_ns: Vec = self - .publish_times - .iter() - .map(|d| d.as_nanos() as f64) - .collect(); - - let data = Data::new(times_ns); - let mean_ns = data.mean().unwrap(); - self.mean_publish_time = Some(Duration::from_nanos(mean_ns as u64)); - } -} diff --git a/benches/bench-consumers/src/runners/mod.rs b/benches/bench-consumers/src/runners/mod.rs deleted file mode 100644 index b1918cbe..00000000 --- a/benches/bench-consumers/src/runners/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod benchmark_results; -pub mod runner_all; -pub mod runner_consumer; -pub mod runner_kv_watcher; -pub mod runner_subscription; diff --git a/benches/bench-consumers/src/runners/runner_all.rs b/benches/bench-consumers/src/runners/runner_all.rs deleted file mode 100644 index 410e4570..00000000 --- a/benches/bench-consumers/src/runners/runner_all.rs +++ /dev/null @@ -1,25 +0,0 @@ -use anyhow::Result; -use nats_publisher::utils::nats::NatsHelper; -use tokio::try_join; - -static MSGS_LIMIT: usize = 5000; - -use super::{ - runner_consumer::run_blocks_consumer, - runner_kv_watcher::run_watch_kv_blocks, - runner_subscription::run_subscriptions, -}; - -#[allow(dead_code)] -pub async fn run_all_benchmarks() -> Result<()> { - let use_nats_compression = false; // adjust as needed - let nats = NatsHelper::connect(use_nats_compression).await?; - - let _ = try_join!( - run_subscriptions(&nats, MSGS_LIMIT), - run_watch_kv_blocks(&nats, MSGS_LIMIT), - run_blocks_consumer(&nats, MSGS_LIMIT), - ); - - Ok(()) -} diff --git a/benches/bench-consumers/src/runners/runner_consumer.rs b/benches/bench-consumers/src/runners/runner_consumer.rs deleted file mode 100644 index bbf02f23..00000000 --- a/benches/bench-consumers/src/runners/runner_consumer.rs +++ /dev/null @@ -1,55 +0,0 @@ -use anyhow::Result; -use async_nats::jetstream::consumer::AckPolicy; -pub use async_nats::jetstream::consumer::{ - pull::Config as PullConsumerConfig, - DeliverPolicy, -}; -use fuel_core_types::blockchain::block::Block; -use fuel_streams_core::prelude::StreamData; -use futures::StreamExt; -use nats_publisher::utils::nats::NatsHelper; - -use super::benchmark_results::BenchmarkResult; - -pub async fn run_blocks_consumer( - nats: &NatsHelper, - limit: usize, -) -> Result<()> { - let mut result = BenchmarkResult::new( - "Blocks Consumer (Ephemeral + AckNone)".into(), - limit, - ); - - let consumer = nats - .stream_blocks - .create_consumer(PullConsumerConfig { - deliver_policy: DeliverPolicy::New, - ack_policy: AckPolicy::None, - ..Default::default() - }) - .await?; - - let mut messages = consumer.messages().await?; - while let Some(message) = messages.next().await { - let msg = message?; - match nats - .data_parser() - .decode::>(&msg.payload) - .await - { - Err(_) => result.increment_error_count(), - Ok(decoded) => { - result - .add_publish_time(decoded.ts_as_millis()) - .increment_message_count(); - if result.is_complete() { - result.finalize(); - println!("{}", result); - break; - } - } - } - } - - Ok(()) -} diff --git a/benches/bench-consumers/src/runners/runner_kv_watcher.rs b/benches/bench-consumers/src/runners/runner_kv_watcher.rs deleted file mode 100644 index a4db8d73..00000000 --- a/benches/bench-consumers/src/runners/runner_kv_watcher.rs +++ /dev/null @@ -1,40 +0,0 @@ -use anyhow::Result; -use fuel_core_types::blockchain::block::Block; -use fuel_streams_core::prelude::StreamData; -use futures::StreamExt; -use nats_publisher::utils::nats::NatsHelper; - -use super::benchmark_results::BenchmarkResult; - -#[allow(dead_code)] -pub async fn run_watch_kv_blocks( - nats: &NatsHelper, - limit: usize, -) -> Result<()> { - let mut result = - BenchmarkResult::new("KV Blocks Watcher".to_string(), limit); - let mut watch = nats.kv_blocks.watch_all().await?; - - while let Some(message) = watch.next().await { - let item = message?; - match nats - .data_parser() - .decode::>(&item.value) - .await - { - Err(_) => result.increment_error_count(), - Ok(decoded) => { - result - .add_publish_time(decoded.ts_as_millis()) - .increment_message_count(); - if result.is_complete() { - result.finalize(); - println!("{}", result); - break; - } - } - } - } - - Ok(()) -} diff --git a/benches/bench-consumers/src/runners/runner_subscription.rs b/benches/bench-consumers/src/runners/runner_subscription.rs deleted file mode 100644 index 1ec3a7f6..00000000 --- a/benches/bench-consumers/src/runners/runner_subscription.rs +++ /dev/null @@ -1,35 +0,0 @@ -use anyhow::Result; -use fuel_core_types::blockchain::block::Block; -use fuel_streams_core::prelude::StreamData; -use futures::StreamExt; -use nats_publisher::utils::nats::NatsHelper; - -use super::benchmark_results::BenchmarkResult; - -#[allow(dead_code)] -pub async fn run_subscriptions(nats: &NatsHelper, limit: usize) -> Result<()> { - let mut result = BenchmarkResult::new("Pub/Sub".to_string(), limit); - let mut subscriber = nats.client.subscribe("blocks.sub.*").await?; - while let Some(message) = subscriber.next().await { - let payload = message.payload; - match nats - .data_parser() - .decode::>(&payload) - .await - { - Err(_) => result.increment_error_count(), - Ok(decoded) => { - result - .add_publish_time(decoded.ts_as_millis()) - .increment_message_count(); - if result.is_complete() { - result.finalize(); - println!("{}", result); - break; - } - } - } - } - - Ok(()) -} diff --git a/benches/load-tester/Cargo.toml b/benches/load-tester/Cargo.toml deleted file mode 100644 index 32f93f06..00000000 --- a/benches/load-tester/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "load-tester" -authors = { workspace = true } -keywords = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -version = { workspace = true } -rust-version = { workspace = true } -publish = false - -[dependencies] -anyhow = { workspace = true } -async-nats = { workspace = true } -chrono = { workspace = true } -clap = { workspace = true } -fuel-streams = { workspace = true } -fuel-streams-core = { workspace = true, features = ["bench-helpers"] } -futures = { workspace = true } -statrs = "0.18.0" -tokio = { workspace = true } diff --git a/benches/load-tester/README.md b/benches/load-tester/README.md deleted file mode 100644 index 984a28a6..00000000 --- a/benches/load-tester/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Running - -To run the load-test suite: - - ```sh - cargo run -- --network testnet --max-subscriptions 10 --step-size 1 - ``` - -Adjustments are to be applied based on the network, max-subscriptions and step-size. diff --git a/benches/load-tester/src/lib.rs b/benches/load-tester/src/lib.rs deleted file mode 100644 index 0edfafa9..00000000 --- a/benches/load-tester/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod runners; diff --git a/benches/load-tester/src/main.rs b/benches/load-tester/src/main.rs deleted file mode 100644 index b13a223c..00000000 --- a/benches/load-tester/src/main.rs +++ /dev/null @@ -1,16 +0,0 @@ -use clap::Parser; -use load_tester::runners::{cli::Cli, runner_all::LoadTesterEngine}; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); - println!("Running load test ..."); - let load_tester = LoadTesterEngine::new( - cli.network, - cli.max_subscriptions, - cli.step_size, - ); - load_tester.run().await?; - println!("Finished load testing!"); - Ok(()) -} diff --git a/benches/load-tester/src/runners/cli.rs b/benches/load-tester/src/runners/cli.rs deleted file mode 100644 index 46cb07ad..00000000 --- a/benches/load-tester/src/runners/cli.rs +++ /dev/null @@ -1,33 +0,0 @@ -use clap::Parser; -use fuel_streams::types::FuelNetwork; - -#[derive(Clone, Parser)] -pub struct Cli { - /// Fuel Network to connect to. - #[arg( - long, - value_name = "NETWORK", - env = "NETWORK", - default_value = "Local", - value_parser = clap::value_parser!(FuelNetwork) - )] - pub network: FuelNetwork, - /// Maximum subscriptions for load testing - #[arg( - long, - value_name = "MAXS", - env = "MAX_SUBS", - default_value = "10", - help = "Maximum subscriptions for load testing." - )] - pub max_subscriptions: u16, - /// Maximum step size for load testing - #[arg( - long, - value_name = "SSIZE", - env = "STEP_SIZE", - default_value = "1", - help = "Maximum step size for load testing." - )] - pub step_size: u16, -} diff --git a/benches/load-tester/src/runners/mod.rs b/benches/load-tester/src/runners/mod.rs deleted file mode 100644 index cc84e6f4..00000000 --- a/benches/load-tester/src/runners/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod cli; -pub mod results; -pub mod runner_all; -pub mod runner_streamable; diff --git a/benches/load-tester/src/runners/results.rs b/benches/load-tester/src/runners/results.rs deleted file mode 100644 index fe1b56b8..00000000 --- a/benches/load-tester/src/runners/results.rs +++ /dev/null @@ -1,118 +0,0 @@ -use core::fmt; -use std::{ - sync::{ - atomic::{AtomicUsize, Ordering}, - RwLock, - }, - time::{Duration, Instant}, -}; - -use chrono::{DateTime, Utc}; -use statrs::statistics::{Data, Distribution}; - -#[derive(Debug)] -pub struct LoadTestTracker { - pub name: String, - pub message_count: AtomicUsize, - pub error_count: AtomicUsize, - start_time: Instant, - pub elapsed_time: RwLock>, - pub messages_per_second: RwLock>, - pub publish_times: RwLock>, - pub mean_publish_time: RwLock>, -} - -impl fmt::Display for LoadTestTracker { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "\n{}\nLoadTest Results: {}\n{}\nTotal Messages: {}\nTotal Errors: {}\nElapsed Time: {:?}\nMessages per Second: {:.2}\nMean Publish Time: {:?}\n{}", - "=".repeat(50), - self.name, - "=".repeat(50), - self.message_count.load(Ordering::Relaxed), - self.error_count.load(Ordering::Relaxed), - self.elapsed_time.read().unwrap().unwrap_or_default(), - self.messages_per_second.read().unwrap().unwrap_or_default(), - self.mean_publish_time.read().unwrap().unwrap_or_default(), - "=".repeat(50) - ) - } -} - -impl LoadTestTracker { - pub fn new(name: String) -> Self { - Self { - name, - message_count: AtomicUsize::new(0), - error_count: AtomicUsize::new(0), - start_time: Instant::now(), - elapsed_time: RwLock::new(None), - messages_per_second: RwLock::new(None), - publish_times: RwLock::new(vec![]), - mean_publish_time: RwLock::new(None), - } - } - - pub fn increment_message_count(&self) { - self.message_count.fetch_add(1, Ordering::Relaxed); - } - - pub fn increment_error_count(&self) { - self.error_count.fetch_add(1, Ordering::Relaxed); - } - - pub fn refresh(&self) -> &Self { - self.calculate_mean_publish_time(); - - let elapsed = self.start_time.elapsed(); - let message_count = self.message_count.load(Ordering::Relaxed); - - if let Ok(mut elapsed_time) = self.elapsed_time.write() { - *elapsed_time = Some(elapsed); - } - - if let Ok(mut messages_per_second) = self.messages_per_second.write() { - *messages_per_second = - Some(message_count as f64 / elapsed.as_secs_f64()); - } - - self - } - - pub fn add_publish_time(&self, timestamp: u128) -> &Self { - let current_time = Utc::now(); - let publish_time = - DateTime::::from_timestamp_millis(timestamp as i64) - .expect("Invalid timestamp"); - let duration = current_time - .signed_duration_since(publish_time) - .to_std() - .expect("Duration calculation failed"); - - if let Ok(mut times) = self.publish_times.write() { - times.push(duration); - } - self - } - - pub fn calculate_mean_publish_time(&self) { - // Lock the mutex to access publish_times - let times = self.publish_times.read().unwrap(); - - if times.is_empty() { - return; - } - - let times_ns: Vec = - times.iter().map(|d| d.as_nanos() as f64).collect(); - drop(times); - - let data = Data::new(times_ns); - let mean_ns = data.mean().unwrap(); - - if let Ok(mut mean_publish_time) = self.mean_publish_time.write() { - *mean_publish_time = Some(Duration::from_nanos(mean_ns as u64)); - } - } -} diff --git a/benches/load-tester/src/runners/runner_all.rs b/benches/load-tester/src/runners/runner_all.rs deleted file mode 100644 index 21ee7815..00000000 --- a/benches/load-tester/src/runners/runner_all.rs +++ /dev/null @@ -1,247 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use anyhow::Result; -use fuel_streams::client::Client; -use fuel_streams_core::prelude::*; -use tokio::task::JoinHandle; - -use super::{ - results::LoadTestTracker, - runner_streamable::run_streamable_consumer, -}; - -pub struct LoadTesterEngine { - max_subscriptions: u16, - step_size: u16, - fuel_network: FuelNetwork, -} - -impl LoadTesterEngine { - pub fn new( - fuel_network: FuelNetwork, - max_subscriptions: u16, - step_size: u16, - ) -> Self { - Self { - fuel_network, - max_subscriptions, - step_size, - } - } -} - -impl LoadTesterEngine { - pub async fn run(&self) -> Result<(), anyhow::Error> { - let client = Client::connect(self.fuel_network).await?; - let mut handles: Vec> = vec![]; - // blocks - let blocks_test_tracker = - Arc::new(LoadTestTracker::new("Blocks Consumer".into())); - let blocks_test_tracker_printer = Arc::clone(&blocks_test_tracker); - - // inputs - let inputs_test_tracker = - Arc::new(LoadTestTracker::new("Inputs Consumer".into())); - let inputs_test_tracker_printer = Arc::clone(&inputs_test_tracker); - - // txs - let txs_test_tracker = - Arc::new(LoadTestTracker::new("Txs Consumer".into())); - let txs_test_tracker_printer = Arc::clone(&txs_test_tracker); - - // receipts - let receipts_test_tracker = - Arc::new(LoadTestTracker::new("Receipts Consumer".into())); - let receipts_test_tracker_printer = Arc::clone(&receipts_test_tracker); - - // utxos - let utxos_test_tracker = - Arc::new(LoadTestTracker::new("Utxos Consumer".into())); - let utxos_test_tracker_printer = Arc::clone(&utxos_test_tracker); - - // logs - let logs_test_tracker = - Arc::new(LoadTestTracker::new("Logs Consumer".into())); - let logs_test_tracker_printer = Arc::clone(&logs_test_tracker); - - // outputs - let outputs_test_tracker = - Arc::new(LoadTestTracker::new("Outputs Consumer".into())); - let outputs_test_tracker_printer = Arc::clone(&outputs_test_tracker); - - // print regularly the tracked metrics - handles.push(tokio::spawn(async move { - loop { - // blocks - blocks_test_tracker_printer.refresh(); - println!("{}", blocks_test_tracker_printer); - - // inputs - inputs_test_tracker_printer.refresh(); - println!("{}", inputs_test_tracker_printer); - - // txs - txs_test_tracker_printer.refresh(); - println!("{}", txs_test_tracker_printer); - - // utxos - utxos_test_tracker_printer.refresh(); - println!("{}", utxos_test_tracker_printer); - - // receipts - receipts_test_tracker_printer.refresh(); - println!("{}", receipts_test_tracker_printer); - - // outputs - outputs_test_tracker_printer.refresh(); - println!("{}", outputs_test_tracker_printer); - - // logs - logs_test_tracker_printer.refresh(); - println!("{}", logs_test_tracker_printer); - - // do a short pause - tokio::time::sleep(Duration::from_secs(5)).await; - } - })); - - // Incrementally increase subscriptions - for current_subs in - (1..=self.max_subscriptions).step_by(self.step_size as usize) - { - let client = client.clone(); - let blocks_test_tracker = Arc::clone(&blocks_test_tracker); - for _ in 0..current_subs { - // blocks - { - let client = client.clone(); - let blocks_test_tracker = Arc::clone(&blocks_test_tracker); - handles.push(tokio::spawn(async move { - if let Err(e) = run_streamable_consumer::( - &client, - blocks_test_tracker, - ) - .await - { - eprintln!( - "Error in blocks subscriptions - {:?}", - e - ); - } - })); - } - // logs - { - let client = client.clone(); - let logs_test_tracker = Arc::clone(&logs_test_tracker); - handles.push(tokio::spawn(async move { - if let Err(e) = run_streamable_consumer::( - &client, - logs_test_tracker, - ) - .await - { - eprintln!("Error in logs subscriptions - {:?}", e); - } - })); - } - // inputs - { - let client = client.clone(); - let inputs_test_tracker = Arc::clone(&inputs_test_tracker); - handles.push(tokio::spawn(async move { - if let Err(e) = run_streamable_consumer::( - &client, - inputs_test_tracker, - ) - .await - { - eprintln!( - "Error in inputs subscriptions - {:?}", - e - ); - } - })); - } - // txs - { - let client = client.clone(); - let txs_test_tracker = Arc::clone(&txs_test_tracker); - handles.push(tokio::spawn(async move { - if let Err(e) = run_streamable_consumer::( - &client, - txs_test_tracker, - ) - .await - { - eprintln!("Error in txs subscriptions - {:?}", e); - } - })); - } - // outputs - { - let client = client.clone(); - let outputs_test_tracker = - Arc::clone(&outputs_test_tracker); - handles.push(tokio::spawn(async move { - if let Err(e) = run_streamable_consumer::( - &client, - outputs_test_tracker, - ) - .await - { - eprintln!( - "Error in outputs subscriptions - {:?}", - e - ); - } - })); - } - // utxos - { - let client = client.clone(); - let utxos_test_tracker = Arc::clone(&utxos_test_tracker); - handles.push(tokio::spawn(async move { - if let Err(e) = run_streamable_consumer::( - &client, - utxos_test_tracker, - ) - .await - { - eprintln!("Error in utxos subscriptions - {:?}", e); - } - })); - } - // receipts - { - let client = client.clone(); - let receipts_test_tracker = - Arc::clone(&receipts_test_tracker); - handles.push(tokio::spawn(async move { - if let Err(e) = run_streamable_consumer::( - &client, - receipts_test_tracker, - ) - .await - { - eprintln!( - "Error in receipts subscriptions - {:?}", - e - ); - } - })); - } - } - - // Small pause between test iterations - tokio::time::sleep(Duration::from_secs(5)).await; - } - - // cleanup - for handle in handles.iter() { - handle.abort(); - } - - Ok(()) - } -} diff --git a/benches/load-tester/src/runners/runner_streamable.rs b/benches/load-tester/src/runners/runner_streamable.rs deleted file mode 100644 index bda6f2db..00000000 --- a/benches/load-tester/src/runners/runner_streamable.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::sync::Arc; - -use anyhow::Result; -pub use async_nats::jetstream::consumer::DeliverPolicy; -use fuel_streams::{client::Client, StreamConfig}; -use fuel_streams_core::prelude::*; -use futures::StreamExt; - -use super::results::LoadTestTracker; - -pub async fn run_streamable_consumer( - client: &Client, - load_test_tracker: Arc, -) -> Result<()> { - // Create a new stream for blocks - let stream = fuel_streams::Stream::::new(client).await; - - // Configure the stream to start from the last published block - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; - - // Subscribe to the block stream with the specified configuration - let mut sub = stream.subscribe_raw_with_config(config).await?; - - // Process incoming blocks - while let Some(bytes) = sub.next().await { - load_test_tracker.increment_message_count(); - let decoded_msg = S::decode_raw(bytes).unwrap(); - - let ts_millis = decoded_msg.ts_as_millis(); - load_test_tracker - .add_publish_time(ts_millis) - .increment_message_count(); - } - - Ok(()) -} diff --git a/benches/nats-publisher/README.md b/benches/nats-publisher/README.md deleted file mode 100644 index 1dbe8ad5..00000000 --- a/benches/nats-publisher/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Running - -1. First make sure you have your `.env` configured properly: - - ```sh - make create-env - ``` - -2. Make sure you have NATS server running within the workspace root: - - ```sh - make start-nats - ``` - -3. The, you can start local node and start publishing on NATS: - ```sh - make run-publisher - ``` diff --git a/benches/nats-publisher/config/nats.conf b/benches/nats-publisher/config/nats.conf deleted file mode 100644 index e2b7d425..00000000 --- a/benches/nats-publisher/config/nats.conf +++ /dev/null @@ -1,5 +0,0 @@ -jetstream: { - max_mem_store: 64MiB, - max_file_store: 10GiB -} -max_payload = 8388608 diff --git a/benches/nats-publisher/src/lib.rs b/benches/nats-publisher/src/lib.rs deleted file mode 100644 index 3bed9398..00000000 --- a/benches/nats-publisher/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[allow(unused)] -pub mod utils; diff --git a/benches/nats-publisher/src/main.rs b/benches/nats-publisher/src/main.rs deleted file mode 100644 index 62050a44..00000000 --- a/benches/nats-publisher/src/main.rs +++ /dev/null @@ -1,78 +0,0 @@ -mod utils; - -use clap::Parser; -use fuel_core_importer::ports::ImporterDatabase; -use fuel_streams_core::prelude::*; -use utils::{blocks::BlockHelper, nats::NatsHelper, tx::TxHelper}; - -#[derive(Parser)] -pub struct Cli { - #[command(flatten)] - fuel_core_config: fuel_core_bin::cli::run::Command, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - fuel_core_bin::cli::init_logging(); - - let cli = Cli::parse(); - let service = - fuel_core_bin::cli::run::get_service(cli.fuel_core_config).await?; - let chain_config = service.shared.config.snapshot_reader.chain_config(); - let chain_id = chain_config.consensus_parameters.chain_id(); - let block_importer = service.shared.block_importer.block_importer.clone(); - let database = service.shared.database.clone(); - - service - .start_and_await() - .await - .expect("Fuel core service startup failed"); - - // ------------------------------------------------------------------------ - // NATS - // ------------------------------------------------------------------------ - let nats = NatsHelper::connect(true).await?; - let block_helper = BlockHelper::new(nats.to_owned(), &database); - let tx_helper = TxHelper::new(nats.to_owned(), &chain_id, &database); - - // ------------------------------------------------------------------------ - // OLD BLOCKS - // ------------------------------------------------------------------------ - tokio::task::spawn({ - let database = database.clone(); - let block_helper = block_helper.clone(); - let _tx_helper = tx_helper.clone(); - let last_height = database.on_chain().latest_block_height()?.unwrap(); - async move { - for height in 0..*last_height { - let height = height.into(); - let block = block_helper.find_by_height(height); - let block = - Block::new(&block, Consensus::default(), Vec::new()); - - block_helper.publish(&block).await?; - // for (index, tx) in block.transactions().iter().enumerate() { - // tx_helper.publish(&block, tx, index).await?; - // } - } - Ok::<(), async_nats::Error>(()) - } - }); - - // ------------------------------------------------------------------------ - // NEW BLOCKS - // ------------------------------------------------------------------------ - let mut subscription = block_importer.subscribe(); - while let Ok(result) = subscription.recv().await { - let result = &**result; - let block = &result.sealed_block.entity; - let block = Block::new(block, Consensus::default(), Vec::new()); - - block_helper.publish(&block).await?; - // for (index, tx) in block.transactions().iter().enumerate() { - // tx_helper.publish(block, tx, index).await?; - // } - } - - Ok(()) -} diff --git a/benches/nats-publisher/src/utils/blocks.rs b/benches/nats-publisher/src/utils/blocks.rs deleted file mode 100644 index 3f192f7c..00000000 --- a/benches/nats-publisher/src/utils/blocks.rs +++ /dev/null @@ -1,93 +0,0 @@ -use async_nats::jetstream::context::Publish; -use fuel_core::combined_database::CombinedDatabase; -use fuel_core_storage::transactional::AtomicView; -use fuel_streams_core::prelude::*; -use tokio::try_join; -use tracing::info; - -use super::nats::NatsHelper; - -#[derive(Clone)] -pub struct BlockHelper { - nats: NatsHelper, - database: CombinedDatabase, -} - -impl BlockHelper { - pub fn new(nats: NatsHelper, database: &CombinedDatabase) -> Self { - Self { - nats, - database: database.to_owned(), - } - } - - pub fn find_by_height(&self, height: FuelCoreBlockHeight) -> FuelCoreBlock { - self.database - .on_chain() - .latest_view() - .unwrap() - .get_sealed_block_by_height(&height) - .unwrap() - .unwrap_or_else(|| { - panic!("NATS Publisher: no block at height {height}") - }) - .entity - } - - pub async fn publish(&self, block: &Block) -> anyhow::Result<()> { - try_join!( - self.publish_core(block), - self.publish_encoded(block), - self.publish_to_kv(block) - )?; - Ok(()) - } -} - -/// Publisher -impl BlockHelper { - async fn publish_core(&self, block: &Block) -> anyhow::Result<()> { - let subject: BlocksSubject = block.into(); - let payload = self.nats.data_parser().encode(block).await?; - self.nats - .context - .publish(subject.parse(), payload.into()) - .await?; - - Ok(()) - } - async fn publish_encoded(&self, block: &Block) -> anyhow::Result<()> { - let height = block.height; - let subject: BlocksSubject = block.into(); - let payload = self.nats.data_parser().encode(block).await?; - let nats_payload = Publish::build() - .message_id(subject.parse()) - .payload(payload.into()); - - self.nats - .context - .send_publish(subject.parse(), nats_payload) - .await? - .await?; - - info!( - "NATS: publishing block {} encoded to stream \"blocks_encoded\"", - height - ); - Ok(()) - } - - async fn publish_to_kv(&self, block: &Block) -> anyhow::Result<()> { - let height = block.height; - let subject: BlocksSubject = block.into(); - - let payload = self.nats.data_parser().encode(block).await?; - self.nats - .kv_blocks - .put(subject.parse(), payload.into()) - .await?; - - info!("NATS: publishing block {} to kv store \"blocks\"", height); - Ok(()) - } -} diff --git a/benches/nats-publisher/src/utils/mod.rs b/benches/nats-publisher/src/utils/mod.rs deleted file mode 100644 index 642417eb..00000000 --- a/benches/nats-publisher/src/utils/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod blocks; -pub mod nats; -pub mod tx; diff --git a/benches/nats-publisher/src/utils/nats.rs b/benches/nats-publisher/src/utils/nats.rs deleted file mode 100644 index 2a6e3ad6..00000000 --- a/benches/nats-publisher/src/utils/nats.rs +++ /dev/null @@ -1,130 +0,0 @@ -use async_nats::{ - jetstream::{ - kv::{self, Store}, - stream::{self, Compression, Stream}, - Context, - }, - ConnectOptions, -}; -use fuel_data_parser::DataParser; -use fuel_streams_core::prelude::FuelNetwork; - -#[allow(dead_code)] -#[derive(Clone)] -pub struct NatsHelper { - pub client: async_nats::Client, - pub kv_blocks: Store, - pub kv_transactions: Store, - pub context: Context, - pub stream_blocks: Stream, - pub stream_transactions: Stream, - pub use_nats_compression: bool, - pub data_parser: DataParser, -} - -impl NatsHelper { - pub async fn connect(use_nats_compression: bool) -> anyhow::Result { - let client = connect().await?; - let ( - context, - kv_blocks, - kv_transactions, - stream_blocks, - stream_transactions, - ) = create_resources(&client, use_nats_compression).await?; - // adjust as needed - let data_parser = DataParser::default(); - - Ok(Self { - client, - context, - kv_blocks, - kv_transactions, - stream_blocks, - stream_transactions, - use_nats_compression, - data_parser, - }) - } - - #[allow(dead_code)] - pub fn data_parser(&self) -> &DataParser { - &self.data_parser - } - - #[allow(dead_code)] - pub fn data_parser_mut(&mut self) -> &mut DataParser { - &mut self.data_parser - } -} - -pub async fn connect() -> anyhow::Result { - Ok(ConnectOptions::new() - .user_and_password("admin".into(), "secret".into()) - .connect(FuelNetwork::Local.to_nats_url()) - .await?) -} - -async fn create_resources( - client: &async_nats::Client, - use_nats_compression: bool, -) -> anyhow::Result<(Context, Store, Store, Stream, Stream)> { - let jetstream = async_nats::jetstream::new(client.clone()); - - // ------------------------------------------------------------------------ - // BLOCKS - // ------------------------------------------------------------------------ - let stream_blocks = jetstream - .get_or_create_stream(stream::Config { - name: "blocks_encoded".into(), - subjects: vec!["blocks.>".into()], - compression: if use_nats_compression { - Some(Compression::S2) - } else { - None - }, - ..Default::default() - }) - .await?; - - // TRANSACTIONS - // ------------------------------------------------------------------------ - let stream_transactions = jetstream - .get_or_create_stream(stream::Config { - name: "transactions_encoded".into(), - subjects: vec!["transactions.>".into()], - compression: if use_nats_compression { - Some(Compression::S2) - } else { - None - }, - ..Default::default() - }) - .await?; - - // KV STORE - // ------------------------------------------------------------------------ - let kv_blocks = jetstream - .create_key_value(kv::Config { - compression: use_nats_compression, - bucket: "blocks".into(), - ..Default::default() - }) - .await?; - - let kv_transactions = jetstream - .create_key_value(kv::Config { - compression: use_nats_compression, - bucket: "transactions".into(), - ..Default::default() - }) - .await?; - - Ok(( - jetstream, - kv_blocks, - kv_transactions, - stream_blocks, - stream_transactions, - )) -} diff --git a/benches/nats-publisher/src/utils/tx.rs b/benches/nats-publisher/src/utils/tx.rs deleted file mode 100644 index 49da5f26..00000000 --- a/benches/nats-publisher/src/utils/tx.rs +++ /dev/null @@ -1,129 +0,0 @@ -use async_nats::jetstream::context::Publish; -use fuel_core::combined_database::CombinedDatabase; -use fuel_core_types::fuel_types::ChainId; -use fuel_streams_core::prelude::*; -use tokio::try_join; -use tracing::info; - -use super::nats::NatsHelper; - -#[allow(unused)] -#[derive(Clone)] -pub struct TxHelper { - nats: NatsHelper, - chain_id: ChainId, - database: CombinedDatabase, -} - -#[allow(unused)] -/// Public -impl TxHelper { - pub fn new( - nats: NatsHelper, - chain_id: &ChainId, - database: &CombinedDatabase, - ) -> Self { - Self { - nats, - chain_id: chain_id.to_owned(), - database: database.to_owned(), - } - } - - pub async fn publish( - &self, - block: &Block, - tx: &Transaction, - index: usize, - ) -> anyhow::Result<()> { - try_join!( - self.publish_core(block, tx, index), - self.publish_encoded(block, tx, index), - self.publish_to_kv(block, tx, index) - )?; - Ok(()) - } -} - -/// Publishers -impl TxHelper { - async fn publish_core( - &self, - block: &Block, - tx: &Transaction, - index: usize, - ) -> anyhow::Result<()> { - let subject = &self.get_subject(tx, block, index); - let payload = self.nats.data_parser().encode(block).await?; - self.nats - .context - .publish(subject.parse(), payload.into()) - .await?; - Ok(()) - } - - async fn publish_encoded( - &self, - block: &Block, - tx: &Transaction, - index: usize, - ) -> anyhow::Result<()> { - let tx_id = &tx.id; - let subject = self.get_subject(tx, block, index); - let payload = self.nats.data_parser().encode(block).await?; - let nats_payload = Publish::build() - .message_id(subject.parse()) - .payload(payload.into()); - - self.nats - .context - .send_publish(subject.parse(), nats_payload) - .await? - .await?; - - info!( - "NATS: publishing transaction {} json to stream \"transactions_encoded\"", - tx_id - ); - Ok(()) - } - - async fn publish_to_kv( - &self, - block: &Block, - tx: &Transaction, - index: usize, - ) -> anyhow::Result<()> { - let tx_id = &tx.id; - let subject = self.get_subject(tx, block, index); - let payload = self.nats.data_parser().encode(block).await?; - self.nats - .kv_transactions - .put(subject.parse(), payload.into()) - .await?; - - info!( - "NATS: publishing transaction {} to kv store \"transactions\"", - tx_id - ); - Ok(()) - } -} - -/// Getters -impl TxHelper { - fn get_subject( - &self, - tx: &Transaction, - block: &Block, - index: usize, - ) -> TransactionsSubject { - // construct tx subject - let mut subject: TransactionsSubject = tx.into(); - subject = subject - .with_index(Some(index)) - .with_block_height(Some(BlockHeight::from(block.height))) - .with_status(Some(tx.status.clone())); - subject - } -} diff --git a/cluster/README.md b/cluster/README.md index 389ef86f..835541bd 100755 --- a/cluster/README.md +++ b/cluster/README.md @@ -10,15 +10,15 @@ The latter is intended for local development, but it also allows us to deploy th The following are prerequisites for spinning up the fuel-data-systems cluster locally: -- kubectl - `https://www.howtoforge.com/how-to-install-kubernetes-with-minikube-ubuntu-20-04/` +- kubectl + `https://www.howtoforge.com/how-to-install-kubernetes-with-minikube-ubuntu-20-04/` -- Tilt: - `https://docs.tilt.dev/install.html` +- Tilt: + `https://docs.tilt.dev/install.html` -- minikube based on the following description: - `https://phoenixnap.com/kb/install-minikube-on-ubuntu` - `https://minikube.sigs.k8s.io/docs/start/` +- minikube based on the following description: + `https://phoenixnap.com/kb/install-minikube-on-ubuntu` + `https://minikube.sigs.k8s.io/docs/start/` ...or alternatively use this tool which will automatically set up your cluster: `https://github.com/tilt-dev/ctlptl##minikube-with-a-built-in-registry` @@ -26,32 +26,36 @@ The following are prerequisites for spinning up the fuel-data-systems cluster lo ## Setup 1. To setup and start the local environment, run: - ```bash - make cluster-setup # Sets up both minikube and kubernetes configuration - ``` - Alternatively, you can run the setup steps individually: - ```bash - make minikube-setup # Sets up minikube with required addons - make k8s-setup # Configures kubernetes with proper namespace and context - ``` + ```bash + make cluster-setup # Sets up both minikube and kubernetes configuration + ``` - You can also start the minikube cluster without running the setup script: - ```bash - make minikube-start # Start minikube cluster - ``` + Alternatively, you can run the setup steps individually: + + ```bash + make minikube-setup # Sets up minikube with required addons + make k8s-setup # Configures kubernetes with proper namespace and context + ``` + + You can also start the minikube cluster without running the setup script: + + ```bash + make minikube-start # Start minikube cluster + ``` 2. Start the Tilt services: - ```bash - make cluster-up # Starts Tiltfile services - ``` + ```bash + make cluster-up # Starts Tiltfile services + ``` You can use the following commands to manage the services: + ```bash -make cluster-up # Start services -make cluster-down # Stop services -make cluster-reset # Reset services -make minikube-start # Start minikube (if you've already run setup before) +make cluster-up # Start services +make cluster-down # Stop services +make cluster-reset # Reset services +make minikube-start # Start minikube (if you've already run setup before) ``` ## Using `k9s` for an interactive terminal UI @@ -62,9 +66,9 @@ Run it with `k9s --context= --namespace=.yaml`. -- Tilt [tutorial](https://docs.tilt.dev/tutorial.html) +- How [kubernetes works](https://www.youtube.com/watch?v=ZuIQurh_kDk) +- Kubernetes [concepts](https://kubernetes.io/docs/concepts/) +- Kubectl [overview](https://kubernetes.io/docs/reference/kubectl/overview/) +- Kubectl [cheat sheet](https://kubernetes.io/docs/reference/kubectl/cheatsheet/) +- Helm [chart tutorial](https://docs.bitnami.com/kubernetes/how-to/create-your-first-helm-chart/), then examine the helm charts in this repository, and the values yaml files that are used to template them. The defults values are in the charts themselves as `values.yaml`, and the values for specific configurations are at `values/.yaml`. +- Tilt [tutorial](https://docs.tilt.dev/tutorial.html) diff --git a/cluster/charts/fuel-streams-publisher/.helmignore b/cluster/charts/fuel-streams-publisher/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/cluster/charts/fuel-streams-publisher/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/cluster/charts/fuel-streams-publisher/Chart.yaml b/cluster/charts/fuel-streams-publisher/Chart.yaml deleted file mode 100644 index 1628fc80..00000000 --- a/cluster/charts/fuel-streams-publisher/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: fuel-streams-publisher -description: A Helm chart for Kubernetes deployment of Fuel streams publisher service -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.7 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: 0.2.0 diff --git a/cluster/charts/fuel-streams-publisher/templates/_helpers.tpl b/cluster/charts/fuel-streams-publisher/templates/_helpers.tpl deleted file mode 100644 index 0bc5f455..00000000 --- a/cluster/charts/fuel-streams-publisher/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "fuel-streams-publisher.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "fuel-streams-publisher.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "fuel-streams-publisher.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "fuel-streams-publisher.labels" -}} -helm.sh/chart: {{ include "fuel-streams-publisher.chart" . }} -{{ include "fuel-streams-publisher.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "fuel-streams-publisher.selectorLabels" -}} -app.kubernetes.io/name: {{ include "fuel-streams-publisher.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "fuel-streams-publisher.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "fuel-streams-publisher.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/cluster/charts/fuel-streams-publisher/templates/hpa.yaml b/cluster/charts/fuel-streams-publisher/templates/hpa.yaml deleted file mode 100644 index b2759de2..00000000 --- a/cluster/charts/fuel-streams-publisher/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "fuel-streams-publisher.fullname" . }} - labels: - {{- include "fuel-streams-publisher.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "fuel-streams-publisher.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/cluster/charts/fuel-streams-publisher/templates/service.yaml b/cluster/charts/fuel-streams-publisher/templates/service.yaml deleted file mode 100644 index 49c7f6fe..00000000 --- a/cluster/charts/fuel-streams-publisher/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service - -metadata: - name: {{ include "fuel-streams-publisher.fullname" . }} - labels: - {{- include "fuel-streams-publisher.labels" . | nindent 4 }} - -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "fuel-streams-publisher.selectorLabels" . | nindent 4 }} diff --git a/cluster/charts/fuel-streams-publisher/templates/serviceaccount.yaml b/cluster/charts/fuel-streams-publisher/templates/serviceaccount.yaml deleted file mode 100644 index b3c456bb..00000000 --- a/cluster/charts/fuel-streams-publisher/templates/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "fuel-streams-publisher.serviceAccountName" . }} - labels: - {{- include "fuel-streams-publisher.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} -{{- end }} diff --git a/cluster/charts/fuel-streams-publisher/templates/statefulset.yaml b/cluster/charts/fuel-streams-publisher/templates/statefulset.yaml deleted file mode 100644 index b0d6b039..00000000 --- a/cluster/charts/fuel-streams-publisher/templates/statefulset.yaml +++ /dev/null @@ -1,147 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "fuel-streams-publisher.fullname" . }} - labels: - {{- include "fuel-streams-publisher.labels" . | nindent 4 }} -spec: - # Define the headless service that governs this StatefulSet - serviceName: {{ include "fuel-streams-publisher.fullname" . | quote }} - # Handle replica count unless autoscaling is enabled - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.config.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "fuel-streams-publisher.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - # Prometheus scraping configuration - {{- if .Values.prometheus.enabled }} - prometheus.io/scrape: {{ .Values.prometheus.scrape | quote }} - prometheus.io/port: {{ .Values.service.port | quote }} - prometheus.io/path: {{ .Values.prometheus.path | quote }} - {{- end }} - # Add checksums to force pod restart when configs change - {{/* checksum/config: {{ include (print $.Template.BasePath "/env-configmap.yaml") . | sha256sum }} */}} - {{/* checksum/secrets: {{ include (print $.Template.BasePath "/env-secrets.yaml") . | sha256sum }} */}} - {{- with .Values.config.annotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "fuel-streams-publisher.labels" . | nindent 8 }} - {{- with .Values.config.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.config.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "fuel-streams-publisher.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.config.podSecurityContext | nindent 8 }} - # Initialize persistent volumes with correct permissions - {{- if .Values.persistence.enabled }} - initContainers: - - name: {{ .Values.persistence.data.containerName }} - image: alpine:latest - imagePullPolicy: IfNotPresent - command: ["/bin/chown"] - args: ["-R", "1000:1000", "{{ .Values.persistence.data.mountPath }}"] - volumeMounts: - - name: {{ .Values.persistence.data.name }} - mountPath: {{ .Values.persistence.data.mountPath }} - - name: {{ .Values.persistence.temp.containerName }} - image: alpine:latest - imagePullPolicy: IfNotPresent - command: ["/bin/chown"] - args: ["-R", "1000:1000", "{{ .Values.persistence.temp.mountPath }}"] - volumeMounts: - - name: {{ .Values.persistence.temp.name }} - mountPath: {{ .Values.persistence.temp.mountPath }} - {{- end }} - # Main application container - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - # Define container ports for application and metrics - ports: - - name: http - containerPort: {{ int .Values.service.port }} - protocol: TCP - # Health check probes - livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} - resources: - {{- toYaml .Values.config.resources | nindent 12 }} - env: - {{- range $key, $value := .Values.env }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - {{- if .Values.extraEnv }} - {{- toYaml .Values.extraEnv | nindent 12 }} - {{- end }} - envFrom: - - configMapRef: - name: {{ include "fuel-streams-publisher.fullname" . }} - optional: true - - secretRef: - name: {{ include "fuel-streams-publisher.fullname" . }} - optional: true - {{- if .Values.envFrom }} - {{- toYaml .Values.envFrom | nindent 12 }} - {{- end }} - - # Mount persistent volumes if enabled - {{- if .Values.persistence.enabled }} - volumeMounts: - - name: {{ .Values.persistence.data.name }} - mountPath: {{ .Values.persistence.data.mountPath }} - readOnly: false - - name: {{ .Values.persistence.temp.name }} - mountPath: {{ .Values.persistence.temp.mountPath }} - readOnly: false - {{- end }} - # Node assignment rules - {{- with .Values.config.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.config.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.config.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - # Persistent volume claims configuration - {{- if .Values.persistence.enabled }} - volumeClaimTemplates: - - metadata: - name: {{ .Values.persistence.data.name }} - spec: - accessModes: - - {{ .Values.persistence.data.accessMode }} - storageClassName: {{ .Values.persistence.data.storageClass }} - resources: - requests: - storage: {{ .Values.persistence.data.size }} - - metadata: - name: {{ .Values.persistence.temp.name }} - spec: - accessModes: - - {{ .Values.persistence.temp.accessMode }} - storageClassName: {{ .Values.persistence.temp.storageClass }} - resources: - requests: - storage: {{ .Values.persistence.temp.size }} - {{- end }} diff --git a/cluster/charts/fuel-streams-publisher/values.yaml b/cluster/charts/fuel-streams-publisher/values.yaml deleted file mode 100644 index 91ca6eef..00000000 --- a/cluster/charts/fuel-streams-publisher/values.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Default values for fuel-streams-publisher -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# These are custom resource definitions that can be overridden by the user -# nameOverride: "" -# fullnameOverride: "" - -# general configurations -config: - replicaCount: 1 - imagePullSecrets: [] - annotations: {} - labels: {} - nodeSelector: {} - tolerations: [] - affinity: {} - resources: {} - podSecurityContext: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # resources: - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -image: - repository: ghcr.io/fuellabs/fuel-streams-publisher - pullPolicy: Always - tag: "latest" - -serviceAccount: - create: true - automount: true - # annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - # name: "" - -service: - type: ClusterIP - port: 8080 - -prometheus: - enabled: true - scrape: true - path: /metrics - -securityContext: - capabilities: - drop: [ALL] - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - -livenessProbe: {} -readinessProbe: {} - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 3 - targetCPUUtilizationPercentage: 80 - targetMemoryUtilizationPercentage: 80 - -persistence: - enabled: true - data: - name: rocks-db-vol - containerName: update-rocks-db-vol - mountPath: /mnt/db - size: 500Gi - storageClass: gp3-generic - accessMode: ReadWriteOnce - temp: - name: tmp-vol - containerName: update-tmp-vol - mountPath: /tmp - size: 5Gi - storageClass: gp3-generic - accessMode: ReadWriteOnce - -# Additional environment variables with complex structures -# extraEnv: -# - name: RELAYER -# valueFrom: -# secretKeyRef: -# name: fuel-streams-publisher -# key: RELAYER -# - name: KEYPAIR -# valueFrom: -# secretKeyRef: -# name: fuel-streams-publisher -# key: KEYPAIR -# - name: NATS_ADMIN_PASS -# valueFrom: -# secretKeyRef: -# name: fuel-streams-publisher -# key: NATS_ADMIN_PASS -# Optional: Bulk environment references -# envFrom: -# - configMapRef: -# name: additional-config -# - secretRef: -# name: additional-secrets - -env: - RELAYER_V2_LISTENING_CONTRACTS: "0xAEB0c00D0125A8a788956ade4f4F12Ead9f65DDf" - RELAYER_DA_DEPLOY_HEIGHT: "20620434" - RELAYER_LOG_PAGE_SIZE: "2000" - SYNC_HEADER_BATCH_SIZE: "100" - P2P_PORT: "30333" - RESERVED_NODES: "/dnsaddr/mainnet.fuel.network" - CHAIN_CONFIG: "mainnet" - PUBLISHER_MAX_THREADS: "32" - DB_PATH: "/mnt/db/" - POA_INSTANT: "false" - SERVICE_NAME: "NATS Publisher Node" - NATS_URL: "nats:4222" diff --git a/cluster/charts/fuel-streams/Chart.lock b/cluster/charts/fuel-streams/Chart.lock index 44606f01..3b1e4e2c 100644 --- a/cluster/charts/fuel-streams/Chart.lock +++ b/cluster/charts/fuel-streams/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ - version: 1.2.6 + version: 1.2.8 - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ - version: 1.2.6 + version: 1.2.8 - name: nats repository: https://nats-io.github.io/k8s/helm/charts/ - version: 1.2.6 -digest: sha256:ad7948ef2413ea2f9af239829570612f04abd624844413ad73600fd67ce2e9b6 -generated: "2024-12-08T20:11:41.144678-03:00" + version: 1.2.8 +digest: sha256:a5f3dd64e1a20f7c9d58894359f6f909f33d14772355ee70033fd411219bcc7e +generated: "2024-12-18T16:59:13.903435-03:00" diff --git a/cluster/charts/fuel-streams/Chart.yaml b/cluster/charts/fuel-streams/Chart.yaml index 398418cc..228de229 100755 --- a/cluster/charts/fuel-streams/Chart.yaml +++ b/cluster/charts/fuel-streams/Chart.yaml @@ -2,20 +2,15 @@ apiVersion: v2 appVersion: "1.0" description: A Helm chart for Kubernetes name: fuel-streams -version: 0.1.3 +version: 0.7.2 dependencies: - name: nats - version: 1.2.6 + version: 1.2.8 repository: https://nats-io.github.io/k8s/helm/charts/ alias: nats-core condition: nats-core.enabled - name: nats - version: 1.2.6 + version: 1.2.8 repository: https://nats-io.github.io/k8s/helm/charts/ alias: nats-publisher condition: nats-publisher.enabled - - name: nats - version: 1.2.6 - repository: https://nats-io.github.io/k8s/helm/charts/ - alias: nats-client - condition: nats-client.enabled diff --git a/cluster/charts/fuel-streams/templates/_blocks.tpl b/cluster/charts/fuel-streams/templates/_blocks.tpl index fe70c729..580c16e6 100644 --- a/cluster/charts/fuel-streams/templates/_blocks.tpl +++ b/cluster/charts/fuel-streams/templates/_blocks.tpl @@ -57,4 +57,56 @@ readinessProbe: startupProbe: {{- include "merge" (dict "context" .context "service" .service "defaultKey" "startupProbe" "path" "config.startupProbe") | nindent 2 }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} + +{{/* +Configure nats accounts +*/}} +{{- define "nats-accounts" -}} +data: + auth.conf: | + accounts { + SYS: { + users: [ + {user: $NATS_SYSTEM_USER, password: $NATS_SYSTEM_PASS} + ] + } + ADMIN: { + jetstream: enabled + users: [ + {user: $NATS_ADMIN_USER, password: $NATS_ADMIN_PASS} + ] + } + PUBLIC: { + jetstream: enabled + users: [ + { + user: $NATS_PUBLIC_USER + password: $NATS_PUBLIC_PASS + permissions: { + subscribe: ">" + publish: { + deny: [ + "*.by_id.>" + "*.blocks.>" + "*.transactions.>" + "*.inputs.>" + "*.outputs.>" + "*.receipts.>" + "*.logs.>" + "*.utxos.>" + "$JS.API.STREAM.CREATE.>" + "$JS.API.STREAM.UPDATE.>" + "$JS.API.STREAM.DELETE.>" + "$JS.API.STREAM.PURGE.>" + "$JS.API.STREAM.RESTORE.>" + "$JS.API.STREAM.MSG.DELETE.>" + "$JS.API.CONSUMER.DURABLE.CREATE.>" + ] + } + } + } + ] + } + } +{{- end }} diff --git a/cluster/charts/fuel-streams/templates/_helpers.tpl b/cluster/charts/fuel-streams/templates/_helpers.tpl index 4344d0aa..e3de60d9 100644 --- a/cluster/charts/fuel-streams/templates/_helpers.tpl +++ b/cluster/charts/fuel-streams/templates/_helpers.tpl @@ -3,6 +3,16 @@ Expand the name of the chart. If nameOverride is provided in Values.config, use that instead of .Chart.Name. The result is truncated to 63 chars and has any trailing "-" removed to comply with Kubernetes naming rules. Returns: String - The chart name, truncated and cleaned +Example: + Given: + .Chart.Name = "fuel-streams" + .Values.config.nameOverride = "custom-name" + Result: "custom-name" + + Given: + .Chart.Name = "fuel-streams" + .Values.config.nameOverride = null + Result: "fuel-streams" */}} {{- define "fuel-streams.name" -}} {{- default .Chart.Name .Values.config.nameOverride | trunc 63 | trimSuffix "-" }} @@ -17,6 +27,24 @@ This template follows these rules: - If not, concatenate release name and chart name with a hyphen The result is truncated to 63 chars and has any trailing "-" removed to comply with Kubernetes naming rules. Returns: String - The fully qualified app name, truncated and cleaned +Example: + Given: + .Values.config.fullnameOverride = "override-name" + Result: "override-name" + + Given: + .Release.Name = "my-release" + .Chart.Name = "fuel-streams" + .Values.config.nameOverride = null + .Values.config.fullnameOverride = null + Result: "my-release-fuel-streams" + + Given: + .Release.Name = "fuel-streams-prod" + .Chart.Name = "fuel-streams" + .Values.config.nameOverride = null + .Values.config.fullnameOverride = null + Result: "fuel-streams-prod" */}} {{- define "fuel-streams.fullname" -}} {{- if .Values.config.fullnameOverride }} @@ -49,26 +77,43 @@ Includes: - Selector labels (app name and instance) - App version (if defined) - Managed-by label indicating Helm management +Parameters: + - name: Optional custom name to use instead of the default name + - .: Full context (passed automatically or as "context") Returns: Map - A set of key-value pairs representing Kubernetes labels +Example: + {{- include "fuel-streams.labels" . }} + # Or with custom name: + {{- include "fuel-streams.labels" (dict "name" "custom-name" "context" $) }} */}} {{- define "fuel-streams.labels" -}} -helm.sh/chart: {{ include "fuel-streams.chart" . }} -{{ include "fuel-streams.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- $context := default . .context -}} +helm.sh/chart: {{ include "fuel-streams.chart" $context }} +{{ include "fuel-streams.selectorLabels" (dict "name" .name "context" $context) }} +{{- if $context.Chart.AppVersion }} +app.kubernetes.io/version: {{ $context.Chart.AppVersion | quote }} {{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/managed-by: {{ $context.Release.Service }} {{- end }} {{/* Selector labels Core identifying labels used for object selection and service discovery. These labels should be used consistently across all related resources. +Parameters: + - name: Optional custom name to use instead of the default name + - .: Full context (passed automatically or as "context") Returns: Map - A set of key-value pairs for Kubernetes selector labels +Example: + {{- include "fuel-streams.selectorLabels" . }} + # Or with custom name: + {{- include "fuel-streams.selectorLabels" (dict "name" "custom-name" "context" $) }} */}} {{- define "fuel-streams.selectorLabels" -}} -app.kubernetes.io/name: {{ include "fuel-streams.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} +{{- $context := default . .context -}} +{{- $name := default (include "fuel-streams.name" $context) .name -}} +app.kubernetes.io/name: {{ $name }} +app.kubernetes.io/instance: {{ $context.Release.Name }} {{- end }} {{/* @@ -172,4 +217,4 @@ Returns: Value if it exists and is not empty {{- if and $value (not (empty $value)) (not (eq (kindOf $value) "invalid")) }} {{- toYaml $value | nindent 0 }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/cluster/charts/fuel-streams/templates/_hpa.yaml b/cluster/charts/fuel-streams/templates/_hpa.yaml new file mode 100644 index 00000000..2f539b35 --- /dev/null +++ b/cluster/charts/fuel-streams/templates/_hpa.yaml @@ -0,0 +1,55 @@ +{{- define "k8s.hpa" -}} +{{- $service := .service -}} +{{- $context := .context -}} +{{- $autoscaling := $service.autoscaling -}} +{{- if $autoscaling.enabled }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + {{- include "k8s.metadata" (dict "context" $context "suffix" (printf "-%s" $service.name)) | nindent 2 }} + labels: + {{- include "fuel-streams.labels" (dict "name" $service.name "context" $context) | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "fuel-streams.fullname" $context }}-{{ $service.name }} + minReplicas: {{ $autoscaling.minReplicas }} + maxReplicas: {{ $autoscaling.maxReplicas }} + behavior: + scaleDown: + stabilizationWindowSeconds: {{ $autoscaling.behavior.scaleDown.stabilizationWindowSeconds | default 300 }} + policies: + - type: Percent + value: {{ $autoscaling.behavior.scaleDown.percentValue | default 100 }} + periodSeconds: {{ $autoscaling.behavior.scaleDown.periodSeconds | default 15 }} + scaleUp: + stabilizationWindowSeconds: {{ $autoscaling.behavior.scaleUp.stabilizationWindowSeconds | default 0 }} + policies: + - type: Percent + value: {{ $autoscaling.behavior.scaleUp.percentValue | default 100 }} + periodSeconds: {{ $autoscaling.behavior.scaleUp.periodSeconds | default 15 }} + - type: Pods + value: {{ $autoscaling.behavior.scaleUp.podValue | default 4 }} + periodSeconds: {{ $autoscaling.behavior.scaleUp.periodSeconds | default 15 }} + selectPolicy: Max + metrics: + {{- if $autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ $autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if $autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ $autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/cluster/charts/fuel-streams/templates/certificate.yaml b/cluster/charts/fuel-streams/templates/certificate.yaml deleted file mode 100644 index 971e1070..00000000 --- a/cluster/charts/fuel-streams/templates/certificate.yaml +++ /dev/null @@ -1,52 +0,0 @@ -{{- $tls := .Values.tls }} -{{- $externalService := .Values.externalService }} -{{- if and $tls.enabled $externalService.dns }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - {{- include "k8s.metadata" (dict "context" . "suffix" "-cert-validator") | nindent 2 }} - labels: - {{- include "fuel-streams.labels" . | nindent 4 }} - {{- include "set-value" (dict "context" $tls "path" "labels") | nindent 4 }} - app.kubernetes.io/service: external-service - annotations: - cert-manager.io/cluster-issuer: {{ $tls.issuer }} - kubernetes.io/ingress.class: nginx - acme.cert-manager.io/http01-ingress-class: nginx - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - {{- include "set-value" (dict "context" $tls "path" "annotations") | nindent 4 }} -spec: - ingressClassName: nginx - rules: - - host: {{ $externalService.dns }} - http: - paths: - - path: /.well-known/acme-challenge/ - pathType: Prefix - backend: - service: - name: cm-acme-http-solver - port: - number: 8089 ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - {{- include "k8s.metadata" (dict "context" . "suffix" "-ws-cert") | nindent 2 }} - labels: - {{- include "fuel-streams.labels" . | nindent 4 }} - {{- include "set-value" (dict "context" $tls "path" "labels") | nindent 4 }} - app.kubernetes.io/service: external-service - annotations: - {{- include "set-value" (dict "context" $tls "path" "annotations") | nindent 4 }} -spec: - secretName: {{ include "fuel-streams.fullname" . }}-ws-tls - duration: {{ $tls.duration }} - renewBefore: {{ $tls.renewBefore }} - dnsNames: - - {{ $externalService.dns }} - issuerRef: - name: {{ $tls.issuer }} - kind: ClusterIssuer -{{- end }} diff --git a/cluster/charts/fuel-streams/templates/common-config.yaml b/cluster/charts/fuel-streams/templates/common-config.yaml new file mode 100644 index 00000000..0ffe75dd --- /dev/null +++ b/cluster/charts/fuel-streams/templates/common-config.yaml @@ -0,0 +1,21 @@ +{{- if .Values.commonConfigMap.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: fuel-streams-config + labels: + app.kubernetes.io/instance: fuel-streams +data: + {{ .Values.commonConfigMap.data | toYaml | nindent 2 }} +{{- end }} +{{- if .Values.localSecrets.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: fuel-streams-keys + labels: + app.kubernetes.io/instance: fuel-streams +stringData: + {{ .Values.localSecrets.data | toYaml | nindent 2 }} +{{- end }} diff --git a/cluster/charts/fuel-streams/templates/consumer/statefulset.yaml b/cluster/charts/fuel-streams/templates/consumer/statefulset.yaml new file mode 100644 index 00000000..a1f73522 --- /dev/null +++ b/cluster/charts/fuel-streams/templates/consumer/statefulset.yaml @@ -0,0 +1,84 @@ +{{- $consumer := .Values.consumer -}} +{{- if $consumer.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + {{- include "k8s.metadata" (dict "context" . "suffix" "-consumer") | nindent 2 }} + annotations: + {{- include "set-value" (dict "context" $consumer "path" "config.annotations") | nindent 4 }} + labels: + {{- include "fuel-streams.labels" (dict "name" "consumer" "context" .) | nindent 4 }} + {{- include "set-value" (dict "context" $consumer "path" "config.labels") | nindent 4 }} +spec: + serviceName: {{ include "fuel-streams.fullname" . }}-consumer + {{- if not $consumer.autoscaling.enabled }} + replicas: {{ $consumer.config.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "fuel-streams.selectorLabels" (dict "name" "consumer" "context" .) | nindent 6 }} + + template: + metadata: + annotations: + {{- include "set-value" (dict "context" $consumer "path" "config.podAnnotations") | nindent 8 }} + labels: + {{- include "fuel-streams.labels" (dict "name" "consumer" "context" .) | nindent 8 }} + {{- include "set-value" (dict "context" $consumer "path" "config.labels") | nindent 8 }} + + spec: + {{- if .Values.serviceAccount.create }} + serviceAccountName: {{ include "fuel-streams.serviceAccountName" . }} + {{- end }} + {{- include "set-field-and-value" (dict "context" $consumer "field" "imagePullSecrets" "path" "config.imagePullSecrets") | nindent 6 }} + {{- include "set-field-and-value" (dict "context" $consumer "field" "nodeSelector" "path" "config.nodeSelector") | nindent 6 }} + {{- include "set-field-and-value" (dict "context" $consumer "field" "affinity" "path" "config.affinity") | nindent 6 }} + {{- include "set-field-and-value" (dict "context" $consumer "field" "tolerations" "path" "config.tolerations") | nindent 6 }} + {{- include "k8s.security-context" (dict "context" . "service" "consumer") | nindent 6 }} + + containers: + - name: consumer + image: "{{ $consumer.image.repository }}:{{ $consumer.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ $consumer.image.pullPolicy }} + command: ["/usr/src/sv-consumer"] + args: + - "--nats-url" + - "$(NATS_URL)" + - "--nats-publisher-url" + - "$(NATS_PUBLISHER_URL)" + {{- with $consumer.image.args }} + {{- toYaml . | nindent 10 }} + {{- end }} + + ports: + - name: consumer + containerPort: {{ $consumer.port }} + protocol: TCP + {{- with $consumer.config.ports }} + {{- toYaml . | nindent 12 }} + {{- end }} + + {{- include "set-field-and-value" (dict "context" $consumer "field" "resources" "path" "config.resources") | nindent 10 }} + {{- include "k8s.probes" (dict "context" . "service" "consumer") | nindent 10 }} + {{- include "k8s.container-security-context" (dict "context" . "service" "consumer") | nindent 10 }} + + env: + - name: PORT + value: {{ $consumer.port | quote }} + {{- with $consumer.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + + envFrom: + - configMapRef: + name: {{ include "fuel-streams.fullname" $ }}-config + optional: true + - secretRef: + name: {{ include "fuel-streams.fullname" $ }}-keys + optional: true + {{- with $consumer.envFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + +{{- include "k8s.hpa" (dict "context" . "service" (dict "name" "consumer" "autoscaling" $consumer.autoscaling)) }} +{{- end }} diff --git a/cluster/charts/fuel-streams/templates/external-service.yaml b/cluster/charts/fuel-streams/templates/external-service.yaml deleted file mode 100644 index 4b2c4602..00000000 --- a/cluster/charts/fuel-streams/templates/external-service.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- $externalService := .Values.externalService }} -{{- if and $externalService.enabled $externalService.dns }} -apiVersion: v1 -kind: Service -metadata: - {{- include "k8s.metadata" (dict "context" . "suffix" "-external") | nindent 2 }} - labels: - {{- include "fuel-streams.labels" . | nindent 4 }} - {{- include "set-value" (dict "context" $externalService "path" "labels") | nindent 4 }} - app.kubernetes.io/service: external-service - annotations: - external-dns.alpha.kubernetes.io/hostname: {{ $externalService.dns | quote }} - external-dns.alpha.kubernetes.io/cloudflare-proxied: "false" - service.beta.kubernetes.io/aws-load-balancer-attributes: load_balancing.cross_zone.enabled=true - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip - service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing - service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true,stickiness.enabled=true,stickiness.type=source_ip,load_balancing.cross_zone.enabled=true - service.beta.kubernetes.io/aws-load-balancer-type: external - service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "WebSocket=true" - {{- include "set-value" (dict "context" $externalService "path" "annotations") | nindent 2 }} -spec: - type: LoadBalancer - loadBalancerClass: service.k8s.aws/nlb - externalTrafficPolicy: Local - ports: - {{- toYaml $externalService.ports | nindent 4 }} - selector: - {{- include "fuel-streams.selectorLabels" . | nindent 4 }} - app.kubernetes.io/service: external-service -{{- end }} diff --git a/cluster/charts/fuel-streams/templates/publisher/network-configmap.yaml b/cluster/charts/fuel-streams/templates/publisher/network-configmap.yaml deleted file mode 100644 index 18a644ef..00000000 --- a/cluster/charts/fuel-streams/templates/publisher/network-configmap.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- $publisher := .Values.publisher }} -apiVersion: v1 -kind: ConfigMap -metadata: - {{- include "k8s.metadata" (dict "context" . "suffix" "-network-config") | nindent 2 }} - annotations: - {{- include "set-value" (dict "context" $publisher "path" "config.annotations") | nindent 4 }} - labels: - {{- include "fuel-streams.labels" . | nindent 4 }} - {{- include "set-value" (dict "context" $publisher "path" "config.labels") | nindent 4 }} - app.kubernetes.io/component: publisher -data: - P2P_PORT: "30333" - DB_PATH: {{ .Values.publisher.storage.mountPath | quote }} - POA_INSTANT: "false" - SERVICE_NAME: "Publisher Node ({{ $publisher.network }})" - {{- if eq $publisher.network "mainnet" }} - RELAYER_V2_LISTENING_CONTRACTS: "0xAEB0c00D0125A8a788956ade4f4F12Ead9f65DDf" - RELAYER_DA_DEPLOY_HEIGHT: "20620434" - RELAYER_LOG_PAGE_SIZE: "2000" - SYNC_HEADER_BATCH_SIZE: "100" - RESERVED_NODES: "/dnsaddr/mainnet.fuel.network" - CHAIN_CONFIG: "mainnet" - {{- else if eq $publisher.network "testnet" }} - RELAYER_V2_LISTENING_CONTRACTS: "0x01855B78C1f8868DE70e84507ec735983bf262dA" - RELAYER_DA_DEPLOY_HEIGHT: "5827607" - RELAYER_LOG_PAGE_SIZE: "2000" - SYNC_HEADER_BATCH_SIZE: "100" - RESERVED_NODES: "/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmDxoChB7AheKNvCVpD4PHJwuDGn8rifMBEHmEynGHvHrf,/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmHnANNk4HjAxQV66BNCRxd2MBUU89ijboZkE69aLuSn1g,/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmVE468rpkh2X1kzz8qQXmqNFiPxU5Lrya28nZdbRUdVJX" - CHAIN_CONFIG: "testnet" - {{- end }} diff --git a/cluster/charts/fuel-streams/templates/publisher/statefulset.yaml b/cluster/charts/fuel-streams/templates/publisher/statefulset.yaml index 0ad2c022..9220c741 100644 --- a/cluster/charts/fuel-streams/templates/publisher/statefulset.yaml +++ b/cluster/charts/fuel-streams/templates/publisher/statefulset.yaml @@ -7,9 +7,8 @@ metadata: annotations: {{- include "set-value" (dict "context" $publisher "path" "config.annotations") | nindent 4 }} labels: - {{- include "fuel-streams.labels" . | nindent 4 }} + {{- include "fuel-streams.labels" (dict "name" "publisher" "context" .) | nindent 4 }} {{- include "set-value" (dict "context" $publisher "path" "config.labels") | nindent 4 }} - app.kubernetes.io/component: publisher spec: serviceName: {{ include "fuel-streams.fullname" . }}-publisher {{- if not $publisher.autoscaling.enabled }} @@ -17,18 +16,15 @@ spec: {{- end }} selector: matchLabels: - {{- include "fuel-streams.selectorLabels" . | nindent 6 }} - {{- include "set-value" (dict "context" $publisher "path" "config.selectorLabels") | nindent 6 }} - app.kubernetes.io/component: publisher + {{- include "fuel-streams.selectorLabels" (dict "name" "publisher" "context" .) | nindent 6 }} template: metadata: annotations: {{- include "set-value" (dict "context" $publisher "path" "config.podAnnotations") | nindent 8 }} labels: - {{- include "fuel-streams.selectorLabels" . | nindent 8 }} + {{- include "fuel-streams.labels" (dict "name" "publisher" "context" .) | nindent 8 }} {{- include "set-value" (dict "context" $publisher "path" "config.labels") | nindent 8 }} - app.kubernetes.io/component: publisher spec: {{- if .Values.serviceAccount.create }} @@ -45,7 +41,7 @@ spec: emptyDir: {} - name: var-dir emptyDir: {} - + initContainers: - name: update-{{ $publisher.storage.name }} image: alpine:latest @@ -81,12 +77,74 @@ spec: - name: publisher image: "{{ $publisher.image.repository }}:{{ $publisher.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ $publisher.image.pullPolicy }} + command: ["/usr/src/sv-publisher"] + args: + # Common arguments + - "--enable-relayer" + - "--enable-p2p" + - "--keypair" + - "$(KEYPAIR)" + - "--relayer" + - "$(RELAYER)" + - "--ip" + - "0.0.0.0" + - "--port" + - "$(PORT)" + - "--peering-port" + - "30333" + - "--utxo-validation" + - "--poa-instant" + - "false" + - "--db-path" + - "$(DB_PATH)" + - "--nats-url" + - "$(NATS_URL)" + - "--sync-header-batch-size" + - "100" + - "--relayer-log-page-size" + - "2000" + - "--sync-block-stream-buffer-size" + - "50" + - "--max-database-cache-size" + - "17179869184" + - "--state-rewind-duration" + - "136y" + - "--request-timeout" + - "60" + - "--graphql-max-complexity" + - "1000000000" + {{- if eq $publisher.network "mainnet" }} + # Mainnet specific args + - "--service-name" + - "Publisher Node (Mainnet)" + - "--snapshot" + - "./chain-config/mainnet" + - "--reserved-nodes" + - "/dnsaddr/mainnet.fuel.network" + - "--relayer-v2-listening-contracts" + - "0xAEB0c00D0125A8a788956ade4f4F12Ead9f65DDf" + - "--relayer-da-deploy-height" + - "20620434" + {{- else if eq $publisher.network "testnet" }} + # Testnet specific args + - "--service-name" + - "Publisher Node (Testnet)" + - "--snapshot" + - "./chain-config/testnet" + - "--reserved-nodes" + - "/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmDxoChB7AheKNvCVpD4PHJwuDGn8rifMBEHmEynGHvHrf,/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmHnANNk4HjAxQV66BNCRxd2MBUU89ijboZkE69aLuSn1g,/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmVE468rpkh2X1kzz8qQXmqNFiPxU5Lrya28nZdbRUdVJX" + - "--relayer-v2-listening-contracts" + - "0x01855B78C1f8868DE70e84507ec735983bf262dA" + - "--relayer-da-deploy-height" + - "5827607" + {{- end }} + ports: - name: http - containerPort: {{ int $publisher.service.port }} + containerPort: {{ int $publisher.port }} protocol: TCP - {{- if $publisher.ports }} - {{- toYaml $publisher.ports | nindent 12 }} + {{- with $publisher.config.ports }} + {{- toYaml . | nindent 12 }} {{- end }} {{- include "set-field-and-value" (dict "context" $publisher "field" "resources" "path" "config.resources") | nindent 10 }} @@ -96,22 +154,20 @@ spec: env: - name: TMPDIR value: "/var/fuel-streams/tmp" - {{- range $key, $value := $publisher.env }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - {{- with $publisher.extraEnv }} + - name: DB_PATH + value: {{ $publisher.storage.mountPath | default "/mnt/db" | quote }} + - name: PORT + value: {{ $publisher.port | quote }} + {{- with $publisher.env }} {{- toYaml . | nindent 12 }} {{- end }} envFrom: - configMapRef: - name: {{ include "fuel-streams.fullname" $ }}-network-config - - configMapRef: - name: {{ include "fuel-streams.fullname" $ }}-publisher + name: {{ include "fuel-streams.fullname" $ }}-config optional: true - secretRef: - name: {{ include "fuel-streams.fullname" $ }}-publisher + name: {{ include "fuel-streams.fullname" $ }}-keys optional: true {{- with $publisher.envFrom }} {{- toYaml . | nindent 12 }} diff --git a/cluster/charts/fuel-streams/templates/secret-creator.yaml b/cluster/charts/fuel-streams/templates/secret-creator.yaml index 04719e93..a64043ec 100755 --- a/cluster/charts/fuel-streams/templates/secret-creator.yaml +++ b/cluster/charts/fuel-streams/templates/secret-creator.yaml @@ -4,7 +4,7 @@ kind: Role metadata: {{- include "k8s.metadata" (dict "context" . "suffix" "-secret-creator") | nindent 2 }} labels: - {{- include "fuel-streams.labels" . | nindent 2 }} + {{- include "fuel-streams.labels" (dict "name" "secret-creator" "context" .) | nindent 2 }} rules: - apiGroups: [""] # "" indicates the core API group resources: ["pods"] @@ -20,6 +20,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: {{- include "k8s.metadata" (dict "context" . "suffix" "-secret-creator") | nindent 2 }} + labels: + {{- include "fuel-streams.labels" (dict "name" "secret-creator" "context" .) | nindent 2 }} subjects: - kind: ServiceAccount name: {{ include "fuel-streams.serviceAccountName" . }} diff --git a/cluster/charts/fuel-streams/templates/service-account.yaml b/cluster/charts/fuel-streams/templates/service-account.yaml index 8201ea7c..eac6647b 100755 --- a/cluster/charts/fuel-streams/templates/service-account.yaml +++ b/cluster/charts/fuel-streams/templates/service-account.yaml @@ -4,6 +4,6 @@ kind: ServiceAccount metadata: {{- include "k8s.metadata" (dict "context" . "suffix" "-service-account") | nindent 2 }} labels: - {{- include "fuel-streams.labels" . | nindent 4 }} + {{- include "fuel-streams.labels" (dict "name" "service-account" "context" .) | nindent 4 }} automountServiceAccountToken: {{ .Values.serviceAccount.automount }} {{- end -}} diff --git a/cluster/charts/fuel-streams/templates/webserver/deployment.yaml b/cluster/charts/fuel-streams/templates/webserver/deployment.yaml index 73935fac..b669f5c5 100644 --- a/cluster/charts/fuel-streams/templates/webserver/deployment.yaml +++ b/cluster/charts/fuel-streams/templates/webserver/deployment.yaml @@ -1,4 +1,5 @@ {{- $webserver := .Values.webserver -}} +{{- $service := $webserver.service -}} {{- if $webserver.enabled -}} apiVersion: apps/v1 kind: Deployment @@ -7,31 +8,25 @@ metadata: annotations: {{- include "set-value" (dict "context" $webserver "path" "config.annotations") | nindent 4 }} labels: - {{- include "fuel-streams.labels" . | nindent 4 }} + {{- include "fuel-streams.labels" (dict "name" "webserver" "context" .) | nindent 4 }} {{- include "set-value" (dict "context" $webserver "path" "config.labels") | nindent 4 }} app.kubernetes.io/component: webserver - app.kubernetes.io/service: external-service - spec: {{- if not $webserver.autoscaling.enabled }} replicas: {{ $webserver.config.replicaCount }} {{- end }} selector: matchLabels: - {{- include "fuel-streams.selectorLabels" . | nindent 6 }} - {{- include "set-value" (dict "context" $webserver "path" "config.selectorLabels") | nindent 6 }} - app.kubernetes.io/component: webserver - app.kubernetes.io/service: external-service + {{- include "fuel-streams.selectorLabels" (dict "name" "webserver" "context" .) | nindent 6 }} template: metadata: annotations: {{- include "set-value" (dict "context" $webserver "path" "config.podAnnotations") | nindent 8 }} labels: - {{- include "fuel-streams.selectorLabels" . | nindent 8 }} + {{- include "fuel-streams.labels" (dict "name" "webserver" "context" .) | nindent 8 }} {{- include "set-value" (dict "context" $webserver "path" "config.labels") | nindent 8 }} app.kubernetes.io/component: webserver - app.kubernetes.io/service: external-service spec: {{- if .Values.serviceAccount.create }} @@ -51,48 +46,32 @@ spec: ports: - name: webserver - containerPort: {{ $webserver.port }} + containerPort: {{ $service.port }} protocol: TCP - {{- if $webserver.ports }} - {{- toYaml $webserver.ports | nindent 12 }} - {{- end }} {{- include "set-field-and-value" (dict "context" $webserver "field" "resources" "path" "config.resources") | nindent 10 }} {{- include "k8s.probes" (dict "context" . "service" "webserver") | nindent 10 }} {{- include "k8s.container-security-context" (dict "context" . "service" "webserver") | nindent 10 }} - env: - {{- range $key, $value := $webserver.env }} - - name: {{ $key }} - value: {{ $value | quote }} + envFrom: + - configMapRef: + name: {{ include "fuel-streams.fullname" $ }}-config + optional: true + - secretRef: + name: {{ include "fuel-streams.fullname" $ }}-keys + optional: true + {{- with $webserver.envFrom }} + {{- toYaml . | nindent 12 }} {{- end }} - {{- with $webserver.extraEnv }} + + env: + - name: NETWORK + value: {{ $webserver.network | quote }} + - name: PORT + value: {{ $service.port | quote }} + {{- with $webserver.env }} {{- toYaml . | nindent 12 }} {{- end }} -{{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - {{- include "k8s.metadata" (dict "context" . "suffix" "-webserver") | nindent 2 }} - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - {{- include "set-value" (dict "context" $webserver "path" "config.annotations") | nindent 4 }} - labels: - {{- include "fuel-streams.labels" . | nindent 4 }} - {{- include "set-value" (dict "context" $webserver "path" "config.labels") | nindent 4 }} - app.kubernetes.io/component: webserver - app.kubernetes.io/service: external-service -spec: - type: LoadBalancer - ports: - - name: webserver - port: {{ $webserver.port }} - targetPort: {{ $webserver.port }} - protocol: TCP - selector: - {{- include "fuel-streams.selectorLabels" . | nindent 4 }} - {{- include "set-value" (dict "context" $webserver "path" "config.selectorLabels") | nindent 4 }} - app.kubernetes.io/component: webserver - app.kubernetes.io/service: external-service +{{- include "k8s.hpa" (dict "context" . "service" (dict "name" "webserver" "autoscaling" $webserver.autoscaling)) }} +{{- end }} \ No newline at end of file diff --git a/cluster/charts/fuel-streams/templates/webserver/service.yaml b/cluster/charts/fuel-streams/templates/webserver/service.yaml new file mode 100644 index 00000000..345c03a4 --- /dev/null +++ b/cluster/charts/fuel-streams/templates/webserver/service.yaml @@ -0,0 +1,37 @@ +{{- $service := .Values.webserver.service }} +{{- if and .Values.webserver.enabled $service.enabled }} +apiVersion: v1 +kind: Service +metadata: + {{- include "k8s.metadata" (dict "context" . "suffix" "-webserver-nlb") | nindent 2 }} + annotations: + {{- if $service.dns }} + external-dns.alpha.kubernetes.io/hostname: {{ $service.dns }} + external-dns.alpha.kubernetes.io/cloudflare-proxied: "false" + {{- end }} + service.beta.kubernetes.io/aws-load-balancer-attributes: load_balancing.cross_zone.enabled=true + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true,stickiness.enabled=true,stickiness.type=source_ip,load_balancing.cross_zone.enabled=true + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "WebSocket=true" + {{- include "set-value" (dict "context" $service "path" "annotations") | nindent 2 }} + labels: + {{- include "fuel-streams.labels" (dict "name" "webserver" "context" .) | nindent 4 }} + {{- include "set-value" (dict "context" $service "path" "labels") | nindent 4 }} + app.kubernetes.io/component: webserver +spec: + type: {{ $service.type }} + loadBalancerClass: service.k8s.aws/nlb + externalTrafficPolicy: Local + ports: + - appProtocol: tcp + name: websocket + port: {{ $service.port }} + protocol: TCP + targetPort: {{ $service.port }} + selector: + {{- include "fuel-streams.selectorLabels" (dict "name" "webserver" "context" .) | nindent 4 }} + app.kubernetes.io/component: webserver +{{- end }} diff --git a/cluster/charts/fuel-streams/tests/certificate_test.yaml b/cluster/charts/fuel-streams/tests/certificate_test.yaml index 179f8f4c..0c93d1ad 100644 --- a/cluster/charts/fuel-streams/tests/certificate_test.yaml +++ b/cluster/charts/fuel-streams/tests/certificate_test.yaml @@ -53,7 +53,7 @@ tests: documentIndex: 1 - equal: path: metadata.name - value: RELEASE-NAME-fuel-streams-ws-cert + value: RELEASE-NAME-sv-webserver-cert documentIndex: 1 - equal: path: spec.dnsNames[0] diff --git a/cluster/charts/fuel-streams/tests/consumer/deployment_test.yaml b/cluster/charts/fuel-streams/tests/consumer/deployment_test.yaml new file mode 100644 index 00000000..534914e2 --- /dev/null +++ b/cluster/charts/fuel-streams/tests/consumer/deployment_test.yaml @@ -0,0 +1,305 @@ +suite: Testing Consumer deployment +templates: + - templates/consumer/deployment.yaml +tests: + - it: should not create deployment when consumer is disabled + set: + consumer.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should create deployment with correct kind and metadata + set: + consumer.enabled: true + asserts: + - isKind: + of: Deployment + documentIndex: 0 + - isAPIVersion: + of: apps/v1 + documentIndex: 0 + - equal: + path: metadata.name + value: RELEASE-NAME-fuel-streams-consumer + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: consumer + - equal: + path: metadata.labels["app.kubernetes.io/service"] + value: external-service + documentIndex: 0 + + - it: should set correct selector labels + set: + consumer.enabled: true + asserts: + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: consumer + documentIndex: 0 + - isSubset: + path: spec.selector.matchLabels + content: + app.kubernetes.io/name: fuel-streams + app.kubernetes.io/instance: RELEASE-NAME + documentIndex: 0 + + - it: should set image configuration correctly + set: + consumer.enabled: true + consumer.image.repository: ghcr.io/fuellabs/sv-webserver + consumer.image.tag: latest + consumer.image.pullPolicy: Always + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/fuellabs/sv-webserver:latest + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + documentIndex: 0 + + - it: should use chart version when tag is not specified + set: + consumer.enabled: true + consumer.image.repository: ghcr.io/fuellabs/sv-webserver + consumer.image.tag: null + Chart: + Version: "1.0" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/fuellabs/sv-webserver:1.0 + documentIndex: 0 + + - it: should configure ports correctly + set: + consumer.enabled: true + consumer.port: 8082 + consumer.ports: + - name: metrics + containerPort: 9090 + protocol: TCP + asserts: + - lengthEqual: + path: spec.template.spec.containers[0].ports + count: 2 + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].ports + content: + name: consumer + containerPort: 8082 + protocol: TCP + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + containerPort: 9090 + protocol: TCP + documentIndex: 0 + + - it: should set replicas when autoscaling is disabled + set: + consumer.enabled: true + consumer.autoscaling.enabled: false + consumer.config.replicaCount: 3 + asserts: + - equal: + path: spec.replicas + value: 3 + documentIndex: 0 + + - it: should not set replicas when autoscaling is enabled + set: + consumer.enabled: true + consumer.autoscaling.enabled: true + consumer.config.replicaCount: 3 + asserts: + - isNull: + path: spec.replicas + documentIndex: 0 + + - it: should merge environment variables correctly + set: + consumer.enabled: true + consumer.env: + RUST_LOG: info + APP_PORT: "8080" + consumer.extraEnv: + - name: EXTRA_VAR + value: "extra-value" + - name: SECRET_VAR + valueFrom: + secretKeyRef: + name: my-secret + key: my-key + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: RUST_LOG + value: "info" + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: APP_PORT + value: "8080" + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: EXTRA_VAR + value: "extra-value" + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: SECRET_VAR + valueFrom: + secretKeyRef: + name: my-secret + key: my-key + documentIndex: 0 + + - it: should set security context when specified + set: + consumer.enabled: true + consumer.config.securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + asserts: + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 1000 + documentIndex: 0 + - equal: + path: spec.template.spec.securityContext.runAsGroup + value: 3000 + documentIndex: 0 + - equal: + path: spec.template.spec.securityContext.fsGroup + value: 2000 + documentIndex: 0 + + - it: should set resource limits and requests + set: + consumer.enabled: true + consumer.config.resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 100m + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 128Mi + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: 50m + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 64Mi + documentIndex: 0 + + - it: should create HPA with correct configuration when autoscaling is enabled + set: + consumer.enabled: true + consumer.autoscaling.enabled: true + consumer.autoscaling.minReplicas: 2 + consumer.autoscaling.maxReplicas: 5 + consumer.autoscaling.targetCPUUtilizationPercentage: 75 + consumer.autoscaling.targetMemoryUtilizationPercentage: 85 + asserts: + - hasDocuments: + count: 2 + - isKind: + of: HorizontalPodAutoscaler + documentIndex: 1 + - equal: + path: spec.minReplicas + value: 2 + documentIndex: 1 + - equal: + path: spec.maxReplicas + value: 5 + documentIndex: 1 + - equal: + path: spec.metrics[0].resource.target.averageUtilization + value: 75 + documentIndex: 1 + - equal: + path: spec.metrics[1].resource.target.averageUtilization + value: 85 + documentIndex: 1 + + - it: should configure HPA scaling behavior correctly + set: + consumer.enabled: true + consumer.autoscaling.enabled: true + consumer.autoscaling.behavior.scaleDown.stabilizationWindowSeconds: 400 + consumer.autoscaling.behavior.scaleDown.percentValue: 50 + consumer.autoscaling.behavior.scaleUp.stabilizationWindowSeconds: 60 + consumer.autoscaling.behavior.scaleUp.percentValue: 200 + consumer.autoscaling.behavior.scaleUp.podValue: 6 + asserts: + - equal: + path: spec.behavior.scaleDown.stabilizationWindowSeconds + value: 400 + documentIndex: 1 + - equal: + path: spec.behavior.scaleDown.policies[0].type + value: Percent + documentIndex: 1 + - equal: + path: spec.behavior.scaleDown.policies[0].value + value: 50 + documentIndex: 1 + - equal: + path: spec.behavior.scaleUp.stabilizationWindowSeconds + value: 60 + documentIndex: 1 + - equal: + path: spec.behavior.scaleUp.policies[0].type + value: Percent + documentIndex: 1 + - equal: + path: spec.behavior.scaleUp.policies[0].value + value: 200 + documentIndex: 1 + - equal: + path: spec.behavior.scaleUp.policies[1].type + value: Pods + documentIndex: 1 + - equal: + path: spec.behavior.scaleUp.policies[1].value + value: 6 + documentIndex: 1 + - equal: + path: spec.behavior.scaleUp.selectPolicy + value: Max + documentIndex: 1 + + - it: should not create HPA when autoscaling is disabled + set: + consumer.enabled: true + consumer.autoscaling.enabled: false + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Deployment + documentIndex: 0 diff --git a/cluster/charts/fuel-streams/tests/external_service_test.yaml b/cluster/charts/fuel-streams/tests/external_service_test.yaml index 11a70a5e..611cb452 100644 --- a/cluster/charts/fuel-streams/tests/external_service_test.yaml +++ b/cluster/charts/fuel-streams/tests/external_service_test.yaml @@ -73,7 +73,7 @@ tests: path: spec.selector content: app.kubernetes.io/name: fuel-streams - app.kubernetes.io/service: external-service + app.kubernetes.io/service: external-ws - it: should set correct annotations set: diff --git a/cluster/charts/fuel-streams/tests/publisher/network-configmap.yaml b/cluster/charts/fuel-streams/tests/publisher/network-configmap.yaml deleted file mode 100644 index dfb5a7b3..00000000 --- a/cluster/charts/fuel-streams/tests/publisher/network-configmap.yaml +++ /dev/null @@ -1,87 +0,0 @@ -suite: Testing Publisher network configmap -templates: - - templates/publisher/network-configmap.yaml -tests: - - it: should configure mainnet correctly - set: - publisher.enabled: true - publisher.network: mainnet - publisher.storage.mountPath: /mnt/db - asserts: - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: RELEASE-NAME-fuel-streams-network-config - - equal: - path: data.P2P_PORT - value: "30333" - - equal: - path: data.DB_PATH - value: /mnt/db - - equal: - path: data.POA_INSTANT - value: "false" - - equal: - path: data.SERVICE_NAME - value: "Publisher Node (mainnet)" - - equal: - path: data.RELAYER_V2_LISTENING_CONTRACTS - value: "0xAEB0c00D0125A8a788956ade4f4F12Ead9f65DDf" - - equal: - path: data.RELAYER_DA_DEPLOY_HEIGHT - value: "20620434" - - equal: - path: data.RELAYER_LOG_PAGE_SIZE - value: "2000" - - equal: - path: data.SYNC_HEADER_BATCH_SIZE - value: "100" - - equal: - path: data.RESERVED_NODES - value: "/dnsaddr/mainnet.fuel.network" - - equal: - path: data.CHAIN_CONFIG - value: "mainnet" - - - it: should configure testnet correctly - set: - publisher.enabled: true - publisher.network: testnet - publisher.storage.mountPath: /mnt/db - asserts: - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: RELEASE-NAME-fuel-streams-network-config - - equal: - path: data.P2P_PORT - value: "30333" - - equal: - path: data.DB_PATH - value: /mnt/db - - equal: - path: data.POA_INSTANT - value: "false" - - equal: - path: data.SERVICE_NAME - value: "Publisher Node (testnet)" - - equal: - path: data.RELAYER_V2_LISTENING_CONTRACTS - value: "0x01855B78C1f8868DE70e84507ec735983bf262dA" - - equal: - path: data.RELAYER_DA_DEPLOY_HEIGHT - value: "5827607" - - equal: - path: data.RELAYER_LOG_PAGE_SIZE - value: "2000" - - equal: - path: data.SYNC_HEADER_BATCH_SIZE - value: "100" - - equal: - path: data.RESERVED_NODES - value: "/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmDxoChB7AheKNvCVpD4PHJwuDGn8rifMBEHmEynGHvHrf,/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmHnANNk4HjAxQV66BNCRxd2MBUU89ijboZkE69aLuSn1g,/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmVE468rpkh2X1kzz8qQXmqNFiPxU5Lrya28nZdbRUdVJX" - - equal: - path: data.CHAIN_CONFIG - value: "testnet" diff --git a/cluster/charts/fuel-streams/tests/publisher/statefulset.yaml b/cluster/charts/fuel-streams/tests/publisher/statefulset.yaml index fd3c3f6b..eaa6af79 100644 --- a/cluster/charts/fuel-streams/tests/publisher/statefulset.yaml +++ b/cluster/charts/fuel-streams/tests/publisher/statefulset.yaml @@ -10,36 +10,36 @@ tests: of: StatefulSet - equal: path: metadata.name - value: RELEASE-NAME-fuel-streams-publisher + value: RELEASE-NAME-sv-publisher - it: should set correct image and tag set: publisher.enabled: true - publisher.image.repository: ghcr.io/fuellabs/fuel-streams-publisher + publisher.image.repository: ghcr.io/fuellabs/sv-publisher publisher.image.tag: latest asserts: - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/fuellabs/fuel-streams-publisher:latest + value: ghcr.io/fuellabs/sv-publisher:latest - it: should use chart version when tag is not specified set: publisher.enabled: true - publisher.image.repository: ghcr.io/fuellabs/fuel-streams-publisher + publisher.image.repository: ghcr.io/fuellabs/sv-publisher publisher.image.tag: null Chart: Version: "1.0" asserts: - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/fuellabs/fuel-streams-publisher:1.0 + value: ghcr.io/fuellabs/sv-publisher:1.0 - it: should merge environment variables correctly set: publisher.enabled: true publisher.env: - CHAIN_CONFIG: "testnet" # Override default - NEW_VAR: "new-value" # Add new var + CHAIN_CONFIG: "testnet" # Override default + NEW_VAR: "new-value" # Add new var publisher.extraEnv: - name: SIMPLE_VAR value: "simple-value" diff --git a/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml b/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml index 84833c59..9d5a148d 100644 --- a/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml +++ b/cluster/charts/fuel-streams/tests/webserver/deployment_test.yaml @@ -19,7 +19,7 @@ tests: of: apps/v1 - equal: path: metadata.name - value: RELEASE-NAME-fuel-streams-webserver + value: RELEASE-NAME-sv-webserver - equal: path: metadata.labels["app.kubernetes.io/component"] value: webserver @@ -40,13 +40,13 @@ tests: - it: should set image configuration correctly set: webserver.enabled: true - webserver.image.repository: ghcr.io/fuellabs/fuel-streams-webserver + webserver.image.repository: ghcr.io/fuellabs/sv-webserver webserver.image.tag: latest webserver.image.pullPolicy: Always asserts: - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/fuellabs/fuel-streams-webserver:latest + value: ghcr.io/fuellabs/sv-webserver:latest - equal: path: spec.template.spec.containers[0].imagePullPolicy value: Always @@ -54,14 +54,14 @@ tests: - it: should use chart version when tag is not specified set: webserver.enabled: true - webserver.image.repository: ghcr.io/fuellabs/fuel-streams-webserver + webserver.image.repository: ghcr.io/fuellabs/sv-webserver webserver.image.tag: null Chart: Version: "1.0" asserts: - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/fuellabs/fuel-streams-webserver:1.0 + value: ghcr.io/fuellabs/sv-webserver:1.0 - it: should configure ports correctly set: @@ -186,4 +186,4 @@ tests: value: 50m - equal: path: spec.template.spec.containers[0].resources.requests.memory - value: 64Mi \ No newline at end of file + value: 64Mi diff --git a/cluster/charts/fuel-streams/values-local.yaml b/cluster/charts/fuel-streams/values-local.yaml new file mode 100644 index 00000000..46254d83 --- /dev/null +++ b/cluster/charts/fuel-streams/values-local.yaml @@ -0,0 +1,135 @@ +config: + createRoles: true + healthChecks: true + +commonConfigMap: + enabled: true + data: + AWS_S3_BUCKET_NAME: "fuel-streams-staging" + AWS_ENDPOINT_URL: "https://localhost:9000" + AWS_REGION: "us-east-1" + AWS_S3_ENABLED: "false" + USE_METRICS: "false" + NATS_URL: "fuel-streams-nats-core:4222" + NATS_PUBLISHER_URL: "fuel-streams-nats-publisher:4222" + NATS_SYSTEM_USER: "sys" + NATS_SYSTEM_PASS: "sys" + NATS_ADMIN_USER: "admin" + NATS_ADMIN_PASS: "admin" + NATS_PUBLIC_USER: "default_user" + NATS_PUBLIC_PASS: "" + +# Reduce storage requirements for local development +publisher: + image: + repository: sv-publisher + pullPolicy: IfNotPresent + tag: latest + + storage: + size: 10Gi + storageClass: "standard" # Use default storage class + + config: + replicaCount: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +consumer: + enabled: true + image: + repository: sv-consumer + pullPolicy: IfNotPresent + tag: latest + + config: + replicaCount: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +webserver: + enabled: true + image: + repository: sv-webserver + pullPolicy: IfNotPresent + tag: latest + + service: + enabled: true + port: 9003 + + tls: + enabled: false + +# NATS Core configuration for local development +nats-core: + enabled: true + container: + env: + GOMEMLIMIT: 1GiB + merge: + envFrom: + - configMapRef: + name: fuel-streams-config + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + + config: + cluster: + replicas: 3 + + jetstream: + fileStore: + pvc: + size: 10Gi + storageClassName: "standard" + + merge: + jetstream: + max_file_store: << 10GiB >> + max_memory_store: << 1GiB >> + +# NATS Publisher configuration for local development +nats-publisher: + enabled: true + container: + env: + GOMEMLIMIT: 1GiB + merge: + envFrom: + - configMapRef: + name: fuel-streams-config + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + + config: + jetstream: + fileStore: + pvc: + size: 10Gi + storageClassName: "standard" + + merge: + jetstream: + max_file_store: << 10GiB >> + max_memory_store: << 1GiB >> diff --git a/cluster/charts/fuel-streams/values.yaml b/cluster/charts/fuel-streams/values.yaml index 912e3f0e..be933eb5 100755 --- a/cluster/charts/fuel-streams/values.yaml +++ b/cluster/charts/fuel-streams/values.yaml @@ -1,6 +1,3 @@ -docker: - registry: registry.dev.svc.cluster.local:5000 - config: # Override the name and fullname of the chart nameOverride: "" @@ -72,28 +69,32 @@ startupProbe: failureThreshold: 6 successThreshold: 1 -tls: - enabled: false - issuer: "letsencrypt-prod" - duration: "2160h" - renewBefore: "360h" - annotations: {} - labels: {} +# ------------------------------------------------------------------------------------------------- +# Global ConfigMap +# ------------------------------------------------------------------------------------------------- -externalService: +commonConfigMap: + enabled: true + data: + AWS_S3_BUCKET_NAME: "fuel-streams-staging" + AWS_ENDPOINT_URL: "https://s3.us-east-1.amazonaws.com" + AWS_REGION: "us-east-1" + AWS_S3_ENABLED: "true" + USE_METRICS: "false" + NATS_URL: "fuel-streams-nats-core:4222" + NATS_PUBLISHER_URL: "fuel-streams-nats-publisher:4222" + NATS_SYSTEM_USER: "sys" + NATS_SYSTEM_PASS: "sys" + NATS_ADMIN_USER: "admin" + NATS_ADMIN_PASS: "admin" + NATS_PUBLIC_USER: "default_user" + NATS_PUBLIC_PASS: "" + +# This is a secret that is used for local development +# It is not used in production +localSecrets: enabled: false - dns: "streams.svc.cluster.local" - labels: {} - annotations: {} - ports: - - name: websocket - port: 8443 - targetPort: websocket - protocol: TCP - - name: webserver - port: 8082 - targetPort: http - protocol: TCP + data: {} # ------------------------------------------------------------------------------------------------- # Monitoring @@ -109,15 +110,18 @@ monitoring: publisher: enabled: true network: mainnet + port: 8080 image: - repository: fuel-streams-publisher - pullPolicy: Never - tag: "latest" + repository: ghcr.io/fuellabs/sv-publisher + pullPolicy: Always + tag: latest + args: [] - service: - type: ClusterIP - port: 8080 + # You can override the env variables for the container here + # using a map or an array of key-value pairs + env: [] + envFrom: [] prometheus: enabled: false @@ -126,8 +130,8 @@ publisher: storage: name: rocks-db - size: 10Gi - storageClass: standard + size: 500Gi + storageClass: "gp3-generic" accessMode: ReadWriteOnce mountPath: /mnt/db @@ -154,48 +158,102 @@ publisher: maxReplicas: 3 targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + percentValue: 100 + periodSeconds: 15 + scaleUp: + stabilizationWindowSeconds: 0 + percentValue: 100 + podValue: 4 + periodSeconds: 15 + +# ------------------------------------------------------------------------------------------------- +# Consumer configuration +# ------------------------------------------------------------------------------------------------- + +consumer: + enabled: true + port: 8080 + image: + repository: ghcr.io/fuellabs/sv-consumer + pullPolicy: Always + tag: latest + args: [] + + # You can override the env variables for the container here + # using a map or an array of key-value pairs + env: [] + envFrom: [] + + config: + replicaCount: 3 + labels: {} + annotations: {} + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + imagePullSecrets: [] + ports: [] + livenessProbe: {} + readinessProbe: {} + startupProbe: {} + securityContext: {} + containerSecurityContext: {} + resources: {} - env: - PUBLISHER_MAX_THREADS: "32" - NATS_URL: "fuel-streams-nats-publisher:4222" - HISTORICAL: "true" - - # Additional environment variables with complex structures - # extraEnv: [] - # - name: RELAYER - # valueFrom: - # secretKeyRef: - # name: fuel-streams-publisher - # key: RELAYER - # - name: KEYPAIR - # valueFrom: - # secretKeyRef: - # name: fuel-streams-publisher - # key: KEYPAIR - # - name: NATS_ADMIN_PASS - # valueFrom: - # secretKeyRef: - # name: fuel-streams-publisher - # key: NATS_ADMIN_PASS - # Optional: Bulk environment references - # envFrom: {} - # - configMapRef: - # name: additional-config - # - secretRef: - # name: additional-secrets + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + percentValue: 100 + periodSeconds: 15 + scaleUp: + stabilizationWindowSeconds: 0 + percentValue: 100 + podValue: 4 + periodSeconds: 15 # ------------------------------------------------------------------------------------------------- # WebServer configuration # ------------------------------------------------------------------------------------------------- webserver: - enabled: true - port: 9003 + enabled: false + network: mainnet image: - repository: fuel-streams-ws - pullPolicy: Never - tag: "latest" + repository: ghcr.io/fuellabs/sv-webserver + pullPolicy: Always + tag: latest + + service: + enabled: true + type: LoadBalancer + port: 9003 + dns: "stream-staging.fuel.network" + annotations: {} + labels: {} + + tls: + enabled: true + issuer: "letsencrypt-prod" + duration: "2160h" + renewBefore: "360h" + annotations: {} + labels: {} + + # You can override the env variables for the container here + # using a map or an array of key-value pairs + env: [] + envFrom: [] config: replicaCount: 1 @@ -212,64 +270,25 @@ webserver: startupProbe: {} securityContext: {} containerSecurityContext: {} - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi + resources: {} autoscaling: enabled: false minReplicas: 1 - maxReplicas: 3 + maxReplicas: 5 targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + percentValue: 100 + periodSeconds: 15 + scaleUp: + stabilizationWindowSeconds: 0 + percentValue: 100 + podValue: 4 + periodSeconds: 15 - env: - STREAMER_MAX_WORKERS: "10" - API_PORT: 9003 - JWT_AUTH_SECRET: "secret" - USE_ELASTIC_LOGGING: false - USE_METRICS: true - AWS_S3_ENABLED: true - NATS_URL: "fuel-streams-nats-publisher:4222" - NETWORK: testnet - - # Additional environment variables with complex structures - # extraEnv: [] - # - name: AWS_ACCESS_KEY_ID - # valueFrom: - # secretKeyRef: - # name: fuel-streams-webserver - # key: AWS_ACCESS_KEY_ID - # - name: AWS_SECRET_ACCESS_KEY - # valueFrom: - # secretKeyRef: - # name: fuel-streams-webserver - # key: AWS_SECRET_ACCESS_KEY - # - name: AWS_REGION - # valueFrom: - # secretKeyRef: - # name: fuel-streams-webserver - # key: AWS_REGION - # - name: AWS_S3_BUCKET_NAME - # valueFrom: - # secretKeyRef: - # name: fuel-streams-webserver - # key: AWS_S3_BUCKET_NAME - # - name: AWS_ENDPOINT_URL - # valueFrom: - # secretKeyRef: - # name: fuel-streams-webserver - # key: AWS_ENDPOINT_URL - # Optional: Bulk environment references - # envFrom: {} - # - configMapRef: - # name: additional-config - # - secretRef: - # name: additional-secrets # ------------------------------------------------------------------------------------------------- # NATS Core configuration # ------------------------------------------------------------------------------------------------- @@ -295,17 +314,14 @@ nats-core: container: image: repository: nats - tag: 2.10.22-alpine + tag: 2.10.24-alpine env: - GOMEMLIMIT: 8GiB - startupProbe: - initialDelaySeconds: 60 - periodSeconds: 10 - failureThreshold: 1080 - resources: - requests: - cpu: 8 - memory: 8Gi + GOMEMLIMIT: 7GiB + merge: + resources: + requests: + cpu: 2 + memory: 8Gi service: enabled: true @@ -314,6 +330,8 @@ nats-core: enabled: true cluster: enabled: true + websocket: + enabled: true leafnodes: enabled: true monitor: @@ -323,145 +341,46 @@ nats-core: cluster: enabled: true port: 6222 - replicas: 3 + replicas: 5 routeURLs: useFQDN: true - jetstream: - enabled: true - fileStore: - dir: /data - pvc: - enabled: true - size: 500Gi - storageClassName: standard - - leafnodes: + websocket: enabled: true - port: 7422 - - monitor: - enabled: false - port: 8222 - - merge: - jetstream: - max_file_store: << 500GiB >> - max_memory_store: << 7168MiB >> - domain: "central" - max_payload: << 8MiB >> - -# ------------------------------------------------------------------------------------------------- -# NATS Client configuration -# ------------------------------------------------------------------------------------------------- - -nats-client: - enabled: true - - global: - labels: - app.kubernetes.io/service: external-service - - natsBox: - enabled: false - - promExporter: - enabled: false - - container: - image: - repository: nats - tag: 2.10.22-alpine - env: - GOMEMLIMIT: 2GiB - startupProbe: - initialDelaySeconds: 60 - periodSeconds: 10 - failureThreshold: 1080 - resources: - requests: - cpu: 2 - memory: 2Gi - - service: - enabled: true - ports: - nats: - enabled: true - websocket: - enabled: true - monitor: - enabled: false - - statefulSet: - merge: - spec: - replicas: 2 - - podTemplate: - topologySpreadConstraints: - kubernetes.io/hostname: - maxSkew: 1 - whenUnsatisfiable: DoNotSchedule + port: 8443 - config: jetstream: enabled: true fileStore: dir: /data pvc: enabled: true - size: 20Gi - storageClassName: standard - merge: - domain: "client" + size: 2000Gi + storageClassName: "gp3-generic" leafnodes: enabled: true port: 7422 - remotes: - - url: "nats://nats-core:7422" - - websocket: - enabled: true - port: 8443 - merge: - no_tls: true - no_auth_user: default_user monitor: enabled: false port: 8222 merge: + max_payload: << 32MiB >> jetstream: - max_file_store: << 20GiB >> - max_memory_store: << 2048MiB >> - max_payload: << 8MiB >> - accounts: - USERS: - jetstream: enabled - users: - - user: default_user - permissions: - subscribe: ">" - publish: - deny: - - "*.by_id.>" - - "*.blocks.>" - - "*.transactions.>" - - "*.inputs.>" - - "*.outputs.>" - - "*.receipts.>" - - "*.logs.>" - - "*.utxos.>" - - "$JS.API.STREAM.CREATE.>" - - "$JS.API.STREAM.UPDATE.>" - - "$JS.API.STREAM.DELETE.>" - - "$JS.API.STREAM.PURGE.>" - - "$JS.API.STREAM.RESTORE.>" - - "$JS.API.STREAM.MSG.DELETE.>" - - "$JS.API.CONSUMER.DURABLE.CREATE.>" + domain: CORE + sync_interval: << 30s >> + max_outstanding_catchup: << 512MiB >> + max_file_store: << 2000GiB >> + max_memory_store: << 7GiB >> + system_account: SYS + $include: auth.conf + + configMap: + merge: + $tplYaml: | + {{- include "nats-accounts" . | nindent 8 }} # ------------------------------------------------------------------------------------------------- # NATS Publisher configuration @@ -476,40 +395,33 @@ nats-publisher: promExporter: enabled: false + statefulSet: + merge: + spec: + replicas: 5 + container: image: repository: nats - tag: 2.10.22-alpine + tag: 2.10.24-alpine env: - GOMEMLIMIT: 3GiB - startupProbe: - initialDelaySeconds: 60 - periodSeconds: 10 - failureThreshold: 1080 - resources: - requests: - cpu: 4 - memory: 4Gi + GOMEMLIMIT: 7GiB + merge: + resources: + requests: + cpu: 2 + memory: 8Gi service: enabled: true ports: nats: enabled: true + leafnodes: + enabled: true monitor: enabled: false - statefulSet: - merge: - spec: - replicas: 3 - - podTemplate: - topologySpreadConstraints: - kubernetes.io/hostname: - maxSkew: 1 - whenUnsatisfiable: DoNotSchedule - config: jetstream: enabled: true @@ -517,23 +429,33 @@ nats-publisher: dir: /data pvc: enabled: true - size: 50Gi - storageClassName: standard - merge: - domain: "publisher" + size: 100Gi + storageClassName: "gp3-generic" leafnodes: enabled: true port: 7422 - remotes: - - url: "nats://nats-core:7422" + merge: + remotes: + - urls: ["nats-leaf://admin:admin@fuel-streams-nats-core:7422"] + account: ADMIN monitor: enabled: false port: 8222 merge: + max_payload: << 32MiB >> jetstream: - max_file_store: << 50GiB >> - max_memory_store: << 3072MiB >> - max_payload: << 8MiB >> + domain: PUBLISHER + sync_interval: << 30s >> + max_outstanding_catchup: << 512MiB >> + max_file_store: << 100GiB >> + max_memory_store: << 7GiB >> + system_account: SYS + $include: auth.conf + + configMap: + merge: + $tplYaml: | + {{- include "nats-accounts" . | nindent 8 }} diff --git a/cluster/docker/docker-compose.yml b/cluster/docker/docker-compose.yml index 1966f3c3..34b76756 100644 --- a/cluster/docker/docker-compose.yml +++ b/cluster/docker/docker-compose.yml @@ -1,38 +1,66 @@ services: - nats: + nats-core: + profiles: + - all + - nats image: nats:latest - container_name: nats + container_name: nats-core restart: always ports: - 4222:4222 - - 8222:8222 - - 8443:8443 volumes: - - ./nats.conf:/etc/nats/nats.conf + - ./nats-config/core.conf:/etc/nats/nats.conf + - ./nats-config/accounts.conf:/etc/nats/accounts.conf command: - - -m - - "8222" - - --name=fuel-streams-publisher-server + - --name=fuel-streams-nats-core - --js - --config=/etc/nats/nats.conf - -D env_file: - ./../../.env + + nats-publisher: + profiles: + - all + - nats + image: nats:latest + container_name: nats-publisher + restart: always + ports: + - 4333:4222 + volumes: + - ./nats-config/publisher.conf:/etc/nats/nats.conf + - ./nats-config/accounts.conf:/etc/nats/accounts.conf + command: + - --name=fuel-streams-nats-publisher + - --js + - --config=/etc/nats/nats.conf + - -D + env_file: + - ./../../.env + depends_on: + - nats-core + localstack: + profiles: + - all + - localstack image: localstack/localstack:latest container_name: localstack restart: always ports: - - "4566:4566" # LocalStack main gateway port - - "4572:4572" # S3 service port (optional) + - "4566:4566" # LocalStack main gateway port + - "4572:4572" # S3 service port (optional) environment: - - SERVICES=s3 # Enable just S3 service + - SERVICES=s3 # Enable just S3 service - DEBUG=1 - - AWS_ACCESS_KEY_ID=test - - AWS_SECRET_ACCESS_KEY=test - - DEFAULT_REGION=us-east-1 - - DEFAULT_BUCKETS=fuel-streams-local + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - DEFAULT_REGION=${AWS_REGION} + - DEFAULT_BUCKETS=${AWS_S3_BUCKET_NAME} volumes: - ./localstack-data:/var/lib/localstack - /var/run/docker.sock:/var/run/docker.sock - ./init-localstack.sh:/etc/localstack/init/ready.d/init-localstack.sh + env_file: + - ./../../.env diff --git a/cluster/docker/fuel-streams-publisher.Dockerfile b/cluster/docker/fuel-streams-publisher.Dockerfile deleted file mode 100644 index a1b7f00f..00000000 --- a/cluster/docker/fuel-streams-publisher.Dockerfile +++ /dev/null @@ -1,130 +0,0 @@ -# Stage 1: Build -FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx -FROM --platform=$BUILDPLATFORM rust:1.81.0 AS chef - -ARG TARGETPLATFORM -RUN cargo install cargo-chef && rustup target add wasm32-unknown-unknown -WORKDIR /build/ - -COPY --from=xx / / - -# hadolint ignore=DL3008 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - lld \ - clang \ - libclang-dev \ - && xx-apt-get update \ - && xx-apt-get install -y libc6-dev g++ binutils \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - - -FROM chef AS planner -ENV CARGO_NET_GIT_FETCH_WITH_CLI=true -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - - -FROM chef AS builder -ARG DEBUG_SYMBOLS=false -ENV CARGO_NET_GIT_FETCH_WITH_CLI=true -ENV CARGO_PROFILE_RELEASE_DEBUG=$DEBUG_SYMBOLS -COPY --from=planner /build/recipe.json recipe.json -RUN echo $CARGO_PROFILE_RELEASE_DEBUG -# Build our project dependencies, not our application! -RUN \ - --mount=type=cache,target=/usr/local/cargo/registry/index \ - --mount=type=cache,target=/usr/local/cargo/registry/cache \ - --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/build/target \ - xx-cargo chef cook --release --no-default-features -p fuel-streams-publisher --recipe-path recipe.json -# Up to this point, if our dependency tree stays the same, -# all layers should be cached. -COPY . . -# build application -RUN \ - --mount=type=cache,target=/usr/local/cargo/registry/index \ - --mount=type=cache,target=/usr/local/cargo/registry/cache \ - --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/build/target \ - xx-cargo build --release --no-default-features -p fuel-streams-publisher \ - && xx-verify ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-publisher \ - && cp ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-publisher /root/fuel-streams-publisher \ - && cp ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-publisher.d /root/fuel-streams-publisher.d - -# Stage 2: Run -FROM ubuntu:22.04 AS run - -ARG IP=0.0.0.0 -ARG PORT=4000 -ARG TELEMETRY_PORT=8080 -ARG P2P_PORT=30333 -ARG DB_PATH=/mnt/db/ -ARG POA_INSTANT=false -ARG RELAYER_LOG_PAGE_SIZE=2000 -ARG SERVICE_NAME="NATS Publisher Node" -ARG SYNC_HEADER_BATCH_SIZE=100 -ARG RESERVED_NODES=/dns4/p2p-testnet.fuel.network/tcp/30333/p2p/16Uiu2HAmDxoChB7AheKNvCVpD4PHJwuDGn8rifMBEHmEynGHvHrf - -ENV IP=$IP -ENV PORT=$PORT -ENV TELEMETRY_PORT=$TELEMETRY_PORT -ENV DB_PATH=$DB_PATH -ENV POA_INSTANT=false -ENV RELAYER_LOG_PAGE_SIZE=$RELAYER_LOG_PAGE_SIZE -ENV SERVICE_NAME=$SERVICE_NAME -ENV SYNC_HEADER_BATCH_SIZE=$SYNC_HEADER_BATCH_SIZE -ENV RESERVED_NODES=$RESERVED_NODES -ENV HISTORICAL=false - -ENV KEYPAIR= -ENV RELAYER= -ENV RELAYER_V2_LISTENING_CONTRACTS= -ENV RELAYER_DA_DEPLOY_HEIGHT= -ENV CHAIN_CONFIG= -ENV NATS_URL= -ENV USE_METRICS= -ENV USE_ELASTIC_LOGGING= - -WORKDIR /usr/src - -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends ca-certificates curl \ - # Clean up - && apt-get autoremove -y \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /root/fuel-streams-publisher . -COPY --from=builder /root/fuel-streams-publisher.d . - -COPY /cluster/chain-config ./chain-config -EXPOSE ${PORT} -EXPOSE ${P2P_PORT} -EXPOSE ${TELEMETRY_PORT} - -# https://stackoverflow.com/a/44671685 -# https://stackoverflow.com/a/40454758 -# hadolint ignore=DL3025 -CMD exec ./fuel-streams-publisher \ - --service-name "${SERVICE_NAME}" \ - --keypair $KEYPAIR \ - --relayer $RELAYER \ - --ip $IP \ - --port $PORT \ - --telemetry-port $TELEMETRY_PORT \ - --peering-port $P2P_PORT \ - --db-path "${DB_PATH}" \ - --utxo-validation \ - --poa-instant $POA_INSTANT \ - --snapshot ./chain-config/${CHAIN_CONFIG} \ - --enable-p2p \ - --reserved-nodes $RESERVED_NODES \ - --sync-header-batch-size $SYNC_HEADER_BATCH_SIZE \ - --enable-relayer \ - --relayer-v2-listening-contracts $RELAYER_V2_LISTENING_CONTRACTS \ - --relayer-da-deploy-height $RELAYER_DA_DEPLOY_HEIGHT \ - --relayer-log-page-size $RELAYER_LOG_PAGE_SIZE \ - --sync-block-stream-buffer-size 30 \ - $([ "$HISTORICAL" = "true" ] && echo "--historical") diff --git a/cluster/docker/init-localstack.sh b/cluster/docker/init-localstack.sh index 9ad8d0d7..befa0901 100755 --- a/cluster/docker/init-localstack.sh +++ b/cluster/docker/init-localstack.sh @@ -3,5 +3,6 @@ set -e echo "Creating S3 bucket in LocalStack..." -awslocal s3 mb s3://fuel-streams-test -echo "Bucket created: fuel-streams-test" +BUCKET_NAME=${AWS_S3_BUCKET_NAME:-fuel-streams-test} +awslocal s3 mb "s3://${BUCKET_NAME}" +echo "Bucket created: ${BUCKET_NAME}" diff --git a/cluster/docker/nats-config/accounts.conf b/cluster/docker/nats-config/accounts.conf new file mode 100644 index 00000000..59a035c8 --- /dev/null +++ b/cluster/docker/nats-config/accounts.conf @@ -0,0 +1,15 @@ +accounts { + SYS: { + users: [{user: $NATS_SYSTEM_USER, password: $NATS_SYSTEM_PASS}] + }, + ADMIN: { + users: [{user: $NATS_ADMIN_USER, password: $NATS_ADMIN_PASS}] + jetstream: enabled + }, + PUBLIC: { + users: [{user: $NATS_PUBLIC_USER, password: $NATS_PUBLIC_PASS}] + jetstream: enabled + } +} + +system_account: SYS diff --git a/cluster/docker/nats-config/client.conf b/cluster/docker/nats-config/client.conf new file mode 100644 index 00000000..08d9e2b3 --- /dev/null +++ b/cluster/docker/nats-config/client.conf @@ -0,0 +1,18 @@ +port: 4222 +server_name: client-server + +jetstream { + store_dir: "./data/store_client" + domain: CLIENT +} + +leafnodes { + remotes: [ + { + urls: ["nats://admin:admin@nats-core:7422"] + account: "ADMIN" + } + ] +} + +include ./accounts.conf diff --git a/cluster/docker/nats-config/core.conf b/cluster/docker/nats-config/core.conf new file mode 100644 index 00000000..2e8f11f4 --- /dev/null +++ b/cluster/docker/nats-config/core.conf @@ -0,0 +1,13 @@ +port: 4222 +server_name: core-server + +jetstream { + store_dir: "./data/core" + domain: CORE +} + +leafnodes { + port: 7422 +} + +include ./accounts.conf diff --git a/cluster/docker/nats-config/publisher.conf b/cluster/docker/nats-config/publisher.conf new file mode 100644 index 00000000..52de2113 --- /dev/null +++ b/cluster/docker/nats-config/publisher.conf @@ -0,0 +1,18 @@ +port: 4222 +server_name: leaf-server + +jetstream { + store_dir: "./data/store_leaf" + domain: LEAF +} + +leafnodes { + remotes: [ + { + urls: ["nats://admin:admin@nats-core:7422"] + account: "ADMIN" + } + ] +} + +include ./accounts.conf diff --git a/cluster/docker/nats.conf b/cluster/docker/nats.conf deleted file mode 100644 index 783a0450..00000000 --- a/cluster/docker/nats.conf +++ /dev/null @@ -1,52 +0,0 @@ -port = 4222 -http_port = 8222 -server_name = "fuel-nats-server" - -authorization = { - timeout = 5 - ADMIN = { - publish = ">" - subscribe = ">" - } - default_permissions = { - subscribe = ">" - publish = { - deny = [ - "*.blocks.>", - "*.transactions.>", - "*.inputs.>", - "*.outputs.>", - "*.receipts.>", - "*.logs.>", - "*.utxos.>", - "$JS.API.STREAM.CREATE.>", - "$JS.API.STREAM.UPDATE.>", - "$JS.API.STREAM.DELETE.>", - "$JS.API.STREAM.PURGE.>", - "$JS.API.STREAM.RESTORE.>", - "$JS.API.STREAM.MSG.DELETE.>", - "$JS.API.CONSUMER.DURABLE.CREATE.>", - ] - } - } - users = [ - { user = admin, password = $NATS_ADMIN_PASS, permissions = $ADMIN }, - { user = default_user } - ] -} - -jetstream = { - max_file_store = 21474836480 -} - -max_payload = 8388608 - -websocket = { - port = 8443 - no_tls = true - same_origin = false - allowed_origins = [] - compression = false - handshake_timeout = "10s" - no_auth_user = default_user -} diff --git a/cluster/docker/fuel-streams-ws.Dockerfile b/cluster/docker/sv-consumer.Dockerfile similarity index 67% rename from cluster/docker/fuel-streams-ws.Dockerfile rename to cluster/docker/sv-consumer.Dockerfile index 5e4a7a8a..5b20d1cd 100644 --- a/cluster/docker/fuel-streams-ws.Dockerfile +++ b/cluster/docker/sv-consumer.Dockerfile @@ -38,7 +38,7 @@ RUN \ --mount=type=cache,target=/usr/local/cargo/registry/cache \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/build/target \ - xx-cargo chef cook --release --no-default-features -p fuel-streams-ws --recipe-path recipe.json + xx-cargo chef cook --release --no-default-features -p sv-consumer --recipe-path recipe.json # Up to this point, if our dependency tree stays the same, # all layers should be cached. COPY . . @@ -48,29 +48,16 @@ RUN \ --mount=type=cache,target=/usr/local/cargo/registry/cache \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/build/target \ - xx-cargo build --release --no-default-features -p fuel-streams-ws \ - && xx-verify ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-ws \ - && cp ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-ws /root/fuel-streams-ws \ - && cp ./target/$(xx-cargo --print-target-triple)/release/fuel-streams-ws.d /root/fuel-streams-ws.d + xx-cargo build --release --no-default-features -p sv-consumer \ + && xx-verify ./target/$(xx-cargo --print-target-triple)/release/sv-consumer \ + && cp ./target/$(xx-cargo --print-target-triple)/release/sv-consumer /root/sv-consumer \ + && cp ./target/$(xx-cargo --print-target-triple)/release/sv-consumer.d /root/sv-consumer.d # Stage 2: Run FROM ubuntu:22.04 AS run -ARG API_PORT=9003 - -ENV API_PORT=$API_PORT -ENV NATS_URL= -ENV NETWORK= -ENV USE_METRICS= -ENV USE_ELASTIC_LOGGING= -ENV AWS_S3_ENABLED= -ENV AWS_ACCESS_KEY_ID= -ENV AWS_SECRET_ACCESS_KEY= -ENV AWS_REGION= -ENV AWS_ENDPOINT_URL= -ENV AWS_S3_BUCKET_NAME= -ENV JWT_AUTH_SECRET= - +ARG PORT=8080 +ENV PORT=$PORT WORKDIR /usr/src RUN apt-get update -y \ @@ -80,12 +67,10 @@ RUN apt-get update -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /root/fuel-streams-ws . -COPY --from=builder /root/fuel-streams-ws.d . +COPY --from=builder /root/sv-consumer . +COPY --from=builder /root/sv-consumer.d . -EXPOSE ${API_PORT} +EXPOSE ${PORT} -# https://stackoverflow.com/a/44671685 -# https://stackoverflow.com/a/40454758 -# hadolint ignore=DL3025 -CMD exec ./fuel-streams-ws +WORKDIR /usr/src +CMD ["./sv-consumer"] diff --git a/cluster/docker/sv-publisher.Dockerfile b/cluster/docker/sv-publisher.Dockerfile new file mode 100644 index 00000000..de9b042f --- /dev/null +++ b/cluster/docker/sv-publisher.Dockerfile @@ -0,0 +1,81 @@ +# Stage 1: Build +FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx +FROM --platform=$BUILDPLATFORM rust:1.81.0 AS chef + +# Add package name as build argument +ARG TARGETPLATFORM + +RUN cargo install cargo-chef && rustup target add wasm32-unknown-unknown +WORKDIR /build/ + +COPY --from=xx / / + +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + lld \ + clang \ + libclang-dev \ + && xx-apt-get update \ + && xx-apt-get install -y libc6-dev g++ binutils \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +FROM chef AS planner +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +ARG PACKAGE_NAME +ARG DEBUG_SYMBOLS=false +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_PROFILE_RELEASE_DEBUG=$DEBUG_SYMBOLS +COPY --from=planner /build/recipe.json recipe.json +RUN echo $CARGO_PROFILE_RELEASE_DEBUG +# Build our project dependencies, not our application! +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/build/target \ + xx-cargo chef cook --release --no-default-features -p sv-publisher --recipe-path recipe.json +# Up to this point, if our dependency tree stays the same, +# all layers should be cached. +COPY . . +# build application +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/build/target \ + xx-cargo build --release --no-default-features -p sv-publisher \ + && xx-verify ./target/$(xx-cargo --print-target-triple)/release/sv-publisher \ + && cp ./target/$(xx-cargo --print-target-triple)/release/sv-publisher /root/sv-publisher \ + && cp ./target/$(xx-cargo --print-target-triple)/release/sv-publisher.d /root/sv-publisher.d + +# Stage 2: Run +FROM ubuntu:22.04 AS run + +ARG PORT=4000 +ARG P2P_PORT=30333 +ARG DB_PATH=/mnt/db +ENV PORT="${PORT}" + +WORKDIR /usr/src + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /root/sv-publisher . +COPY --from=builder /root/sv-publisher.d . + +COPY /cluster/chain-config ./chain-config +EXPOSE ${PORT} +EXPOSE ${P2P_PORT} + +WORKDIR /usr/src +CMD ["./sv-publisher", "--port", "${PORT}", "--peering-port", "${P2P_PORT}", "--db-path", "${DB_PATH}"] diff --git a/cluster/docker/sv-webserver.Dockerfile b/cluster/docker/sv-webserver.Dockerfile new file mode 100644 index 00000000..75140bd4 --- /dev/null +++ b/cluster/docker/sv-webserver.Dockerfile @@ -0,0 +1,75 @@ +# Stage 1: Build +FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx +FROM --platform=$BUILDPLATFORM rust:1.81.0 AS chef + +ARG TARGETPLATFORM +RUN cargo install cargo-chef && rustup target add wasm32-unknown-unknown +WORKDIR /build/ + +COPY --from=xx / / + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + lld \ + clang \ + libclang-dev \ + && xx-apt-get update \ + && xx-apt-get install -y libc6-dev g++ binutils \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +FROM chef AS planner +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + + +FROM chef AS builder +ARG DEBUG_SYMBOLS=false +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_PROFILE_RELEASE_DEBUG=$DEBUG_SYMBOLS +COPY --from=planner /build/recipe.json recipe.json +RUN echo $CARGO_PROFILE_RELEASE_DEBUG +# Build our project dependencies, not our application! +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/build/target \ + xx-cargo chef cook --release --no-default-features -p sv-webserver --recipe-path recipe.json +# Up to this point, if our dependency tree stays the same, +# all layers should be cached. +COPY . . +# build application +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry/index \ + --mount=type=cache,target=/usr/local/cargo/registry/cache \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/build/target \ + xx-cargo build --release --no-default-features -p sv-webserver \ + && xx-verify ./target/$(xx-cargo --print-target-triple)/release/sv-webserver \ + && cp ./target/$(xx-cargo --print-target-triple)/release/sv-webserver /root/sv-webserver \ + && cp ./target/$(xx-cargo --print-target-triple)/release/sv-webserver.d /root/sv-webserver.d + +# Stage 2: Run +FROM ubuntu:22.04 AS run + +ARG PORT=9003 +ENV PORT=$PORT + +WORKDIR /usr/src + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + # Clean up + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /root/sv-webserver . +COPY --from=builder /root/sv-webserver.d . + +EXPOSE ${PORT} +CMD ["./sv-webserver"] diff --git a/cluster/scripts/build_docker.sh b/cluster/scripts/build_docker.sh new file mode 100755 index 00000000..0a5eebb7 --- /dev/null +++ b/cluster/scripts/build_docker.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +# Remove the -g flag from set +set -euo pipefail + +# Help/Usage function +usage() { + cat << EOF +Usage: $(basename "$0") [OPTIONS] + +Build a Docker image using specified parameters. + +Options: + --dockerfile Path to Dockerfile (default: cluster/docker/sv-publisher.Dockerfile) + --build-args Additional Docker build arguments (optional) + -h, --help Show this help message + +Environment variables: + TAG Docker image tag (default: latest) + DOCKER_HOST Docker daemon socket (optional) + +Examples: + $(basename "$0") --dockerfile ./Dockerfile + $(basename "$0") --dockerfile ./Dockerfile --build-args "--build-arg KEY=VALUE" +EOF + exit 1 +} + +# Show help if no arguments or help flag +if [[ $# -eq 0 ]] || [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then + usage +fi + +# Default values +DOCKERFILE="cluster/docker/sv-publisher.Dockerfile" +IMAGE_NAME=${EXPECTED_IMAGE:-"sv-publisher"} +TAG=${EXPECTED_TAG:-"latest"} +BUILD_ARGS="" + +# Parse named arguments +while [[ $# -gt 0 ]]; do + case $1 in + --dockerfile) + DOCKERFILE="$2" + shift 2 + ;; + --build-args) + BUILD_ARGS="$2" + shift 2 + ;; + *) + echo "Error: Unknown argument '$1'" + usage + ;; + esac +done + +# Validate required files exist +if [[ ! -f "$DOCKERFILE" ]]; then + echo "Error: Dockerfile not found at $DOCKERFILE" + exit 1 +fi + +# Ensure we're using minikube's docker daemon +if [[ -n "${DOCKER_HOST:-}" ]]; then + echo "Using provided DOCKER_HOST: $DOCKER_HOST" +else + eval $(minikube docker-env) +fi + +echo "Building image ${IMAGE_NAME}:${TAG} using ${DOCKERFILE}" +echo "Build args: ${BUILD_ARGS}" + +# Build the docker image with build args if provided +if [[ -n "${BUILD_ARGS}" ]]; then + docker build ${BUILD_ARGS} -t "${IMAGE_NAME}:${TAG}" -f "${DOCKERFILE}" . +else + docker build -t "${IMAGE_NAME}:${TAG}" -f "${DOCKERFILE}" . +fi diff --git a/cluster/scripts/build_publisher.sh b/cluster/scripts/build_publisher.sh deleted file mode 100755 index 0fe43d4b..00000000 --- a/cluster/scripts/build_publisher.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Use environment variables provided by Tilt if available -IMAGE_NAME=${EXPECTED_IMAGE:-"fuel-streams-publisher"} -TAG=${EXPECTED_TAG:-"latest"} -DOCKERFILE="cluster/docker/fuel-streams-publisher.Dockerfile" - -# Ensure we're using minikube's docker daemon if not already set -if [ -z "${DOCKER_HOST:-}" ]; then - eval $(minikube docker-env) -fi - -# Build the docker image -docker build -t ${IMAGE_NAME}:${TAG} -f ${DOCKERFILE} . diff --git a/cluster/scripts/build_streamer.sh b/cluster/scripts/build_streamer.sh index 48dd5375..90d90edd 100755 --- a/cluster/scripts/build_streamer.sh +++ b/cluster/scripts/build_streamer.sh @@ -3,9 +3,9 @@ set -euo pipefail # Use environment variables provided by Tilt if available -IMAGE_NAME=${EXPECTED_IMAGE:-"fuel-streams-ws"} +IMAGE_NAME=${EXPECTED_IMAGE:-"sv-webserver"} TAG=${EXPECTED_TAG:-"latest"} -DOCKERFILE="docker/fuel-streams-ws.Dockerfile" +DOCKERFILE="docker/sv-webserver.Dockerfile" # Ensure we're using minikube's docker daemon if not already set if [ -z "${DOCKER_HOST:-}" ]; then diff --git a/cluster/scripts/gen_env_secret.sh b/cluster/scripts/gen_env_secret.sh index df0680d3..46a589f8 100755 --- a/cluster/scripts/gen_env_secret.sh +++ b/cluster/scripts/gen_env_secret.sh @@ -4,15 +4,12 @@ source .env # Generate the YAML configuration -cat <cluster/charts/fuel-streams/values-publisher-secrets.yaml -publisher: - extraEnv: - - name: RELAYER - value: "${RELAYER:-}" - - name: KEYPAIR - value: "${KEYPAIR:-}" - - name: NATS_ADMIN_PASS - value: "${NATS_ADMIN_PASS:-}" +cat << EOF > cluster/charts/fuel-streams/values-secrets.yaml +localSecrets: + enabled: true + data: + RELAYER: "${RELAYER:-}" + KEYPAIR: "${KEYPAIR:-}" EOF -echo "Generated values-publisher-secrets.yaml with environment variables" +echo "Generated values-secrets.yaml with environment variables" diff --git a/cluster/scripts/setup_k8s.sh b/cluster/scripts/setup_k8s.sh deleted file mode 100755 index bb7537e8..00000000 --- a/cluster/scripts/setup_k8s.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -[[ $DEBUG = true ]] && set -x -set -euo pipefail - -# Parse command line arguments -NAMESPACE="${1:-fuel-streams}" # Use first argument, default to "fuel-streams" if not provided - -# Configure namespace and context -echo -e "\n\033[1;33mConfiguring ${NAMESPACE} namespace and context:\033[0m" - -# Check if namespace exists -if kubectl get namespace ${NAMESPACE} &>/dev/null; then - echo "Namespace ${NAMESPACE} already exists" -else - echo "Creating namespace ${NAMESPACE}..." - kubectl create namespace ${NAMESPACE} -fi - -# Switch to minikube context -if ! kubectl config current-context | grep -q "minikube"; then - echo "Switching to minikube context..." - kubectl config use-context minikube -else - echo "Already in minikube context" -fi - -# Set namespace for current context -CURRENT_NAMESPACE=$(kubectl config view --minify --output 'jsonpath={..namespace}') -if [ "$CURRENT_NAMESPACE" != "${NAMESPACE}" ]; then - echo "Setting current namespace to ${NAMESPACE}..." - kubectl config set-context --current --cluster=minikube --namespace=${NAMESPACE} -else - echo "Context namespace is already set to ${NAMESPACE}" -fi - -# Verify context configuration -echo -e "\n\033[1;33mVerifying cluster context:\033[0m" -kubectl config get-contexts diff --git a/cluster/scripts/setup_minikube.sh b/cluster/scripts/setup_minikube.sh index 0c01374a..cf4009cd 100755 --- a/cluster/scripts/setup_minikube.sh +++ b/cluster/scripts/setup_minikube.sh @@ -4,12 +4,12 @@ set -euo pipefail # Check if minikube is installed -if ! command -v minikube &>/dev/null; then +if ! command -v minikube &> /dev/null; then echo "Installing minikube..." - sudo curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && - sudo chmod +x minikube && - sudo cp minikube /usr/local/bin/ && - sudo rm minikube + sudo curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 \ + && sudo chmod +x minikube \ + && sudo cp minikube /usr/local/bin/ \ + && sudo rm minikube else echo "minikube is already installed" fi @@ -17,13 +17,9 @@ fi # Delete any existing minikube cluster minikube delete -# Default values for resources -DEFAULT_DISK_SIZE='50000mb' -DEFAULT_MEMORY='12000mb' - -# Get parameters with defaults -DISK_SIZE=${1:-$DEFAULT_DISK_SIZE} -MEMORY=${2:-$DEFAULT_MEMORY} +# Set disk and memory size, using defaults if not provided +DISK_SIZE=${1:-'50000mb'} +MEMORY=${2:-'12000mb'} # Start minikube with specified resources minikube start \ @@ -32,6 +28,56 @@ minikube start \ --memory="$MEMORY" \ --cpus 8 +minikube addons enable metrics-server +minikube addons enable registry + +# Remove existing registry proxy container if running +if docker ps -a | grep -q "minikube-registry-proxy"; then + echo "Removing existing registry proxy container..." + docker rm -f minikube-registry-proxy +fi + +# Forward minikube registry to localhost +docker run --rm -d \ + --network=host \ + --name minikube-registry-proxy \ + alpine ash -c "apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:$(minikube ip):5000" + # Display minikube status echo -e "\n\033[1;33mMinikube Status:\033[0m" minikube status + +# Parse command line arguments +NAMESPACE="${1:-fuel-streams}" # Use first argument, default to "fuel-streams" if not provided + +# Configure namespace and context +echo -e "\n\033[1;33mConfiguring ${NAMESPACE} namespace and context:\033[0m" + +# Check if namespace exists +if kubectl get namespace ${NAMESPACE} &> /dev/null; then + echo "Namespace ${NAMESPACE} already exists" +else + echo "Creating namespace ${NAMESPACE}..." + kubectl create namespace ${NAMESPACE} +fi + +# Switch to minikube context +if ! kubectl config current-context | grep -q "minikube"; then + echo "Switching to minikube context..." + kubectl config use-context minikube +else + echo "Already in minikube context" +fi + +# Set namespace for current context +CURRENT_NAMESPACE=$(kubectl config view --minify --output 'jsonpath={..namespace}') +if [ "$CURRENT_NAMESPACE" != "${NAMESPACE}" ]; then + echo "Setting current namespace to ${NAMESPACE}..." + kubectl config set-context --current --cluster=minikube --namespace=${NAMESPACE} +else + echo "Context namespace is already set to ${NAMESPACE}" +fi + +# Verify context configuration +echo -e "\n\033[1;33mVerifying cluster context:\033[0m" +kubectl config get-contexts diff --git a/crates/fuel-streams-core/Cargo.toml b/crates/fuel-streams-core/Cargo.toml index a2f5acb2..9e6178fc 100644 --- a/crates/fuel-streams-core/Cargo.toml +++ b/crates/fuel-streams-core/Cargo.toml @@ -11,24 +11,40 @@ version = { workspace = true } rust-version = { workspace = true } [dependencies] +anyhow = { workspace = true } async-nats = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } displaydoc = { workspace = true } -fuel-core-client = { workspace = true } +fuel-core = { workspace = true, default-features = false, features = [ + "p2p", + "relayer", + "rocksdb", + "test-helpers", +] } +fuel-core-bin = { workspace = true, default-features = false, features = [ + "p2p", + "relayer", + "rocksdb", +] } +fuel-core-client = { workspace = true, default-features = false, features = ["std"] } fuel-core-importer = { workspace = true } -fuel-core-types = { workspace = true } +fuel-core-services = { workspace = true, default-features = false, features = ["test-helpers"] } +fuel-core-storage = { workspace = true } +fuel-core-types = { workspace = true, default-features = false, features = ["std", "serde"] } fuel-data-parser = { workspace = true } fuel-networks = { workspace = true } fuel-streams-macros = { workspace = true } +fuel-streams-nats = { workspace = true } fuel-streams-storage = { workspace = true } futures = { workspace = true } hex = { workspace = true } pretty_assertions = { workspace = true, optional = true } serde = { workspace = true } -sha2 = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/crates/fuel-streams-core/README.md b/crates/fuel-streams-core/README.md index 221aca64..69684062 100644 --- a/crates/fuel-streams-core/README.md +++ b/crates/fuel-streams-core/README.md @@ -61,10 +61,10 @@ use futures::StreamExt; #[tokio::main] async fn main() -> BoxedResult<()> { // Connect to NATS server - let nats_opts = NatsClientOpts::new(FuelNetwork::Local); + let nats_opts = NatsClientOpts::admin_opts(); let nats_client = NatsClient::connect(&nats_opts).await?; - let s3_opts = S3ClientOpts::new(FuelNetwork::Local); + let s3_opts = S3ClientOpts::new(S3Env::Local, S3Role::Admin); let s3_client = Arc::new(S3Client::new(&s3_opts).await?); // Create a stream for blocks diff --git a/crates/fuel-streams-publisher/src/publisher/fuel_core_like.rs b/crates/fuel-streams-core/src/fuel_core_like.rs similarity index 91% rename from crates/fuel-streams-publisher/src/publisher/fuel_core_like.rs rename to crates/fuel-streams-core/src/fuel_core_like.rs index 921cf743..85d2458f 100644 --- a/crates/fuel-streams-publisher/src/publisher/fuel_core_like.rs +++ b/crates/fuel-streams-core/src/fuel_core_like.rs @@ -2,15 +2,8 @@ use std::{sync::Arc, time::Duration}; use fuel_core::{ combined_database::CombinedDatabase, - database::{ - database_description::{on_chain::OnChain, DatabaseHeight}, - Database, - }, + database::{database_description::on_chain::OnChain, Database}, fuel_core_graphql_api::ports::DatabaseBlocks, - state::{ - generic_database::GenericDatabase, - iterable_key_value_view::IterableKeyValueViewWrapper, - }, }; use fuel_core_bin::FuelService; use fuel_core_importer::ports::ImporterDatabase; @@ -19,17 +12,12 @@ use fuel_core_types::{ blockchain::consensus::{Consensus, Sealed}, fuel_types::BlockHeight, }; -use fuel_streams_core::types::*; use tokio::{sync::broadcast::Receiver, time::sleep}; -pub type OffchainDatabase = GenericDatabase< - IterableKeyValueViewWrapper< - fuel_core::fuel_core_graphql_api::storage::Column, - >, ->; +use crate::types::*; /// Interface for `fuel-core` related logic. -/// This was introduced to simplify mocking and testing the `fuel-streams-publisher` crate. +/// This was introduced to simplify mocking and testing the `sv-publisher` crate. #[async_trait::async_trait] pub trait FuelCoreLike: Sync + Send { async fn start(&self) -> anyhow::Result<()>; @@ -42,6 +30,7 @@ pub trait FuelCoreLike: Sync + Send { fn base_asset_id(&self) -> &FuelCoreAssetId; fn chain_id(&self) -> &FuelCoreChainId; + fn fuel_service(&self) -> &FuelService; fn database(&self) -> &CombinedDatabase; fn onchain_database(&self) -> &Database { @@ -60,14 +49,19 @@ pub trait FuelCoreLike: Sync + Send { &self, ) -> Receiver; - fn get_latest_block_height(&self) -> anyhow::Result { + fn get_latest_block_height(&self) -> anyhow::Result { Ok(self .onchain_database() .latest_block_height()? - .map(|h| h.as_u64()) + .map(|h| *h) .unwrap_or_default()) } + fn get_tx_status( + &self, + tx_id: &FuelCoreBytes32, + ) -> anyhow::Result>; + fn get_receipts( &self, tx_id: &FuelCoreBytes32, @@ -198,6 +192,10 @@ impl FuelCoreLike for FuelCore { Ok(()) } + fn fuel_service(&self) -> &FuelService { + &self.fuel_service + } + async fn stop(&self) { if matches!( self.fuel_service.state(), @@ -262,12 +260,18 @@ impl FuelCoreLike for FuelCore { .subscribe() } + fn get_tx_status( + &self, + tx_id: &FuelCoreBytes32, + ) -> anyhow::Result> { + Ok(self.offchain_database()?.get_tx_status(tx_id)?) + } + fn get_receipts( &self, tx_id: &FuelCoreBytes32, ) -> anyhow::Result>> { let receipts = self - .offchain_database()? .get_tx_status(tx_id)? .map(|status| match &status { FuelCoreTransactionStatus::Success { receipts, .. } diff --git a/crates/fuel-streams-core/src/fuel_core_types.rs b/crates/fuel-streams-core/src/fuel_core_types.rs index d42f0009..b5461427 100644 --- a/crates/fuel-streams-core/src/fuel_core_types.rs +++ b/crates/fuel-streams-core/src/fuel_core_types.rs @@ -1,5 +1,7 @@ -/// FuelCore Types -/// Allows flexilibity of aggregating and transforming them for different payload types +use fuel_core::state::{ + generic_database::GenericDatabase, + iterable_key_value_view::IterableKeyValueViewWrapper, +}; pub use fuel_core_client::client::schema::Tai64Timestamp as FuelCoreTai64Timestamp; pub use fuel_core_importer::ImporterResult as FuelCoreImporterResult; pub use fuel_core_types::{ @@ -29,7 +31,9 @@ pub use fuel_core_types::{ Input as FuelCoreInput, MessageId as FuelCoreMessageId, Output as FuelCoreOutput, + PanicInstruction as FuelCorePanicInstruction, Receipt as FuelCoreReceipt, + ScriptExecutionResult as FuelCoreScriptExecutionResult, StorageSlot as FuelCoreStorageSlot, Transaction as FuelCoreTransaction, TxId as FuelCoreTxId, @@ -52,3 +56,9 @@ pub use fuel_core_types::{ }, tai64::Tai64 as FuelCoreTai64, }; + +pub type OffchainDatabase = GenericDatabase< + IterableKeyValueViewWrapper< + fuel_core::fuel_core_graphql_api::storage::Column, + >, +>; diff --git a/crates/fuel-streams-core/src/inputs/types.rs b/crates/fuel-streams-core/src/inputs/types.rs index fcd5b01d..91ff8c1b 100644 --- a/crates/fuel-streams-core/src/inputs/types.rs +++ b/crates/fuel-streams-core/src/inputs/types.rs @@ -1,3 +1,5 @@ +use fuel_core_types::fuel_crypto; + use crate::types::*; // Input enum @@ -171,3 +173,24 @@ pub struct InputMessage { pub sender: Address, pub witness_index: u16, } + +impl InputMessage { + pub fn compute_message_id(&self) -> MessageId { + let hasher = fuel_crypto::Hasher::default() + .chain(self.sender.as_ref()) + .chain(self.recipient.as_ref()) + .chain(self.nonce.as_ref()) + .chain(self.amount.to_be_bytes()) + .chain(self.data.as_ref()); + + (*hasher.finalize()).into() + } + + pub fn computed_utxo_id(&self) -> UtxoId { + let message_id = self.compute_message_id(); + UtxoId { + tx_id: Bytes32::from(message_id), + output_index: 0, + } + } +} diff --git a/crates/fuel-streams-core/src/lib.rs b/crates/fuel-streams-core/src/lib.rs index 11fe85b5..ba8b30f8 100644 --- a/crates/fuel-streams-core/src/lib.rs +++ b/crates/fuel-streams-core/src/lib.rs @@ -9,7 +9,7 @@ pub mod transactions; pub mod utxos; pub mod nats { - pub use fuel_streams_storage::nats::*; + pub use fuel_streams_nats::*; } pub mod s3 { @@ -20,6 +20,7 @@ pub mod stream; pub mod subjects; +pub mod fuel_core_like; mod fuel_core_types; mod primitive_types; pub mod types; @@ -28,7 +29,15 @@ pub use stream::*; pub mod prelude { pub use fuel_networks::*; + #[allow(unused_imports)] pub use fuel_streams_macros::subject::*; - pub use crate::{nats::*, s3::*, stream::*, subjects::*, types::*}; + pub use crate::{ + fuel_core_like::*, + nats::*, + s3::*, + stream::*, + subjects::*, + types::*, + }; } diff --git a/crates/fuel-streams-core/src/logs/types.rs b/crates/fuel-streams-core/src/logs/types.rs index 4fdb7473..927216d4 100644 --- a/crates/fuel-streams-core/src/logs/types.rs +++ b/crates/fuel-streams-core/src/logs/types.rs @@ -27,94 +27,30 @@ pub enum Log { }, } -impl From for Log { - fn from(value: FuelCoreReceipt) -> Self { +impl From for Log { + fn from(value: Receipt) -> Self { match value { - FuelCoreReceipt::Log { - id, - ra, - rb, - rc, - rd, - pc, - is, - } => Log::WithoutData { - id: id.into(), - ra, - rb, - rc, - rd, - pc, - is, + Receipt::Log(log) => Log::WithoutData { + id: log.id, + ra: log.ra, + rb: log.rb, + rc: log.rc, + rd: log.rd, + pc: log.pc, + is: log.is, }, - FuelCoreReceipt::LogData { - id, - ra, - rb, - ptr, - len, - digest, - pc, - is, - data, - } => Log::WithData { - id: id.into(), - ra, - rb, - ptr, - len, - digest: digest.into(), - pc, - is, - data, + Receipt::LogData(log) => Log::WithData { + id: log.id, + ra: log.ra, + rb: log.rb, + ptr: log.ptr, + len: log.len, + digest: log.digest, + pc: log.pc, + is: log.is, + data: log.data, }, _ => panic!("Invalid receipt type"), } } } - -/// Introduced majorly allow delegating serialization and deserialization to `fuel-core`'s Receipt -impl From for FuelCoreReceipt { - fn from(log: Log) -> FuelCoreReceipt { - match log { - Log::WithoutData { - id, - ra, - rb, - rc, - rd, - pc, - is, - } => FuelCoreReceipt::Log { - id: id.into(), - ra, - rb, - rc, - rd, - pc, - is, - }, - Log::WithData { - id, - ra, - rb, - ptr, - len, - digest, - pc, - is, - data, - } => FuelCoreReceipt::LogData { - id: id.into(), - ra, - rb, - ptr, - len, - digest: digest.into(), - pc, - is, - data, - }, - } - } -} diff --git a/crates/fuel-streams-core/src/outputs/types.rs b/crates/fuel-streams-core/src/outputs/types.rs index 607bcd72..ddedd9a0 100644 --- a/crates/fuel-streams-core/src/outputs/types.rs +++ b/crates/fuel-streams-core/src/outputs/types.rs @@ -4,11 +4,11 @@ use crate::types::*; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] pub enum Output { - Coin(CoinOutput), + Coin(OutputCoin), Contract(OutputContract), - Change(ChangeOutput), - Variable(VariableOutput), - ContractCreated(ContractCreated), + Change(OutputChange), + Variable(OutputVariable), + ContractCreated(OutputContractCreated), } impl From<&FuelCoreOutput> for Output { @@ -18,7 +18,7 @@ impl From<&FuelCoreOutput> for Output { amount, asset_id, to, - } => Output::Coin(CoinOutput { + } => Output::Coin(OutputCoin { amount: *amount, asset_id: asset_id.into(), to: to.into(), @@ -30,7 +30,7 @@ impl From<&FuelCoreOutput> for Output { amount, asset_id, to, - } => Output::Change(ChangeOutput { + } => Output::Change(OutputChange { amount: *amount, asset_id: asset_id.into(), to: to.into(), @@ -39,7 +39,7 @@ impl From<&FuelCoreOutput> for Output { amount, asset_id, to, - } => Output::Variable(VariableOutput { + } => Output::Variable(OutputVariable { amount: *amount, asset_id: asset_id.into(), to: to.into(), @@ -47,7 +47,7 @@ impl From<&FuelCoreOutput> for Output { FuelCoreOutput::ContractCreated { contract_id, state_root, - } => Output::ContractCreated(ContractCreated { + } => Output::ContractCreated(OutputContractCreated { contract_id: contract_id.into(), state_root: state_root.into(), }), @@ -57,7 +57,7 @@ impl From<&FuelCoreOutput> for Output { #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct CoinOutput { +pub struct OutputCoin { pub amount: u64, pub asset_id: AssetId, pub to: Address, @@ -65,7 +65,7 @@ pub struct CoinOutput { #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ChangeOutput { +pub struct OutputChange { pub amount: u64, pub asset_id: AssetId, pub to: Address, @@ -73,7 +73,7 @@ pub struct ChangeOutput { #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct VariableOutput { +pub struct OutputVariable { pub amount: u64, pub asset_id: AssetId, pub to: Address, @@ -99,7 +99,7 @@ impl From<&FuelCoreOutputContract> for OutputContract { #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ContractCreated { - contract_id: ContractId, - state_root: Bytes32, +pub struct OutputContractCreated { + pub contract_id: ContractId, + pub state_root: Bytes32, } diff --git a/crates/fuel-streams-core/src/primitive_types.rs b/crates/fuel-streams-core/src/primitive_types.rs index 997b4221..158f23af 100644 --- a/crates/fuel-streams-core/src/primitive_types.rs +++ b/crates/fuel-streams-core/src/primitive_types.rs @@ -1,4 +1,8 @@ -use fuel_core_types::fuel_types; +use fuel_core_types::{ + fuel_asm::RawInstruction, + fuel_tx::PanicReason, + fuel_types, +}; pub use serde::{Deserialize, Serialize}; use crate::fuel_core_types::*; @@ -158,6 +162,58 @@ generate_byte_type_wrapper!(Salt, fuel_types::Salt, 32); generate_byte_type_wrapper!(MessageId, fuel_types::MessageId, 32); generate_byte_type_wrapper!(BlockId, fuel_types::Bytes32, 32); generate_byte_type_wrapper!(Signature, fuel_types::Bytes64, 64); +generate_byte_type_wrapper!(TxId, fuel_types::TxId, 32); + +/// Implements bidirectional conversions between `Bytes32` and a given type. +/// +/// This macro generates implementations of the `From` trait to convert: +/// - From `Bytes32` to the target type +/// - From a reference to `Bytes32` to the target type +/// - From the target type to `Bytes32` +/// - From a reference of the target type to `Bytes32` +/// +/// The target type must be a 32-byte type that can be converted to/from `[u8; 32]`. +/// +/// # Example +/// ```ignore +/// impl_bytes32_conversions!(ContractId); +/// ``` +macro_rules! impl_bytes32_conversions { + ($type:ty) => { + impl From for $type { + fn from(value: Bytes32) -> Self { + let bytes: [u8; 32] = value.0.into(); + <$type>::from(bytes) + } + } + impl From<&Bytes32> for $type { + fn from(value: &Bytes32) -> Self { + value.clone().into() + } + } + impl From<$type> for Bytes32 { + fn from(value: $type) -> Self { + let bytes: [u8; 32] = value.0.into(); + Bytes32::from(bytes) + } + } + impl From<&$type> for Bytes32 { + fn from(value: &$type) -> Self { + value.clone().into() + } + } + }; +} + +impl_bytes32_conversions!(MessageId); +impl_bytes32_conversions!(ContractId); +impl_bytes32_conversions!(AssetId); +impl_bytes32_conversions!(Address); +impl_bytes32_conversions!(BlobId); +impl_bytes32_conversions!(Nonce); +impl_bytes32_conversions!(Salt); +impl_bytes32_conversions!(BlockId); +impl_bytes32_conversions!(TxId); impl From for BlockId { fn from(value: FuelCoreBlockId) -> Self { @@ -165,18 +221,6 @@ impl From for BlockId { } } -impl From for MessageId { - fn from(value: Bytes32) -> Self { - let bytes: [u8; 32] = value.0.into(); - MessageId::from(bytes) - } -} -impl From<&Bytes32> for MessageId { - fn from(value: &Bytes32) -> Self { - value.clone().into() - } -} - #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] pub struct HexString(pub Vec); impl_hex_serde!(HexString); @@ -186,13 +230,26 @@ impl From<&[u8]> for HexString { HexString(value.to_vec()) } } - +impl From for HexString { + fn from(value: Bytes32) -> Self { + Self::from(value.0.as_ref()) + } +} +impl TryFrom for Bytes32 { + type Error = String; + fn try_from(value: HexString) -> Result { + let bytes: [u8; 32] = value + .0 + .try_into() + .map_err(|_| "Invalid length for Bytes32".to_string())?; + Ok(Bytes32::from(bytes)) + } +} impl std::fmt::Display for HexString { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "0x{}", hex::encode(&self.0)) } } - impl std::str::FromStr for HexString { type Err = String; fn from_str(s: &str) -> Result { @@ -200,6 +257,16 @@ impl std::str::FromStr for HexString { hex::decode(s).map(HexString).map_err(|e| e.to_string()) } } +impl AsRef<[u8]> for HexString { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} +impl HexString { + pub fn zeroed() -> Self { + HexString(vec![0u8; 32]) + } +} #[derive( Debug, @@ -230,22 +297,17 @@ impl From for TxPointer { } #[derive( - Debug, - Default, - Copy, - Clone, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - Serialize, - Deserialize, + Debug, Default, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, )] #[serde(rename_all = "camelCase")] pub struct UtxoId { - tx_id: FuelCoreTxId, - output_index: u16, + pub tx_id: Bytes32, + pub output_index: u16, +} +impl From<&UtxoId> for HexString { + fn from(value: &UtxoId) -> Self { + value.to_owned().into() + } } impl From for UtxoId { fn from(value: FuelCoreUtxoId) -> Self { @@ -255,11 +317,67 @@ impl From for UtxoId { impl From<&FuelCoreUtxoId> for UtxoId { fn from(value: &FuelCoreUtxoId) -> Self { Self { - tx_id: *value.tx_id(), + tx_id: value.tx_id().into(), output_index: value.output_index(), } } } +impl From for HexString { + fn from(value: UtxoId) -> Self { + let mut bytes = Vec::with_capacity(34); + bytes.extend_from_slice(value.tx_id.0.as_ref()); + bytes.extend_from_slice(&value.output_index.to_be_bytes()); + HexString(bytes) + } +} + +#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] +pub struct PanicInstruction { + pub reason: PanicReason, + pub instruction: RawInstruction, +} +impl From for PanicInstruction { + fn from(value: FuelCorePanicInstruction) -> Self { + Self { + reason: value.reason().to_owned(), + instruction: value.instruction().to_owned(), + } + } +} + +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + Default, + serde::Serialize, + serde::Deserialize, +)] +#[repr(u64)] +pub enum ScriptExecutionResult { + Success, + Revert, + Panic, + // Generic failure case since any u64 is valid here + GenericFailure(u64), + #[default] + Unknown, +} +impl From for ScriptExecutionResult { + fn from(value: FuelCoreScriptExecutionResult) -> Self { + match value { + FuelCoreScriptExecutionResult::Success => Self::Success, + FuelCoreScriptExecutionResult::Revert => Self::Revert, + FuelCoreScriptExecutionResult::Panic => Self::Panic, + FuelCoreScriptExecutionResult::GenericFailure(value) => { + Self::GenericFailure(value) + } + } + } +} /// Macro to implement conversion from a type to `Bytes32`. /// diff --git a/crates/fuel-streams-core/src/receipts/types.rs b/crates/fuel-streams-core/src/receipts/types.rs index b32e654e..565185d8 100644 --- a/crates/fuel-streams-core/src/receipts/types.rs +++ b/crates/fuel-streams-core/src/receipts/types.rs @@ -1,106 +1,365 @@ +use fuel_core_types::fuel_asm::Word; +use serde::{self, Deserialize, Serialize}; + use crate::types::*; -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum Receipt { + Call(CallReceipt), + Return(ReturnReceipt), + ReturnData(ReturnDataReceipt), + Panic(PanicReceipt), + Revert(RevertReceipt), + Log(LogReceipt), + LogData(LogDataReceipt), + Transfer(TransferReceipt), + TransferOut(TransferOutReceipt), + ScriptResult(ScriptResultReceipt), + MessageOut(MessageOutReceipt), + Mint(MintReceipt), + Burn(BurnReceipt), +} + +// Individual Receipt Types +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CallReceipt { + pub id: ContractId, + pub to: ContractId, + pub amount: Word, + pub asset_id: AssetId, + pub gas: Word, + pub param1: Word, + pub param2: Word, + pub pc: Word, + pub is: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct Receipt { - pub amount: Option, - pub asset_id: Option, +pub struct ReturnReceipt { + pub id: ContractId, + pub val: Word, + pub pc: Word, + pub is: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReturnDataReceipt { + pub id: ContractId, + pub ptr: Word, + pub len: Word, + pub digest: Bytes32, + pub pc: Word, + pub is: Word, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option>, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PanicReceipt { + pub id: ContractId, + pub reason: PanicInstruction, + pub pc: Word, + pub is: Word, + #[serde(skip_serializing_if = "Option::is_none")] pub contract_id: Option, - pub data: Option, - pub digest: Option, - pub gas: Option, - pub gas_used: Option, - pub id: Option, - pub is: Option, - pub len: Option, - pub nonce: Option, - pub param1: Option, - pub param2: Option, - pub pc: Option, - pub ptr: Option, - pub ra: Option, - pub rb: Option, - pub rc: Option, - pub rd: Option, - pub reason: Option, - pub receipt_type: ReceiptType, - pub recipient: Option
, - pub result: Option, - pub sender: Option
, - pub sub_id: Option, - pub to: Option, - pub to_address: Option
, - pub val: Option, } -impl From<&FuelCoreReceipt> for Receipt { - fn from(r: &FuelCoreReceipt) -> Self { - Receipt { - amount: r.amount().map(Into::into), - asset_id: r.asset_id().copied().map(Into::into), - contract_id: r.contract_id().map(Into::into), - data: r.data().map(Into::into), - digest: r.digest().copied().map(Into::into), - gas: r.gas().map(Into::into), - gas_used: r.gas_used().map(Into::into), - id: r.id().map(Into::into), - is: r.is().map(Into::into), - len: r.len().map(Into::into), - nonce: r.nonce().copied().map(Into::into), - param1: r.param1().map(Into::into), - param2: r.param2().map(Into::into), - pc: r.pc().map(Into::into), - ptr: r.ptr().map(Into::into), - ra: r.ra().map(Into::into), - rb: r.rb().map(Into::into), - rc: r.rc().map(Into::into), - rd: r.rd().map(Into::into), - reason: r.reason().map(Into::into), - receipt_type: r.into(), - recipient: r.recipient().copied().map(Into::into), - result: r.result().map(|r| FuelCoreWord::from(*r)), - sender: r.sender().copied().map(Into::into), - sub_id: r.sub_id().copied().map(Into::into), - to: r.to().copied().map(Into::into), - to_address: r.to_address().copied().map(Into::into), - val: r.val().map(Into::into), +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RevertReceipt { + pub id: ContractId, + pub ra: Word, + pub pc: Word, + pub is: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LogReceipt { + pub id: ContractId, + pub ra: Word, + pub rb: Word, + pub rc: Word, + pub rd: Word, + pub pc: Word, + pub is: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LogDataReceipt { + pub id: ContractId, + pub ra: Word, + pub rb: Word, + pub ptr: Word, + pub len: Word, + pub digest: Bytes32, + pub pc: Word, + pub is: Word, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option>, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferReceipt { + pub id: ContractId, + pub to: ContractId, + pub amount: Word, + pub asset_id: AssetId, + pub pc: Word, + pub is: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransferOutReceipt { + pub id: ContractId, + pub to: Address, + pub amount: Word, + pub asset_id: AssetId, + pub pc: Word, + pub is: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptResultReceipt { + pub result: ScriptExecutionResult, + pub gas_used: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MessageOutReceipt { + pub sender: Address, + pub recipient: Address, + pub amount: Word, + pub nonce: Nonce, + pub len: Word, + pub digest: Bytes32, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option>, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MintReceipt { + pub sub_id: Bytes32, + pub contract_id: ContractId, + pub val: Word, + pub pc: Word, + pub is: Word, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BurnReceipt { + pub sub_id: Bytes32, + pub contract_id: ContractId, + pub val: Word, + pub pc: Word, + pub is: Word, +} + +impl From for Receipt { + fn from(value: FuelCoreReceipt) -> Self { + match value { + FuelCoreReceipt::Call { + id, + to, + amount, + asset_id, + gas, + param1, + param2, + pc, + is, + } => Self::Call(CallReceipt { + id: id.into(), + to: to.into(), + amount, + asset_id: asset_id.into(), + gas, + param1, + param2, + pc, + is, + }), + FuelCoreReceipt::Return { id, val, pc, is } => { + Self::Return(ReturnReceipt { + id: id.into(), + val, + pc, + is, + }) + } + FuelCoreReceipt::ReturnData { + id, + ptr, + len, + digest, + pc, + is, + data, + } => Self::ReturnData(ReturnDataReceipt { + id: id.into(), + ptr, + len, + digest: digest.into(), + pc, + is, + data, + }), + FuelCoreReceipt::Panic { + id, + reason, + pc, + is, + contract_id, + } => Self::Panic(PanicReceipt { + id: id.into(), + reason: reason.into(), + pc, + is, + contract_id: contract_id.map(|id| id.into()), + }), + FuelCoreReceipt::Revert { id, ra, pc, is } => { + Self::Revert(RevertReceipt { + id: id.into(), + ra, + pc, + is, + }) + } + FuelCoreReceipt::Log { + id, + ra, + rb, + rc, + rd, + pc, + is, + } => Self::Log(LogReceipt { + id: id.into(), + ra, + rb, + rc, + rd, + pc, + is, + }), + FuelCoreReceipt::LogData { + id, + ra, + rb, + ptr, + len, + digest, + pc, + is, + data, + } => Self::LogData(LogDataReceipt { + id: id.into(), + ra, + rb, + ptr, + len, + digest: digest.into(), + pc, + is, + data, + }), + FuelCoreReceipt::Transfer { + id, + to, + amount, + asset_id, + pc, + is, + } => Self::Transfer(TransferReceipt { + id: id.into(), + to: to.into(), + amount, + asset_id: asset_id.into(), + pc, + is, + }), + FuelCoreReceipt::TransferOut { + id, + to, + amount, + asset_id, + pc, + is, + } => Self::TransferOut(TransferOutReceipt { + id: id.into(), + to: to.into(), + amount, + asset_id: asset_id.into(), + pc, + is, + }), + FuelCoreReceipt::ScriptResult { result, gas_used } => { + Self::ScriptResult(ScriptResultReceipt { + result: result.into(), + gas_used, + }) + } + FuelCoreReceipt::MessageOut { + sender, + recipient, + amount, + nonce, + len, + digest, + data, + } => Self::MessageOut(MessageOutReceipt { + sender: sender.into(), + recipient: recipient.into(), + amount, + nonce: nonce.into(), + len, + digest: digest.into(), + data, + }), + FuelCoreReceipt::Mint { + sub_id, + contract_id, + val, + pc, + is, + } => Self::Mint(MintReceipt { + sub_id: sub_id.into(), + contract_id: contract_id.into(), + val, + pc, + is, + }), + FuelCoreReceipt::Burn { + sub_id, + contract_id, + val, + pc, + is, + } => Self::Burn(BurnReceipt { + sub_id: sub_id.into(), + contract_id: contract_id.into(), + val, + pc, + is, + }), } } } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum ReceiptType { - Burn, - Call, - Log, - LogData, - MessageOut, - Mint, - Panic, - Return, - ReturnData, - Revert, - ScriptResult, - Transfer, - TransferOut, -} - -impl From<&FuelCoreReceipt> for ReceiptType { - fn from(r: &FuelCoreReceipt) -> Self { - match r { - FuelCoreReceipt::Call { .. } => ReceiptType::Call, - FuelCoreReceipt::Return { .. } => ReceiptType::Return, - FuelCoreReceipt::ReturnData { .. } => ReceiptType::ReturnData, - FuelCoreReceipt::Panic { .. } => ReceiptType::Panic, - FuelCoreReceipt::Revert { .. } => ReceiptType::Revert, - FuelCoreReceipt::Log { .. } => ReceiptType::Log, - FuelCoreReceipt::LogData { .. } => ReceiptType::LogData, - FuelCoreReceipt::Transfer { .. } => ReceiptType::Transfer, - FuelCoreReceipt::TransferOut { .. } => ReceiptType::TransferOut, - FuelCoreReceipt::ScriptResult { .. } => ReceiptType::ScriptResult, - FuelCoreReceipt::MessageOut { .. } => ReceiptType::MessageOut, - FuelCoreReceipt::Mint { .. } => ReceiptType::Mint, - FuelCoreReceipt::Burn { .. } => ReceiptType::Burn, - } +impl From<&FuelCoreReceipt> for Receipt { + fn from(value: &FuelCoreReceipt) -> Self { + value.clone().into() } } diff --git a/crates/fuel-streams-core/src/stream/error.rs b/crates/fuel-streams-core/src/stream/error.rs index 27994989..168155da 100644 --- a/crates/fuel-streams-core/src/stream/error.rs +++ b/crates/fuel-streams-core/src/stream/error.rs @@ -12,37 +12,37 @@ use thiserror::Error; #[derive(Error, DisplayDoc, Debug)] pub enum StreamError { - /// Failed to publish to stream: {subject_name} + /// Failed to publish to stream: {subject_name}, error: {source} PublishFailed { subject_name: String, #[source] source: error::Error, }, - /// Failed to publish to S3 + /// Failed to publish to S3: {0} S3PublishError(#[from] fuel_streams_storage::s3::S3ClientError), - /// Failed to retrieve last published message from stream + /// Failed to retrieve last published message from stream: {0} GetLastPublishedFailed(#[from] error::Error), - /// Failed to create Key-Value Store + /// Failed to create Key-Value Store: {0} StoreCreation(#[from] error::Error), - /// Failed to publish item to Key-Value Store + /// Failed to publish item to Key-Value Store: {0} StorePublish(#[from] PutError), - /// Failed to subscribe to subject in Key-Value Store + /// Failed to subscribe to subject in Key-Value Store: {0} StoreSubscribe(#[from] error::Error), - /// Failed to publish item to stream + /// Failed to publish item to stream: {0} StreamPublish(#[from] CreateError), - /// Failed to create stream + /// Failed to create stream: {0} StreamCreation(#[from] error::Error), - /// Failed to create consumer for stream + /// Failed to create consumer for stream: {0} ConsumerCreate(#[from] error::Error), - /// Failed to consume messages from stream + /// Failed to consume messages from stream: {0} ConsumerMessages(#[from] error::Error), } diff --git a/crates/fuel-streams-ws/src/server/ws/fuel_streams.rs b/crates/fuel-streams-core/src/stream/fuel_streams.rs similarity index 76% rename from crates/fuel-streams-ws/src/server/ws/fuel_streams.rs rename to crates/fuel-streams-core/src/stream/fuel_streams.rs index 526550cb..5e8781ba 100644 --- a/crates/fuel-streams-ws/src/server/ws/fuel_streams.rs +++ b/crates/fuel-streams-core/src/stream/fuel_streams.rs @@ -4,12 +4,11 @@ use async_nats::{ jetstream::{context::CreateStreamErrorKind, stream::State as StreamState}, RequestErrorKind, }; -use fuel_streams::types::Log; -use fuel_streams_core::{prelude::*, SubscriptionConfig}; use futures::stream::BoxStream; -#[derive(Clone)] -/// Streams we currently support publishing to. +use crate::prelude::*; + +#[derive(Clone, Debug)] pub struct FuelStreams { pub transactions: Stream, pub blocks: Stream, @@ -20,6 +19,41 @@ pub struct FuelStreams { pub logs: Stream, } +pub struct FuelStreamsUtils; +impl FuelStreamsUtils { + pub fn is_within_subject_names(subject_name: &str) -> bool { + let subject_names = Self::subjects_names(); + subject_names.contains(&subject_name) + } + + pub fn subjects_names() -> &'static [&'static str] { + &[ + Transaction::NAME, + Block::NAME, + Input::NAME, + Receipt::NAME, + Utxo::NAME, + Log::NAME, + ] + } + + pub fn wildcards() -> Vec<&'static str> { + let nested_wildcards = [ + Transaction::WILDCARD_LIST, + Block::WILDCARD_LIST, + Input::WILDCARD_LIST, + Receipt::WILDCARD_LIST, + Utxo::WILDCARD_LIST, + Log::WILDCARD_LIST, + ]; + nested_wildcards + .into_iter() + .flatten() + .copied() + .collect::>() + } +} + impl FuelStreams { pub async fn new( nats_client: &NatsClient, @@ -37,6 +71,16 @@ impl FuelStreams { } } + pub async fn setup_all( + core_client: &NatsClient, + publisher_client: &NatsClient, + s3_client: &Arc, + ) -> (Self, Self) { + let core_stream = Self::new(core_client, s3_client).await; + let publisher_stream = Self::new(publisher_client, s3_client).await; + (core_stream, publisher_stream) + } + pub async fn subscribe( &self, sub_subject: &str, @@ -61,10 +105,14 @@ impl FuelStreams { )), } } + + pub fn arc(self) -> Arc { + Arc::new(self) + } } #[async_trait::async_trait] -pub trait FuelStreamsExt: Sync + Send + 'static { +pub trait FuelStreamsExt: Sync + Send { fn blocks(&self) -> &Stream; fn transactions(&self) -> &Stream; fn inputs(&self) -> &Stream; @@ -75,70 +123,11 @@ pub trait FuelStreamsExt: Sync + Send + 'static { async fn get_last_published_block(&self) -> anyhow::Result>; - fn subjects_names() -> &'static [&'static str] { - &[ - Transaction::NAME, - Block::NAME, - Input::NAME, - Receipt::NAME, - Utxo::NAME, - Log::NAME, - ] - } - - fn is_within_subject_names(subject_name: &str) -> bool { - let subject_names = Self::subjects_names(); - subject_names.contains(&subject_name) - } - - fn subjects_wildcards(&self) -> &[&'static str] { - &[ - TransactionsSubject::WILDCARD, - BlocksSubject::WILDCARD, - InputsByIdSubject::WILDCARD, - InputsCoinSubject::WILDCARD, - InputsMessageSubject::WILDCARD, - InputsContractSubject::WILDCARD, - ReceiptsLogSubject::WILDCARD, - ReceiptsBurnSubject::WILDCARD, - ReceiptsByIdSubject::WILDCARD, - ReceiptsCallSubject::WILDCARD, - ReceiptsMintSubject::WILDCARD, - ReceiptsPanicSubject::WILDCARD, - ReceiptsReturnSubject::WILDCARD, - ReceiptsRevertSubject::WILDCARD, - ReceiptsLogDataSubject::WILDCARD, - ReceiptsTransferSubject::WILDCARD, - ReceiptsMessageOutSubject::WILDCARD, - ReceiptsReturnDataSubject::WILDCARD, - ReceiptsTransferOutSubject::WILDCARD, - ReceiptsScriptResultSubject::WILDCARD, - UtxosSubject::WILDCARD, - LogsSubject::WILDCARD, - ] - } - - fn wildcards() -> Vec<&'static str> { - let nested_wildcards = [ - Transaction::WILDCARD_LIST, - Block::WILDCARD_LIST, - Input::WILDCARD_LIST, - Receipt::WILDCARD_LIST, - Utxo::WILDCARD_LIST, - Log::WILDCARD_LIST, - ]; - nested_wildcards - .into_iter() - .flatten() - .copied() - .collect::>() - } - async fn get_consumers_and_state( &self, ) -> Result, StreamState)>, RequestErrorKind>; - #[cfg(feature = "test-helpers")] + #[cfg(any(test, feature = "test-helpers"))] async fn is_empty(&self) -> bool; } @@ -167,10 +156,10 @@ impl FuelStreamsExt for FuelStreams { } async fn get_last_published_block(&self) -> anyhow::Result> { - Ok(self - .blocks + self.blocks .get_last_published(BlocksSubject::WILDCARD) - .await?) + .await + .map_err(|e| e.into()) } async fn get_consumers_and_state( @@ -187,7 +176,7 @@ impl FuelStreamsExt for FuelStreams { ]) } - #[cfg(feature = "test-helpers")] + #[cfg(any(test, feature = "test-helpers"))] async fn is_empty(&self) -> bool { self.blocks.is_empty(BlocksSubject::WILDCARD).await && self diff --git a/crates/fuel-streams-core/src/stream/mod.rs b/crates/fuel-streams-core/src/stream/mod.rs index 4ea63aff..a204e8d1 100644 --- a/crates/fuel-streams-core/src/stream/mod.rs +++ b/crates/fuel-streams-core/src/stream/mod.rs @@ -1,7 +1,9 @@ mod error; +mod fuel_streams; mod stream_encoding; mod stream_impl; pub use error::*; +pub use fuel_streams::*; pub use stream_encoding::*; pub use stream_impl::*; diff --git a/crates/fuel-streams-core/src/stream/stream_impl.rs b/crates/fuel-streams-core/src/stream/stream_impl.rs index 186f0896..fc8d974b 100644 --- a/crates/fuel-streams-core/src/stream/stream_impl.rs +++ b/crates/fuel-streams-core/src/stream/stream_impl.rs @@ -10,30 +10,29 @@ use async_nats::{ }; use async_trait::async_trait; use fuel_streams_macros::subject::IntoSubject; -use fuel_streams_storage::*; use futures::{stream::BoxStream, StreamExt, TryStreamExt}; -use sha2::{Digest, Sha256}; use tokio::sync::OnceCell; -use super::{error::StreamError, stream_encoding::StreamEncoder}; +use crate::prelude::*; #[derive(Debug, Clone)] pub struct PublishPacket { pub subject: Arc, pub payload: Arc, - pub s3_path: String, } impl PublishPacket { pub fn new(payload: T, subject: Arc) -> Self { - let s3_path = payload.get_s3_path(); - Self { payload: Arc::new(payload), subject, - s3_path, } } + + pub fn get_s3_path(&self) -> String { + let subject = self.subject.parse(); + subject.replace('.', "/").to_string() + } } /// Trait for types that can be streamed. @@ -65,19 +64,6 @@ pub trait Streamable: StreamEncoder + std::marker::Sized { fn to_packet(&self, subject: Arc) -> PublishPacket { PublishPacket::new(self.clone(), subject) } - - fn get_s3_path(&self) -> String { - format!("v1/{}/{}.json", Self::NAME, self.get_consistent_hash()) - } - - fn get_consistent_hash(&self) -> String { - let serialized = self.encode_self(); - - let mut hasher = Sha256::new(); - hasher.update(serialized); - - format!("{:x}", hasher.finalize()) - } } /// Houses nats-agnostic APIs for publishing and consuming a streamable type @@ -150,13 +136,16 @@ impl Stream { ) -> Self { let namespace = &nats_client.namespace; let bucket_name = namespace.stream_name(S::NAME); + let config = kv::Config { + bucket: bucket_name.to_owned(), + storage: stream::StorageType::File, + history: 1, + compression: true, + ..Default::default() + }; + let store = nats_client - .get_or_create_kv_store(kv::Config { - bucket: bucket_name.to_owned(), - storage: stream::StorageType::File, - history: 1, - ..Default::default() - }) + .get_or_create_kv_store(config) .await .expect("Streams must be created"); @@ -172,15 +161,14 @@ impl Stream { packet: &PublishPacket, ) -> Result { let payload = &packet.payload; - let s3_path = &packet.s3_path; + let s3_path = packet.get_s3_path(); let subject_name = &packet.subject.parse(); - // publish payload to S3 self.s3_client - .put_object(s3_path, payload.encode(subject_name)) + .put_object(&s3_path, payload.encode(subject_name)) .await?; - self.publish_s3_path_to_nats(subject_name, s3_path).await + self.publish_s3_path_to_nats(subject_name, &s3_path).await } async fn publish_s3_path_to_nats( @@ -188,9 +176,9 @@ impl Stream { subject_name: &str, s3_path: &str, ) -> Result { + tracing::debug!("S3 path published: {:?}", s3_path); let data = s3_path.to_string().into_bytes(); let data_size = data.len(); - let result = self.store.create(subject_name, data.into()).await; match result { @@ -377,7 +365,6 @@ impl Stream { wildcard: &str, ) -> Result, StreamError> { let subject_name = &Self::prefix_filter_subject(wildcard); - let message = self .store .stream @@ -400,7 +387,6 @@ impl Stream { nats_payload: Vec, ) -> Result { let s3_path = String::from_utf8(nats_payload).expect("Must be S3 path"); - let s3_object = self .s3_client .get_object(&s3_path) @@ -444,6 +430,10 @@ impl Stream { pub fn store(&self) -> &kv::Store { &self.store } + + pub fn arc(&self) -> Arc { + Arc::new(self.to_owned()) + } } /// Configuration for subscribing to a consumer. diff --git a/crates/fuel-streams-core/src/transactions/types.rs b/crates/fuel-streams-core/src/transactions/types.rs index fe6233b5..adeb64fc 100644 --- a/crates/fuel-streams-core/src/transactions/types.rs +++ b/crates/fuel-streams-core/src/transactions/types.rs @@ -1,3 +1,4 @@ +pub use fuel_core_client::client::types::TransactionStatus as ClientTransactionStatus; use fuel_core_types::fuel_tx; use crate::types::*; @@ -405,7 +406,7 @@ impl Transaction { tx_pointer, upgrade_purpose, witnesses, - receipts: receipts.iter().map(Into::into).collect(), + receipts: receipts.iter().map(|r| r.to_owned().into()).collect(), } } } @@ -502,6 +503,31 @@ impl From<&FuelCoreTransactionStatus> for TransactionStatus { } } +impl From<&ClientTransactionStatus> for TransactionStatus { + fn from(value: &ClientTransactionStatus) -> Self { + match value { + ClientTransactionStatus::Failure { .. } => { + TransactionStatus::Failed + } + ClientTransactionStatus::Submitted { .. } => { + TransactionStatus::Submitted + } + ClientTransactionStatus::SqueezedOut { .. } => { + TransactionStatus::SqueezedOut + } + ClientTransactionStatus::Success { .. } => { + TransactionStatus::Success + } + } + } +} + +impl From for TransactionStatus { + fn from(value: ClientTransactionStatus) -> Self { + (&value).into() + } +} + pub trait FuelCoreTransactionExt { fn inputs(&self) -> &[FuelCoreInput]; fn outputs(&self) -> &Vec; diff --git a/crates/fuel-streams-core/src/types.rs b/crates/fuel-streams-core/src/types.rs index efacbacc..9d1c880f 100644 --- a/crates/fuel-streams-core/src/types.rs +++ b/crates/fuel-streams-core/src/types.rs @@ -14,4 +14,5 @@ pub use crate::{ // ------------------------------------------------------------------------ // General // ------------------------------------------------------------------------ -pub type BoxedResult = Result>; +pub type BoxedError = Box; +pub type BoxedResult = Result; diff --git a/crates/fuel-streams-core/src/utxos/subjects.rs b/crates/fuel-streams-core/src/utxos/subjects.rs index 6238f49e..2d89d4c7 100644 --- a/crates/fuel-streams-core/src/utxos/subjects.rs +++ b/crates/fuel-streams-core/src/utxos/subjects.rs @@ -16,12 +16,12 @@ use crate::types::*; /// # use fuel_streams_core::types::*; /// # use fuel_streams_macros::subject::*; /// let subject = UtxosSubject { -/// hash: Some(MessageId::from([1u8; 32])), +/// utxo_id: Some(HexString::zeroed()), /// utxo_type: Some(UtxoType::Message), /// }; /// assert_eq!( /// subject.parse(), -/// "utxos.message.0x0101010101010101010101010101010101010101010101010101010101010101" +/// "utxos.message.0x0000000000000000000000000000000000000000000000000000000000000000" /// ); /// ``` /// @@ -40,10 +40,10 @@ use crate::types::*; /// # use fuel_streams_core::types::*; /// # use fuel_streams_macros::subject::*; /// let wildcard = UtxosSubject::wildcard( -/// Some(MessageId::from([1u8; 32])), +/// Some(HexString::zeroed()), /// None, /// ); -/// assert_eq!(wildcard, "utxos.*.0x0101010101010101010101010101010101010101010101010101010101010101"); +/// assert_eq!(wildcard, "utxos.*.0x0000000000000000000000000000000000000000000000000000000000000000"); /// ``` /// /// Using the builder pattern: @@ -53,16 +53,16 @@ use crate::types::*; /// # use fuel_streams_core::types::*; /// # use fuel_streams_macros::subject::*; /// let subject = UtxosSubject::new() -/// .with_hash(Some(MessageId::from([1u8; 32]))) +/// .with_utxo_id(Some(HexString::zeroed())) /// .with_utxo_type(Some(UtxoType::Message)); -/// assert_eq!(subject.parse(), "utxos.message.0x0101010101010101010101010101010101010101010101010101010101010101"); +/// assert_eq!(subject.parse(), "utxos.message.0x0000000000000000000000000000000000000000000000000000000000000000"); /// ``` #[derive(Subject, Debug, Clone, Default)] #[subject_wildcard = "utxos.>"] -#[subject_format = "utxos.{utxo_type}.{hash}"] +#[subject_format = "utxos.{utxo_type}.{utxo_id}"] pub struct UtxosSubject { - pub hash: Option, + pub utxo_id: Option, pub utxo_type: Option, } @@ -80,7 +80,7 @@ mod tests { #[test] fn test_utxos_message_subject_creation() { let utxo_subject = UtxosSubject::new() - .with_hash(Some(MessageId::zeroed())) + .with_utxo_id(Some(HexString::zeroed())) .with_utxo_type(Some(UtxoType::Message)); assert_eq!( utxo_subject.to_string(), @@ -91,7 +91,7 @@ mod tests { #[test] fn test_utxos_coin_subject_creation() { let utxo_subject = UtxosSubject::new() - .with_hash(Some(MessageId::zeroed())) + .with_utxo_id(Some(HexString::zeroed())) .with_utxo_type(Some(UtxoType::Coin)); assert_eq!( utxo_subject.to_string(), @@ -102,7 +102,7 @@ mod tests { #[test] fn test_utxos_contract_subject_creation() { let utxo_subject = UtxosSubject::new() - .with_hash(Some(MessageId::zeroed())) + .with_utxo_id(Some(HexString::zeroed())) .with_utxo_type(Some(UtxoType::Contract)); assert_eq!( utxo_subject.to_string(), diff --git a/crates/fuel-streams-core/src/utxos/types.rs b/crates/fuel-streams-core/src/utxos/types.rs index 2e067f29..1fd8daec 100644 --- a/crates/fuel-streams-core/src/utxos/types.rs +++ b/crates/fuel-streams-core/src/utxos/types.rs @@ -7,7 +7,7 @@ pub struct Utxo { pub sender: Option
, pub recipient: Option
, pub nonce: Option, - pub data: Option>, + pub data: Option, pub amount: Option, pub tx_id: Bytes32, } diff --git a/benches/nats-publisher/Cargo.toml b/crates/fuel-streams-executors/Cargo.toml similarity index 52% rename from benches/nats-publisher/Cargo.toml rename to crates/fuel-streams-executors/Cargo.toml index 365a8df3..7b70773d 100644 --- a/benches/nats-publisher/Cargo.toml +++ b/crates/fuel-streams-executors/Cargo.toml @@ -1,5 +1,6 @@ [package] -name = "nats-publisher" +name = "fuel-streams-executors" +description = "Executors for Fuel Streams entities" authors = { workspace = true } keywords = { workspace = true } edition = { workspace = true } @@ -13,16 +14,18 @@ publish = false [dependencies] anyhow = { workspace = true } async-nats = { workspace = true } -clap = { workspace = true } fuel-core = { workspace = true } -fuel-core-bin = { workspace = true } -fuel-core-importer = { workspace = true } -fuel-core-storage = { workspace = true } -fuel-core-types = { workspace = true } -fuel-data-parser = { workspace = true } -fuel-streams-core = { workspace = true } +fuel-streams-core = { workspace = true, features = ["test-helpers"] } +futures = { workspace = true } +num_cpus = { workspace = true } +rayon = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -[dev-dependencies] -criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } +[features] +default = [] +test-helpers = [] diff --git a/crates/fuel-streams-executors/src/blocks.rs b/crates/fuel-streams-executors/src/blocks.rs new file mode 100644 index 00000000..5a145e2d --- /dev/null +++ b/crates/fuel-streams-executors/src/blocks.rs @@ -0,0 +1,70 @@ +use std::sync::Arc; + +use fuel_streams_core::prelude::*; +use futures::stream::FuturesUnordered; +use tokio::task::JoinHandle; + +use crate::*; + +impl Executor { + pub fn process(&self) -> JoinHandle> { + let metadata = self.metadata(); + let block = self.block(); + let block_height = (*metadata.block_height).clone(); + let block_producer = (*metadata.block_producer).clone(); + let packet = PublishPacket::::new( + block.to_owned(), + BlocksSubject { + height: Some(block_height), + producer: Some(block_producer), + } + .arc(), + ); + self.publish(&packet) + } + + pub fn process_all( + payload: Arc, + fuel_streams: &Arc, + semaphore: &Arc, + ) -> FuturesUnordered>> { + let block_stream = fuel_streams.blocks().arc(); + let tx_stream = fuel_streams.transactions().arc(); + let input_stream = fuel_streams.inputs().arc(); + let output_stream = fuel_streams.outputs().arc(); + let receipt_stream = fuel_streams.receipts().arc(); + let log_stream = fuel_streams.logs().arc(); + let utxo_stream = fuel_streams.utxos().arc(); + + let block_executor = Executor::new(&payload, &block_stream, semaphore); + let tx_executor = Executor::new(&payload, &tx_stream, semaphore); + let input_executor = Executor::new(&payload, &input_stream, semaphore); + let output_executor = + Executor::new(&payload, &output_stream, semaphore); + let receipt_executor = + Executor::new(&payload, &receipt_stream, semaphore); + let log_executor = Executor::new(&payload, &log_stream, semaphore); + let utxo_executor = Executor::new(&payload, &utxo_stream, semaphore); + + let transactions = payload.transactions.to_owned(); + let tx_tasks = + transactions + .iter() + .enumerate() + .flat_map(|tx_item @ (_, tx)| { + vec![ + tx_executor.process(tx_item), + input_executor.process(tx), + output_executor.process(tx), + receipt_executor.process(tx), + log_executor.process(tx), + utxo_executor.process(tx), + ] + }); + + let block_task = block_executor.process(); + std::iter::once(block_task) + .chain(tx_tasks.into_iter().flatten()) + .collect::>() + } +} diff --git a/crates/fuel-streams-executors/src/inputs.rs b/crates/fuel-streams-executors/src/inputs.rs new file mode 100644 index 00000000..0c201b3b --- /dev/null +++ b/crates/fuel-streams-executors/src/inputs.rs @@ -0,0 +1,129 @@ +use std::sync::Arc; + +use fuel_streams_core::prelude::*; +use rayon::prelude::*; +use tokio::task::JoinHandle; + +use crate::*; + +impl Executor { + pub fn process( + &self, + tx: &Transaction, + ) -> Vec>> { + let tx_id = tx.id.clone(); + let packets = tx + .inputs + .par_iter() + .enumerate() + .flat_map(move |(index, input)| { + let main_subject = main_subject(input, tx_id.clone(), index); + let identifier_subjects = + identifiers(input, &tx_id, index as u8) + .into_par_iter() + .map(|identifier| identifier.into()) + .map(|subject: InputsByIdSubject| subject.arc()) + .collect::>(); + + let mut packets = vec![input.to_packet(main_subject)]; + packets.extend( + identifier_subjects + .into_iter() + .map(|subject| input.to_packet(subject)), + ); + + packets + }) + .collect::>(); + + packets.iter().map(|packet| self.publish(packet)).collect() + } +} + +fn main_subject( + input: &Input, + tx_id: Bytes32, + index: usize, +) -> Arc { + match input { + Input::Contract(contract) => InputsContractSubject { + tx_id: Some(tx_id), + index: Some(index), + contract_id: Some(contract.contract_id.to_owned().into()), + } + .arc(), + Input::Coin(coin) => InputsCoinSubject { + tx_id: Some(tx_id), + index: Some(index), + owner: Some(coin.owner.to_owned()), + asset_id: Some(coin.asset_id.to_owned()), + } + .arc(), + Input::Message(message) => InputsMessageSubject { + tx_id: Some(tx_id), + index: Some(index), + sender: Some(message.sender.to_owned()), + recipient: Some(message.recipient.to_owned()), + } + .arc(), + } +} + +pub fn identifiers( + input: &Input, + tx_id: &Bytes32, + index: u8, +) -> Vec { + let mut identifiers = match input { + Input::Coin(coin) => { + vec![ + Identifier::Address( + tx_id.to_owned(), + index, + coin.owner.to_owned().into(), + ), + Identifier::AssetID( + tx_id.to_owned(), + index, + coin.asset_id.to_owned().into(), + ), + ] + } + Input::Message(message) => { + vec![ + Identifier::Address( + tx_id.to_owned(), + index, + message.sender.to_owned().into(), + ), + Identifier::Address( + tx_id.to_owned(), + index, + message.recipient.to_owned().into(), + ), + ] + } + Input::Contract(contract) => { + vec![Identifier::ContractID( + tx_id.to_owned(), + index, + contract.contract_id.to_owned(), + )] + } + }; + + match input { + Input::Coin(InputCoin { predicate, .. }) + | Input::Message(InputMessage { predicate, .. }) => { + let predicate_tag = super::sha256(&predicate.0); + identifiers.push(Identifier::PredicateID( + tx_id.to_owned(), + index, + predicate_tag, + )); + } + _ => {} + }; + + identifiers +} diff --git a/crates/fuel-streams-executors/src/lib.rs b/crates/fuel-streams-executors/src/lib.rs new file mode 100644 index 00000000..a227758d --- /dev/null +++ b/crates/fuel-streams-executors/src/lib.rs @@ -0,0 +1,255 @@ +pub mod blocks; +pub mod inputs; +pub mod logs; +pub mod outputs; +pub mod receipts; +pub mod transactions; +pub mod utxos; + +use std::{ + env, + marker::PhantomData, + sync::{Arc, LazyLock}, +}; + +use async_nats::jetstream::context::Publish; +use fuel_streams_core::prelude::*; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::task::JoinHandle; + +pub static PUBLISHER_MAX_THREADS: LazyLock = LazyLock::new(|| { + let available_cpus = num_cpus::get(); + env::var("PUBLISHER_MAX_THREADS") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(available_cpus) +}); + +pub fn sha256(bytes: &[u8]) -> Bytes32 { + let mut sha256 = Sha256::new(); + sha256.update(bytes); + let bytes: [u8; 32] = sha256 + .finalize() + .as_slice() + .try_into() + .expect("Must be 32 bytes"); + + bytes.into() +} + +#[derive(Debug, thiserror::Error)] +pub enum ExecutorError { + #[error("Failed to publish: {0}")] + PublishFailed(String), + #[error("Failed to acquire semaphore: {0}")] + SemaphoreError(#[from] tokio::sync::AcquireError), + #[error("Failed to serialize block payload: {0}")] + Serialization(#[from] serde_json::Error), + #[error("Failed to fetch transaction status: {0}")] + TransactionStatus(String), + #[error("Failed to access offchain database")] + OffchainDatabase(#[from] anyhow::Error), + #[error("Failed to join tasks: {0}")] + JoinError(#[from] tokio::task::JoinError), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Metadata { + pub chain_id: Arc, + pub base_asset_id: Arc, + pub block_producer: Arc
, + pub block_height: Arc, + pub consensus: Arc, +} + +impl Metadata { + pub fn new( + fuel_core: &Arc, + sealed_block: &FuelCoreSealedBlock, + ) -> Self { + let block = sealed_block.entity.clone(); + let consensus = sealed_block.consensus.clone(); + let height = *block.header().consensus().height; + let producer = + consensus.block_producer(&block.id()).unwrap_or_default(); + Self { + chain_id: Arc::new(*fuel_core.chain_id()), + base_asset_id: Arc::new(*fuel_core.base_asset_id()), + block_producer: Arc::new(producer.into()), + block_height: Arc::new(height.into()), + consensus: Arc::new(consensus.into()), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BlockPayload { + pub block: Block, + pub transactions: Vec, + metadata: Metadata, +} + +impl BlockPayload { + pub fn new( + fuel_core: Arc, + sealed_block: &FuelCoreSealedBlock, + metadata: &Metadata, + ) -> Result { + let block = sealed_block.entity.clone(); + let txs = Self::txs_from_fuelcore( + fuel_core.to_owned(), + sealed_block, + metadata, + )?; + let txs_ids = txs.iter().map(|i| i.id.clone()).collect(); + let block_height = block.header().height(); + let consensus = fuel_core.get_consensus(block_height)?; + let block = Block::new(&block, consensus.into(), txs_ids); + Ok(Self { + block, + transactions: txs, + metadata: metadata.to_owned(), + }) + } + + pub fn encode(&self) -> Result { + serde_json::to_string(self).map_err(ExecutorError::from) + } + + pub fn decode(json: &str) -> Result { + serde_json::from_str(json).map_err(ExecutorError::from) + } + + pub fn tx_ids(&self) -> Vec { + self.transactions + .iter() + .map(|tx| tx.id.clone()) + .collect::>() + } + + pub fn message_id(&self) -> String { + let height = self.metadata.block_height.clone(); + format!("block_{height}") + } + + pub fn subject(&self) -> String { + let producer = self.metadata.block_producer.clone(); + let height = self.metadata.block_height.clone(); + format!("block_submitted.{producer}.{height}") + } + + pub fn metadata(&self) -> &Metadata { + &self.metadata + } + + pub fn block_height(&self) -> u32 { + self.block.height + } + + pub fn arc(&self) -> Arc { + Arc::new(self.clone()) + } + + pub fn txs_from_fuelcore( + fuel_core: Arc, + sealed_block: &FuelCoreSealedBlock, + metadata: &Metadata, + ) -> Result, ExecutorError> { + let mut transactions: Vec = vec![]; + let blocks_txs = sealed_block.entity.transactions_vec(); + for tx_item in blocks_txs.iter() { + let tx_id = tx_item.id(&metadata.chain_id); + let receipts = fuel_core.get_receipts(&tx_id)?.unwrap_or_default(); + let tx_status = fuel_core.get_tx_status(&tx_id)?; + let tx_status: TransactionStatus = match tx_status { + Some(status) => (&status).into(), + _ => TransactionStatus::None, + }; + let new_transaction = Transaction::new( + &tx_id.into(), + tx_item, + &tx_status, + &metadata.base_asset_id, + &receipts, + ); + transactions.push(new_transaction); + } + Ok(transactions) + } +} + +impl TryFrom for Publish { + type Error = ExecutorError; + fn try_from(payload: BlockPayload) -> Result { + let message_id = payload.message_id(); + Ok(Publish::build() + .message_id(message_id) + .payload(payload.encode()?.into())) + } +} + +pub struct Executor { + pub stream: Arc>, + payload: Arc, + semaphore: Arc, + __marker: PhantomData, +} + +impl Executor { + pub fn new( + payload: &Arc, + stream: &Arc>, + semaphore: &Arc, + ) -> Self { + Self { + payload: payload.to_owned(), + stream: stream.to_owned(), + semaphore: semaphore.to_owned(), + __marker: PhantomData, + } + } + + fn publish( + &self, + packet: &PublishPacket, + ) -> JoinHandle> { + let wildcard = packet.subject.parse(); + let stream = Arc::clone(&self.stream); + let permit = Arc::clone(&self.semaphore); + + // TODO: add telemetry back again + let packet = packet.clone(); + tokio::spawn({ + async move { + let _permit = permit.acquire().await?; + match stream.publish(&packet).await { + Ok(_) => { + tracing::debug!( + "Successfully published for stream: {wildcard}" + ); + Ok(()) + } + Err(e) => { + tracing::error!("Failed to publish for stream: {wildcard}, error: {e}"); + Err(ExecutorError::PublishFailed(e.to_string())) + } + } + } + }) + } + + pub fn payload(&self) -> Arc { + Arc::clone(&self.payload) + } + pub fn metadata(&self) -> &Metadata { + &self.payload.metadata + } + pub fn block(&self) -> &Block { + &self.payload.block + } + pub fn block_height(&self) -> BlockHeight { + let height = self.block().height; + BlockHeight::from(height) + } +} diff --git a/crates/fuel-streams-executors/src/logs.rs b/crates/fuel-streams-executors/src/logs.rs new file mode 100644 index 00000000..f4ac0308 --- /dev/null +++ b/crates/fuel-streams-executors/src/logs.rs @@ -0,0 +1,38 @@ +use fuel_streams_core::prelude::*; +use rayon::prelude::*; +use tokio::task::JoinHandle; + +use crate::*; + +impl Executor { + pub fn process( + &self, + tx: &Transaction, + ) -> Vec>> { + let block_height = self.block_height(); + let tx_id = tx.id.clone(); + let receipts = tx.receipts.clone(); + let packets = receipts + .par_iter() + .enumerate() + .filter_map(|(index, receipt)| match receipt { + Receipt::Log(LogReceipt { id, .. }) + | Receipt::LogData(LogDataReceipt { id, .. }) => { + Some(PublishPacket::new( + receipt.to_owned().into(), + LogsSubject { + block_height: Some(block_height.clone()), + tx_id: Some(tx_id.to_owned()), + receipt_index: Some(index), + log_id: Some(id.into()), + } + .arc(), + )) + } + _ => None, + }) + .collect::>(); + + packets.iter().map(|packet| self.publish(packet)).collect() + } +} diff --git a/crates/fuel-streams-executors/src/outputs.rs b/crates/fuel-streams-executors/src/outputs.rs new file mode 100644 index 00000000..48e9baa8 --- /dev/null +++ b/crates/fuel-streams-executors/src/outputs.rs @@ -0,0 +1,155 @@ +use std::sync::Arc; + +use fuel_streams_core::prelude::*; +use rayon::prelude::*; +use tokio::task::JoinHandle; + +use crate::*; + +impl Executor { + pub fn process( + &self, + tx: &Transaction, + ) -> Vec>> { + let tx_id = tx.id.clone(); + let packets: Vec> = tx + .outputs + .par_iter() + .enumerate() + .flat_map(|(index, output)| { + let main_subject = main_subject(output, tx, &tx_id, index); + let identifier_subjects = + identifiers(output, tx, &tx_id, index as u8) + .into_par_iter() + .map(|identifier| identifier.into()) + .map(|subject: OutputsByIdSubject| subject.arc()) + .collect::>(); + + let mut packets = vec![output.to_packet(main_subject)]; + packets.extend( + identifier_subjects + .into_iter() + .map(|subject| output.to_packet(subject)), + ); + + packets + }) + .collect(); + + packets.iter().map(|packet| self.publish(packet)).collect() + } +} + +fn main_subject( + output: &Output, + transaction: &Transaction, + tx_id: &Bytes32, + index: usize, +) -> Arc { + match output { + Output::Coin(OutputCoin { to, asset_id, .. }) => OutputsCoinSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index as u16), + to: Some(to.to_owned()), + asset_id: Some(asset_id.to_owned()), + } + .arc(), + Output::Contract(contract) => { + let contract_id = + match find_output_contract_id(transaction, contract) { + Some(contract_id) => contract_id, + None => { + tracing::warn!( + "Contract ID not found for output: {:?}", + output + ); + + Default::default() + } + }; + + OutputsContractSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index as u16), + contract_id: Some(contract_id), + } + .arc() + } + Output::Change(OutputChange { to, asset_id, .. }) => { + OutputsChangeSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index as u16), + to: Some(to.to_owned()), + asset_id: Some(asset_id.to_owned()), + } + .arc() + } + Output::Variable(OutputVariable { to, asset_id, .. }) => { + OutputsVariableSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index as u16), + to: Some(to.to_owned()), + asset_id: Some(asset_id.to_owned()), + } + .arc() + } + Output::ContractCreated(OutputContractCreated { + contract_id, .. + }) => OutputsContractCreatedSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index as u16), + contract_id: Some(contract_id.to_owned()), + } + .arc(), + } +} + +pub fn identifiers( + output: &Output, + tx: &Transaction, + tx_id: &Bytes32, + index: u8, +) -> Vec { + match output { + Output::Change(OutputChange { to, asset_id, .. }) + | Output::Variable(OutputVariable { to, asset_id, .. }) + | Output::Coin(OutputCoin { to, asset_id, .. }) => { + vec![ + Identifier::Address(tx_id.to_owned(), index, to.into()), + Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), + ] + } + Output::Contract(contract) => find_output_contract_id(tx, contract) + .map(|contract_id| { + vec![Identifier::ContractID( + tx_id.to_owned(), + index, + contract_id.into(), + )] + }) + .unwrap_or_default(), + Output::ContractCreated(OutputContractCreated { + contract_id, .. + }) => { + vec![Identifier::ContractID( + tx_id.to_owned(), + index, + contract_id.into(), + )] + } + } +} + +pub fn find_output_contract_id( + tx: &Transaction, + contract: &OutputContract, +) -> Option { + let input_index = contract.input_index as usize; + tx.inputs.get(input_index).and_then(|input| { + if let Input::Contract(input_contract) = input { + Some(input_contract.contract_id.to_owned().into()) + } else { + None + } + }) +} diff --git a/crates/fuel-streams-executors/src/receipts.rs b/crates/fuel-streams-executors/src/receipts.rs new file mode 100644 index 00000000..c99f03fc --- /dev/null +++ b/crates/fuel-streams-executors/src/receipts.rs @@ -0,0 +1,238 @@ +use std::sync::Arc; + +use fuel_streams_core::prelude::*; +use rayon::prelude::*; +use tokio::task::JoinHandle; + +use crate::*; + +impl Executor { + pub fn process( + &self, + tx: &Transaction, + ) -> Vec>> { + let tx_id = tx.id.clone(); + let receipts = tx.receipts.clone(); + let packets: Vec> = receipts + .par_iter() + .enumerate() + .flat_map(|(index, receipt)| { + let main_subject = main_subject(receipt, &tx_id, index); + let identifier_subjects = + identifiers(receipt, &tx_id, index as u8) + .into_par_iter() + .map(|identifier| identifier.into()) + .map(|subject: ReceiptsByIdSubject| subject.arc()) + .collect::>(); + + let receipt: Receipt = receipt.to_owned(); + let mut packets = vec![receipt.to_packet(main_subject)]; + packets.extend( + identifier_subjects + .into_iter() + .map(|subject| receipt.to_packet(subject)), + ); + + packets + }) + .collect(); + + packets.iter().map(|packet| self.publish(packet)).collect() + } +} + +fn main_subject( + receipt: &Receipt, + tx_id: &Bytes32, + index: usize, +) -> Arc { + match receipt { + Receipt::Call(CallReceipt { + id: from, + to, + asset_id, + .. + }) => ReceiptsCallSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + from: Some(from.to_owned()), + to: Some(to.to_owned()), + asset_id: Some(asset_id.to_owned()), + } + .arc(), + Receipt::Return(ReturnReceipt { id, .. }) => ReceiptsReturnSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + id: Some(id.to_owned()), + } + .arc(), + Receipt::ReturnData(ReturnDataReceipt { id, .. }) => { + ReceiptsReturnDataSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + id: Some(id.to_owned()), + } + .arc() + } + Receipt::Panic(PanicReceipt { id, .. }) => ReceiptsPanicSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + id: Some(id.to_owned()), + } + .arc(), + Receipt::Revert(RevertReceipt { id, .. }) => ReceiptsRevertSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + id: Some(id.to_owned()), + } + .arc(), + Receipt::Log(LogReceipt { id, .. }) => ReceiptsLogSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + id: Some(id.to_owned()), + } + .arc(), + Receipt::LogData(LogDataReceipt { id, .. }) => ReceiptsLogDataSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + id: Some(id.to_owned()), + } + .arc(), + Receipt::Transfer(TransferReceipt { + id: from, + to, + asset_id, + .. + }) => ReceiptsTransferSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + from: Some(from.to_owned()), + to: Some(to.to_owned()), + asset_id: Some(asset_id.to_owned()), + } + .arc(), + + Receipt::TransferOut(TransferOutReceipt { + id: from, + to, + asset_id, + .. + }) => ReceiptsTransferOutSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + from: Some(from.to_owned()), + to: Some(to.to_owned()), + asset_id: Some(asset_id.to_owned()), + } + .arc(), + + Receipt::ScriptResult(ScriptResultReceipt { .. }) => { + ReceiptsScriptResultSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + } + .arc() + } + Receipt::MessageOut(MessageOutReceipt { + sender, recipient, .. + }) => ReceiptsMessageOutSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + sender: Some(sender.to_owned()), + recipient: Some(recipient.to_owned()), + } + .arc(), + Receipt::Mint(MintReceipt { + contract_id, + sub_id, + .. + }) => ReceiptsMintSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + contract_id: Some(contract_id.to_owned()), + sub_id: Some((*sub_id).to_owned()), + } + .arc(), + Receipt::Burn(BurnReceipt { + contract_id, + sub_id, + .. + }) => ReceiptsBurnSubject { + tx_id: Some(tx_id.to_owned()), + index: Some(index), + contract_id: Some(contract_id.to_owned()), + sub_id: Some((*sub_id).to_owned()), + } + .arc(), + } +} + +pub fn identifiers( + receipt: &Receipt, + tx_id: &Bytes32, + index: u8, +) -> Vec { + match receipt { + Receipt::Call(CallReceipt { + id: from, + to, + asset_id, + .. + }) => { + vec![ + Identifier::ContractID(tx_id.to_owned(), index, from.into()), + Identifier::ContractID(tx_id.to_owned(), index, to.into()), + Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), + ] + } + Receipt::Return(ReturnReceipt { id, .. }) + | Receipt::ReturnData(ReturnDataReceipt { id, .. }) + | Receipt::Panic(PanicReceipt { id, .. }) + | Receipt::Revert(RevertReceipt { id, .. }) + | Receipt::Log(LogReceipt { id, .. }) + | Receipt::LogData(LogDataReceipt { id, .. }) => { + vec![Identifier::ContractID(tx_id.to_owned(), index, id.into())] + } + Receipt::Transfer(TransferReceipt { + id: from, + to, + asset_id, + .. + }) => { + vec![ + Identifier::ContractID(tx_id.to_owned(), index, from.into()), + Identifier::ContractID(tx_id.to_owned(), index, to.into()), + Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), + ] + } + Receipt::TransferOut(TransferOutReceipt { + id: from, + to, + asset_id, + .. + }) => { + vec![ + Identifier::ContractID(tx_id.to_owned(), index, from.into()), + Identifier::ContractID(tx_id.to_owned(), index, to.into()), + Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), + ] + } + Receipt::MessageOut(MessageOutReceipt { + sender, recipient, .. + }) => { + vec![ + Identifier::Address(tx_id.to_owned(), index, sender.into()), + Identifier::Address(tx_id.to_owned(), index, recipient.into()), + ] + } + Receipt::Mint(MintReceipt { contract_id, .. }) + | Receipt::Burn(BurnReceipt { contract_id, .. }) => { + vec![Identifier::ContractID( + tx_id.to_owned(), + index, + contract_id.into(), + )] + } + _ => Vec::new(), + } +} diff --git a/crates/fuel-streams-executors/src/transactions.rs b/crates/fuel-streams-executors/src/transactions.rs new file mode 100644 index 00000000..396364ca --- /dev/null +++ b/crates/fuel-streams-executors/src/transactions.rs @@ -0,0 +1,82 @@ +use fuel_streams_core::prelude::*; +use rayon::prelude::*; +use tokio::task::JoinHandle; + +use crate::*; + +impl Executor { + pub fn process( + &self, + tx_item: (usize, &Transaction), + ) -> Vec>> { + let block_height = self.block_height(); + packets_from_tx(tx_item, &block_height) + .iter() + .map(|packet| self.publish(packet)) + .collect() + } +} + +fn packets_from_tx( + (index, tx): (usize, &Transaction), + block_height: &BlockHeight, +) -> Vec> { + let estimated_capacity = + 1 + tx.inputs.len() + tx.outputs.len() + tx.receipts.len(); + let tx_id = tx.id.clone(); + let tx_status = tx.status.clone(); + let receipts = tx.receipts.clone(); + + // Main subject + let mut packets = Vec::with_capacity(estimated_capacity); + packets.push( + tx.to_packet( + TransactionsSubject { + block_height: Some(block_height.to_owned()), + index: Some(index), + tx_id: Some(tx_id.to_owned()), + status: Some(tx_status), + kind: Some(tx.kind.to_owned()), + } + .arc(), + ), + ); + + let index_u8 = index as u8; + let mut additional_packets: Vec> = + rayon::iter::once(&tx.kind) + .flat_map(|kind| identifiers(tx, kind, &tx_id, index_u8)) + .chain( + tx.inputs.par_iter().flat_map(|input| { + inputs::identifiers(input, &tx_id, index_u8) + }), + ) + .chain(tx.outputs.par_iter().flat_map(|output| { + outputs::identifiers(output, tx, &tx_id, index_u8) + })) + .chain(receipts.par_iter().flat_map(|receipt| { + receipts::identifiers(receipt, &tx_id, index_u8) + })) + .map(|identifier| TransactionsByIdSubject::from(identifier).arc()) + .map(|subject| tx.to_packet(subject)) + .collect(); + + packets.append(&mut additional_packets); + packets +} + +fn identifiers( + tx: &Transaction, + kind: &TransactionKind, + tx_id: &Bytes32, + index: u8, +) -> Vec { + match kind { + TransactionKind::Script => { + let script_data = &tx.script_data.to_owned().unwrap_or_default().0; + let script_tag = sha256(script_data); + vec![Identifier::ScriptID(tx_id.to_owned(), index, script_tag)] + } + _ => Vec::new(), + } +} diff --git a/crates/fuel-streams-executors/src/utxos.rs b/crates/fuel-streams-executors/src/utxos.rs new file mode 100644 index 00000000..81fdaaf2 --- /dev/null +++ b/crates/fuel-streams-executors/src/utxos.rs @@ -0,0 +1,85 @@ +use fuel_streams_core::prelude::*; +use rayon::prelude::*; +use tokio::task::JoinHandle; + +use crate::*; + +impl Executor { + pub fn process( + &self, + tx: &Transaction, + ) -> Vec>> { + let tx_id = tx.id.clone(); + let packets = tx + .inputs + .par_iter() + .filter_map(|input| utxo_packet(input, &tx_id)) + .collect::>(); + + packets + .into_iter() + .map(|packet| self.publish(&packet)) + .collect() + } +} + +fn utxo_packet(input: &Input, tx_id: &Bytes32) -> Option> { + match input { + Input::Contract(InputContract { utxo_id, .. }) => { + let utxo = Utxo { + utxo_id: utxo_id.to_owned(), + tx_id: tx_id.to_owned(), + ..Default::default() + }; + let subject = UtxosSubject { + utxo_type: Some(UtxoType::Contract), + utxo_id: Some(utxo_id.into()), + } + .arc(); + Some(utxo.to_packet(subject)) + } + Input::Coin(InputCoin { + utxo_id, amount, .. + }) => { + let utxo = Utxo { + utxo_id: utxo_id.to_owned(), + amount: Some(*amount), + tx_id: tx_id.to_owned(), + ..Default::default() + }; + let subject = UtxosSubject { + utxo_type: Some(UtxoType::Coin), + utxo_id: Some(utxo_id.into()), + } + .arc(); + Some(utxo.to_packet(subject)) + } + Input::Message( + input @ InputMessage { + amount, + nonce, + recipient, + sender, + data, + .. + }, + ) => { + let utxo_id = input.computed_utxo_id(); + let utxo = Utxo { + tx_id: tx_id.to_owned(), + utxo_id: utxo_id.to_owned(), + sender: Some(sender.to_owned()), + recipient: Some(recipient.to_owned()), + nonce: Some(nonce.to_owned()), + amount: Some(*amount), + data: Some(data.to_owned()), + }; + let subject = UtxosSubject { + utxo_type: Some(UtxoType::Message), + utxo_id: None, + } + .arc(); + Some(utxo.to_packet(subject)) + } + } +} diff --git a/crates/fuel-streams-nats/Cargo.toml b/crates/fuel-streams-nats/Cargo.toml new file mode 100644 index 00000000..60254899 --- /dev/null +++ b/crates/fuel-streams-nats/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "fuel-streams-nats" +description = "Strategies and adapters for storing fuel streams in NATS" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +async-nats = { workspace = true } +displaydoc = { workspace = true } +dotenvy = { workspace = true } +rand = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } + +[features] +default = [] +test-helpers = [] +bench-helpers = [] diff --git a/crates/fuel-streams-storage/src/nats/error.rs b/crates/fuel-streams-nats/src/error.rs similarity index 100% rename from crates/fuel-streams-storage/src/nats/error.rs rename to crates/fuel-streams-nats/src/error.rs diff --git a/crates/fuel-streams-nats/src/lib.rs b/crates/fuel-streams-nats/src/lib.rs new file mode 100644 index 00000000..762ae550 --- /dev/null +++ b/crates/fuel-streams-nats/src/lib.rs @@ -0,0 +1,15 @@ +/// Houses shared APIs for interacting with NATS for sv-publisher and fuel-streams crates +/// As much as possible, the public interface/APIS should be agnostic of NATS. These can then be extended +/// in the sv-publisher and fuel-streams crates to provide a more opinionated API towards +/// their specific use-cases. +pub mod error; +pub mod nats_client; +pub mod nats_client_opts; +pub mod nats_namespace; +pub mod types; + +pub use error::*; +pub use nats_client::*; +pub use nats_client_opts::*; +pub use nats_namespace::*; +pub use types::*; diff --git a/crates/fuel-streams-storage/src/nats/nats_client.rs b/crates/fuel-streams-nats/src/nats_client.rs similarity index 86% rename from crates/fuel-streams-storage/src/nats/nats_client.rs rename to crates/fuel-streams-nats/src/nats_client.rs index 3c09323b..fd3474fa 100644 --- a/crates/fuel-streams-storage/src/nats/nats_client.rs +++ b/crates/fuel-streams-nats/src/nats_client.rs @@ -14,11 +14,10 @@ use super::{types::*, NatsClientOpts, NatsError, NatsNamespace}; /// Creating a new `NatsClient`: /// /// ```no_run -/// use fuel_streams_storage::nats::*; -/// use fuel_networks::FuelNetwork; +/// use fuel_streams_nats::*; /// /// async fn example() -> Result<(), Box> { -/// let opts = NatsClientOpts::new(FuelNetwork::Local); +/// let opts = NatsClientOpts::public_opts(); /// let client = NatsClient::connect(&opts).await?; /// Ok(()) /// } @@ -27,12 +26,11 @@ use super::{types::*, NatsClientOpts, NatsError, NatsNamespace}; /// Creating a key-value store: /// /// ```no_run -/// use fuel_streams_storage::nats::*; -/// use fuel_networks::FuelNetwork; +/// use fuel_streams_nats::*; /// use async_nats::jetstream::kv; /// /// async fn example() -> Result<(), Box> { -/// let opts = NatsClientOpts::new(FuelNetwork::Local); +/// let opts = NatsClientOpts::public_opts(); /// let client = NatsClient::connect(&opts).await?; /// let kv_config = kv::Config { /// bucket: "my-bucket".into(), @@ -66,7 +64,13 @@ impl NatsClient { source: e, } })?; - let jetstream = async_nats::jetstream::new(nats_client.to_owned()); + + let jetstream = match opts.domain.clone() { + None => async_nats::jetstream::new(nats_client.clone()), + Some(domain) => { + async_nats::jetstream::with_domain(nats_client.clone(), domain) + } + }; info!("Connected to NATS server at {}", url); Ok(Self { diff --git a/crates/fuel-streams-nats/src/nats_client_opts.rs b/crates/fuel-streams-nats/src/nats_client_opts.rs new file mode 100644 index 00000000..f6aaa0a3 --- /dev/null +++ b/crates/fuel-streams-nats/src/nats_client_opts.rs @@ -0,0 +1,227 @@ +use std::time::Duration; + +use async_nats::ConnectOptions; + +use super::NatsNamespace; + +#[derive(Debug, Clone, Eq, PartialEq, Default)] +pub enum NatsAuth { + Admin, + System, + #[default] + Public, + Custom(String, String), +} + +impl NatsAuth { + fn credentials_from_env(&self) -> (String, String) { + match self { + NatsAuth::Admin => ( + dotenvy::var("NATS_ADMIN_USER") + .expect("NATS_ADMIN_USER must be set"), + dotenvy::var("NATS_ADMIN_PASS") + .expect("NATS_ADMIN_PASS must be set"), + ), + NatsAuth::System => ( + dotenvy::var("NATS_SYSTEM_USER") + .expect("NATS_SYSTEM_USER must be set"), + dotenvy::var("NATS_SYSTEM_PASS") + .expect("NATS_SYSTEM_PASS must be set"), + ), + NatsAuth::Public => ("default_user".to_string(), "".to_string()), + NatsAuth::Custom(user, pass) => { + (user.to_string(), pass.to_string()) + } + } + } +} + +/// Configuration options for connecting to NATS +/// +/// # Examples +/// +/// ```no_run +/// use fuel_streams_nats::*; +/// +/// // Create with URL +/// let opts = NatsClientOpts::new("nats://localhost:4222".to_string(), Some(NatsAuth::Admin)); +/// +/// // Create with admin credentials from environment +/// let opts = NatsClientOpts::admin_opts(); +/// +/// // Create with system credentials from environment +/// let opts = NatsClientOpts::system_opts(); +/// +/// // Create with public credentials +/// let opts = NatsClientOpts::public_opts(); +/// ``` +/// +/// Customize options: +/// +/// ```no_run +/// use fuel_streams_nats::*; +/// +/// let opts = NatsClientOpts::new("nats://localhost:4222".to_string(), Some(NatsAuth::Admin)) +/// .with_domain("mydomain") +/// .with_user("myuser") +/// .with_password("mypass") +/// .with_timeout(10); +/// ``` +#[derive(Debug, Clone)] +pub struct NatsClientOpts { + /// The URL of the NATS server. + pub(crate) url: String, + /// The namespace used as a prefix for NATS streams, consumers, and subject names. + pub(crate) namespace: NatsNamespace, + /// The timeout in seconds for NATS operations. + pub(crate) timeout_secs: u64, + /// The domain to use for the NATS client. + pub(crate) domain: Option, + /// The user to use for the NATS client. + pub(crate) user: Option, + /// The password to use for the NATS client. + pub(crate) password: Option, +} + +impl NatsClientOpts { + pub fn new(url: String, auth: Option) -> Self { + let (user, pass) = auth.unwrap_or_default().credentials_from_env(); + Self { + url, + namespace: NatsNamespace::default(), + timeout_secs: 5, + domain: None, + user: Some(user), + password: Some(pass), + } + } + + pub fn from_env(auth: Option) -> Self { + let url = dotenvy::var("NATS_URL").expect("NATS_URL must be set"); + Self::new(url, auth) + } + pub fn admin_opts() -> Self { + Self::from_env(Some(NatsAuth::Admin)) + } + pub fn system_opts() -> Self { + Self::from_env(Some(NatsAuth::System)) + } + pub fn public_opts() -> Self { + Self::from_env(Some(NatsAuth::Public)) + } + + pub fn get_url(&self) -> String { + self.url.clone() + } + + pub fn with_url>(self, url: S) -> Self { + Self { + url: url.into(), + ..self + } + } + + pub fn with_domain>(self, domain: S) -> Self { + Self { + domain: Some(domain.into()), + ..self + } + } + + pub fn with_user>(self, user: S) -> Self { + Self { + user: Some(user.into()), + ..self + } + } + + pub fn with_password>(self, password: S) -> Self { + Self { + password: Some(password.into()), + ..self + } + } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn with_rdn_namespace(self) -> Self { + let namespace = format!(r"namespace-{}", Self::random_int()); + self.with_namespace(&namespace) + } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn with_namespace(self, namespace: &str) -> Self { + let namespace = NatsNamespace::Custom(namespace.to_string()); + Self { namespace, ..self } + } + + pub fn with_timeout(self, secs: u64) -> Self { + Self { + timeout_secs: secs, + ..self + } + } + + pub(super) fn connect_opts(&self) -> ConnectOptions { + let opts = match (self.user.clone(), self.password.clone()) { + (Some(user), Some(pass)) => { + ConnectOptions::with_user_and_password(user, pass) + } + _ => ConnectOptions::new(), + }; + + opts.connection_timeout(Duration::from_secs(self.timeout_secs)) + .max_reconnects(1) + .name(Self::conn_id()) + } + + // This will be useful for debugging and monitoring connections + fn conn_id() -> String { + format!(r"connection-{}", Self::random_int()) + } + + fn random_int() -> u32 { + use rand::Rng; + rand::thread_rng().gen() + } +} + +#[cfg(test)] +mod tests { + use std::env; + + use super::*; + + #[test] + fn test_role_credentials() { + // Setup + env::set_var("NATS_ADMIN_USER", "admin"); + env::set_var("NATS_ADMIN_PASS", "admin_pass"); + + // Test Admin role credentials + let (user, pass) = NatsAuth::Admin.credentials_from_env(); + assert_eq!(user, "admin"); + assert_eq!(pass, "admin_pass"); + + // Cleanup + env::remove_var("NATS_ADMIN_USER"); + env::remove_var("NATS_ADMIN_PASS"); + } + + #[test] + fn test_from_env_with_role() { + // Setup + env::set_var("NATS_URL", "nats://localhost:4222"); + env::set_var("NATS_ADMIN_USER", "admin"); + env::set_var("NATS_ADMIN_PASS", "admin_pass"); + + // Test Admin role + let opts = NatsClientOpts::from_env(Some(NatsAuth::Admin)); + assert_eq!(opts.user, Some("admin".to_string())); + assert_eq!(opts.password, Some("admin_pass".to_string())); + + // Cleanup + env::remove_var("NATS_URL"); + env::remove_var("NATS_ADMIN_USER"); + env::remove_var("NATS_ADMIN_PASS"); + } +} diff --git a/crates/fuel-streams-storage/src/nats/nats_namespace.rs b/crates/fuel-streams-nats/src/nats_namespace.rs similarity index 92% rename from crates/fuel-streams-storage/src/nats/nats_namespace.rs rename to crates/fuel-streams-nats/src/nats_namespace.rs index c89f5fd2..947e8760 100644 --- a/crates/fuel-streams-storage/src/nats/nats_namespace.rs +++ b/crates/fuel-streams-nats/src/nats_namespace.rs @@ -7,7 +7,7 @@ static DEFAULT_NAMESPACE: &str = "fuel"; /// # Examples /// /// ``` -/// use fuel_streams_storage::nats::NatsNamespace; +/// use fuel_streams_nats::NatsNamespace; /// /// let default_namespace = NatsNamespace::default(); /// assert_eq!(default_namespace.to_string(), "fuel"); @@ -44,7 +44,7 @@ impl NatsNamespace { /// # Examples /// /// ``` - /// use fuel_streams_storage::nats::NatsNamespace; + /// use fuel_streams_nats::NatsNamespace; /// /// let namespace = NatsNamespace::default(); /// assert_eq!(namespace.subject_name("test"), "fuel.test"); @@ -61,7 +61,7 @@ impl NatsNamespace { /// # Examples /// /// ``` - /// use fuel_streams_storage::nats::NatsNamespace; + /// use fuel_streams_nats::NatsNamespace; /// /// let namespace = NatsNamespace::default(); /// assert_eq!(namespace.stream_name("test"), "fuel_test"); diff --git a/crates/fuel-streams-storage/src/nats/types.rs b/crates/fuel-streams-nats/src/types.rs similarity index 100% rename from crates/fuel-streams-storage/src/nats/types.rs rename to crates/fuel-streams-nats/src/types.rs diff --git a/crates/fuel-streams-publisher/Cargo.toml b/crates/fuel-streams-publisher/Cargo.toml deleted file mode 100644 index fb19b174..00000000 --- a/crates/fuel-streams-publisher/Cargo.toml +++ /dev/null @@ -1,74 +0,0 @@ -[package] -name = "fuel-streams-publisher" -description = "Fuel library for publishing data streams from events that happen in Fuel chain(s)" -authors = { workspace = true } -keywords = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -version = { workspace = true } -rust-version = { workspace = true } -publish = false - -[dependencies] -actix-cors = { workspace = true } -actix-server = { workspace = true } -actix-web = { workspace = true } -anyhow = { workspace = true } -async-nats = { workspace = true } -async-trait = { workspace = true } -chrono = { workspace = true } -clap = { workspace = true } -derive_more = { version = "1.0", features = ["full"] } -displaydoc = { workspace = true } -dotenvy = { workspace = true } -elasticsearch = "8.15.0-alpha.1" -fuel-core = { workspace = true } -fuel-core-bin = { workspace = true } -fuel-core-importer = { workspace = true } -fuel-core-services = { workspace = true } -fuel-core-storage = { workspace = true } -fuel-core-types = { workspace = true } -fuel-streams = { workspace = true, features = ["test-helpers"] } -fuel-streams-core = { workspace = true, features = ["test-helpers"] } -fuel-streams-storage = { workspace = true, features = ["test-helpers"] } -futures = { workspace = true } -num_cpus = "1.16" -parking_lot = { version = "0.12", features = ["serde"] } -prometheus = { version = "0.13", features = ["process"] } -rand = { workspace = true } -rayon = "1.10.0" -rust_decimal = { version = "1.13" } -serde = { workspace = true } -serde_json = { workspace = true } -serde_prometheus = { version = "0.2" } -sha2 = { workspace = true } -sysinfo = { version = "0.29" } -thiserror = "2.0" -tokio = { workspace = true } -tokio-stream = { workspace = true } -tracing = { workspace = true } -tracing-actix-web = { workspace = true } -url = "2.5" - -[dev-dependencies] -assert_matches = { workspace = true } -mockall = { workspace = true } -mockall_double = { workspace = true } - -[features] -default = [] -test-helpers = [] - -[target.x86_64-unknown-linux-gnu.dependencies] -openssl = { version = "0.10.68", features = ["vendored"] } - -[target.x86_64-unknown-linux-musl.dependencies] -openssl = { version = "0.10.68", features = ["vendored"] } - -[target.aarch64-unknown-linux-gnu.dependencies] -openssl = { version = "0.10.68", features = ["vendored"] } - -[target.aarch64-unknown-linux-musl.dependencies] -openssl = { version = "0.10.68", features = ["vendored"] } diff --git a/crates/fuel-streams-publisher/README.md b/crates/fuel-streams-publisher/README.md deleted file mode 100644 index 9d7eb196..00000000 --- a/crates/fuel-streams-publisher/README.md +++ /dev/null @@ -1,72 +0,0 @@ -
-
- - Logo - -

Fuel Streams Publisher

-

- A binary that subscribes to events from a Fuel client or node and publishes streams consumable via the fuel-streams SDK -

-

- - CI - - - Coverage - -

-

- 📚 Documentation -   - 🐛 Report Bug -   - ✨ Request Feature -

-
- -## 📝 About The Project - -The Fuel Streams Publisher is a binary that subscribes to events emitted from a Fuel client or node and publishes streams that can be consumed via the `fuel-streams` SDK. - -## ⚡️ Getting Started - -### Prerequisites - -- [Rust toolchain](https://www.rust-lang.org/tools/install) -- [Docker](https://www.docker.com/get-started/) (optional) - -### Development - -1. Generate the `KEYPAIR` environment variable: - - ```sh - fuel-core-keygen new --key-type peering -p - ``` - -2. Generate an `INFURA_API_KEY` from [Infura](https://app.infura.io/) - -3. Copy `.env.sample` to `.env` and update the `KEYPAIR` and `INFURA_API_KEY` with the values generated above - -4. Run the binary: - - - From the monorepo's root: - - ```sh - ./scripts/start-publisher.sh - ``` - - - Or using `make` and `docker`: - - ```sh - make start/publisher - ``` - -## 🤝 Contributing - -Contributions are welcome! Please feel free to submit a Pull Request. - -For more information on contributing, please see the [CONTRIBUTING.md](../../CONTRIBUTING.md) file in the root of the repository. - -## 📜 License - -This project is licensed under the `Apache-2.0` license. See [`LICENSE`](../../LICENSE) for more information. diff --git a/crates/fuel-streams-publisher/src/lib.rs b/crates/fuel-streams-publisher/src/lib.rs deleted file mode 100644 index dbbb3cc0..00000000 --- a/crates/fuel-streams-publisher/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub mod cli; -pub mod publisher; -pub mod server; -pub mod telemetry; - -use std::{env, sync::LazyLock}; - -pub use publisher::*; - -pub static PUBLISHER_MAX_THREADS: LazyLock = LazyLock::new(|| { - let available_cpus = num_cpus::get(); - let default_threads = (available_cpus / 3).max(1); // Use 1/3 of CPUs, minimum 1 - - env::var("PUBLISHER_MAX_THREADS") - .ok() - .and_then(|val| val.parse().ok()) - .unwrap_or(default_threads) -}); - -#[cfg(test)] -#[macro_use] -extern crate assert_matches; diff --git a/crates/fuel-streams-publisher/src/main.rs b/crates/fuel-streams-publisher/src/main.rs deleted file mode 100644 index cc667404..00000000 --- a/crates/fuel-streams-publisher/src/main.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::{ - net::{Ipv4Addr, SocketAddrV4}, - sync::Arc, -}; - -use clap::Parser; -use fuel_streams_publisher::{ - cli::Cli, - publisher::shutdown::ShutdownController, - server::{http::create_web_server, state::ServerState}, - shutdown, - telemetry::Telemetry, - FuelCore, - FuelCoreLike, -}; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); - let historical = cli.clone().historical; - - let fuel_core: Arc = - FuelCore::new(cli.fuel_core_config).await?; - fuel_core.start().await?; - - let telemetry = Telemetry::new().await?; - telemetry.start().await?; - - let publisher = fuel_streams_publisher::Publisher::new( - Arc::clone(&fuel_core), - telemetry.clone(), - ) - .await?; - - let state = ServerState::new(publisher.clone()).await; - // create the actix webserver - let server_addr = std::net::SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::UNSPECIFIED, - cli.telemetry_port, - )); - let server = create_web_server(state, server_addr)?; - // get server handle - let server_handle = server.handle(); - // spawn the server in the background - tokio::spawn(async move { - if let Err(err) = server.await { - tracing::error!("Actix Web server error: {:?}", err); - } - }); - tracing::info!("Publisher started."); - - let (shutdown_controller, shutdown_token) = - shutdown::get_controller_and_token(); - ShutdownController::spawn_signal_listener(&shutdown_controller); - - // run publisher until shutdown signal intercepted - if let Err(err) = publisher.run(shutdown_token, historical).await { - tracing::error!("Publisher encountered an error: {:?}", err); - } - tracing::info!("Publisher stopped"); - - // Await the Actix server shutdown - tracing::info!("Stopping actix server ..."); - server_handle.stop(true).await; - tracing::info!("Actix server stopped. Goodbye!"); - - Ok(()) -} diff --git a/crates/fuel-streams-publisher/src/publisher/blocks_streams.rs b/crates/fuel-streams-publisher/src/publisher/blocks_streams.rs deleted file mode 100644 index a40c0e00..00000000 --- a/crates/fuel-streams-publisher/src/publisher/blocks_streams.rs +++ /dev/null @@ -1,394 +0,0 @@ -use std::{cmp::max, sync::Arc}; - -use fuel_streams_core::prelude::*; -use futures::{ - stream::{self, BoxStream}, - StreamExt, - TryStreamExt, -}; -use tokio_stream::wrappers::BroadcastStream; - -use crate::{fuel_core_like::FuelCoreLike, fuel_streams::FuelStreamsExt}; - -pub fn build_blocks_stream<'a>( - fuel_streams: &'a Arc, - fuel_core: &'a Arc, - max_retained_blocks: u64, -) -> BoxStream<'a, anyhow::Result> { - #[derive(Debug, Default, Clone)] - struct State { - has_published_latest: bool, - has_reached_new_blocks_stream: bool, - } - let stream_state = State::default(); - - stream::try_unfold(stream_state, move |mut stream_state| { - let fuel_core = Arc::clone(fuel_core); - let fuel_streams = Arc::clone(fuel_streams); - - async move { - let latest_block_height = fuel_core.get_latest_block_height()?; - - let last_published_block_height = get_last_published_block_height( - fuel_streams, - latest_block_height, - max_retained_blocks, - ) - .await?; - - stream_state.has_published_latest = - latest_block_height == last_published_block_height; - - match stream_state { - State { - has_published_latest: false, - has_reached_new_blocks_stream: false, - } => { - let old_blocks_stream = stream::iter( - last_published_block_height..latest_block_height, - ) - .map({ - let fuel_core = fuel_core.clone(); - - move |height| { - fuel_core.get_sealed_block_by_height(height as u32) - } - }) - .map(Ok) - .boxed(); - - anyhow::Ok(Some((old_blocks_stream, stream_state.clone()))) - } - State { - has_published_latest: true, - has_reached_new_blocks_stream: false, - } => { - let new_blocks_stream = - BroadcastStream::new(fuel_core.blocks_subscription()) - .map(|import_result| { - import_result - .expect("Must get ImporterResult") - .sealed_block - .clone() - }) - .map(Ok) - .boxed(); - - stream_state.has_reached_new_blocks_stream = true; - anyhow::Ok(Some((new_blocks_stream, stream_state.clone()))) - } - State { - has_reached_new_blocks_stream: true, - .. - } => anyhow::Ok(None), - } - } - }) - .try_flatten() - .boxed() -} - -async fn get_last_published_block_height( - fuel_streams: Arc, - latest_block_height: u64, - max_retained_blocks: u64, -) -> anyhow::Result { - let max_last_published_block_height = - max(0, latest_block_height as i64 - max_retained_blocks as i64) as u64; - - Ok(fuel_streams - .get_last_published_block() - .await? - .map(|block| block.height.into()) - .map(|block_height: u64| { - max(block_height, max_last_published_block_height) - }) - .unwrap_or(max_last_published_block_height)) -} - -#[cfg(test)] -mod tests { - use std::{sync::Arc, time::Duration}; - - // TODO: Fix this leaky abstraction - use async_nats::{ - jetstream::stream::State as StreamState, - RequestErrorKind, - }; - use fuel_core::combined_database::CombinedDatabase; - use futures::StreamExt; - use mockall::{ - mock, - predicate::{self, *}, - }; - use tokio::{ - sync::broadcast, - time::{error::Elapsed, timeout}, - }; - - use super::*; - - #[tokio::test] - async fn test_no_old_blocks() { - let mut mock_fuel_core = MockFuelCoreLike::new(); - let mut mock_fuel_streams = MockFuelStreams::default(); - - mock_fuel_core - .expect_get_latest_block_height() - .returning(|| Ok(100)); - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Ok(Some(create_mock_block(100)))); // No old blocks - - mock_fuel_core - .expect_blocks_subscription() - .returning(move || { - let (empty_tx, rx) = broadcast::channel(1); - drop(empty_tx); - rx - }); - - let fuel_core: Arc = Arc::new(mock_fuel_core); - let fuel_streams: Arc = Arc::new(mock_fuel_streams); - - let mut stream = build_blocks_stream(&fuel_streams, &fuel_core, 10); - - assert!(stream.next().await.is_none()); - } - - #[tokio::test] - async fn test_old_blocks_stream() { - let mut mock_fuel_core = MockFuelCoreLike::new(); - let mut mock_fuel_streams = MockFuelStreams::default(); - - mock_fuel_core - .expect_get_latest_block_height() - .returning(|| Ok(105)); - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Ok(Some(create_mock_block(100)))); - for height in 100..105 { - mock_fuel_core - .expect_get_sealed_block_by_height() - .with(predicate::eq(height as u32)) - .returning(move |height| { - create_mock_fuel_core_sealed_block(height as u64) - }); - } - - let fuel_core: Arc = Arc::new(mock_fuel_core); - let fuel_streams: Arc = Arc::new(mock_fuel_streams); - - let mut stream = build_blocks_stream(&fuel_streams, &fuel_core, 10); - - for height in 100..105 { - let block = stream.next().await.unwrap().unwrap(); - assert_eq!(block.entity.header().consensus().height, height.into()); - } - } - - #[tokio::test] - async fn test_infinite_new_blocks_streams() { - let mut mock_fuel_core = MockFuelCoreLike::new(); - let mut mock_fuel_streams = MockFuelStreams::default(); - - mock_fuel_core - .expect_get_latest_block_height() - .returning(|| Ok(100)); - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Ok(Some(create_mock_block(100)))); // has published latest block already - - let (tx, _) = broadcast::channel(4); - - mock_fuel_core - .expect_blocks_subscription() - .returning(move || tx.clone().subscribe()); - - let fuel_core: Arc = Arc::new(mock_fuel_core); - let fuel_streams: Arc = Arc::new(mock_fuel_streams); - - let mut blocks_stream = - build_blocks_stream(&fuel_streams, &fuel_core, 10); - - assert_matches!( - timeout(Duration::from_secs(1), async { - blocks_stream.next().await - }) - .await, - Err(Elapsed { .. }) - ); - } - - #[tokio::test] - async fn test_new_blocks_streams_that_ends() { - let mut mock_fuel_core = MockFuelCoreLike::new(); - let mut mock_fuel_streams = MockFuelStreams::default(); - - mock_fuel_core - .expect_get_latest_block_height() - .returning(|| Ok(100)); - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Ok(Some(create_mock_block(100)))); // has published latest block already - - let (tx, _) = broadcast::channel(4); - - mock_fuel_core - .expect_blocks_subscription() - .returning(move || { - let tx = tx.clone(); - let subscription = tx.subscribe(); - - tx.send(create_mock_importer_result(101)).ok(); - tx.send(create_mock_importer_result(102)).ok(); - - subscription - }); - - let fuel_core: Arc = Arc::new(mock_fuel_core); - let fuel_streams: Arc = Arc::new(mock_fuel_streams); - - let mut stream = build_blocks_stream(&fuel_streams, &fuel_core, 10); - - for height in 101..=102 { - let block = stream.next().await.unwrap().unwrap(); - assert_eq!(block.entity.header().consensus().height, height.into()); - } - } - - #[tokio::test] - async fn test_get_last_published_block_height() { - let mut mock_fuel_streams = MockFuelStreams::default(); - - // Case 1: `get_last_published_block` returns Some(block) - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Ok(Some(create_mock_block(50)))); - - let fuel_streams = Arc::new(mock_fuel_streams); - - let result = - get_last_published_block_height(fuel_streams.clone(), 100, 40) - .await - .unwrap(); - assert_eq!(result, 60); // max(50, max_last_published_block_height=60) - - // Case 2: `get_last_published_block` returns None - let mut mock_fuel_streams = MockFuelStreams::default(); - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Ok(None)); - - let fuel_streams = Arc::new(mock_fuel_streams); - - let result = - get_last_published_block_height(fuel_streams.clone(), 100, 40) - .await - .unwrap(); - assert_eq!(result, 60); // No block, fallback to max_last_published_block_height - - // Case 3: `get_last_published_block` returns an error - let mut mock_fuel_streams = MockFuelStreams::default(); - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Err(anyhow::anyhow!("Error fetching block"))); - - let fuel_streams = Arc::new(mock_fuel_streams); - - let result = - get_last_published_block_height(fuel_streams.clone(), 100, 40) - .await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().to_string(), "Error fetching block"); - - // Case 4: `get_last_published_block` returns Some(block) where block.height < max_last_published_block_height - let mut mock_fuel_streams = MockFuelStreams::default(); - mock_fuel_streams - .expect_get_last_published_block() - .returning(|| Ok(Some(create_mock_block(30)))); - - let fuel_streams = Arc::new(mock_fuel_streams); - - let result = - get_last_published_block_height(fuel_streams.clone(), 100, 40) - .await - .unwrap(); - assert_eq!(result, 60); // max(30, max_last_published_block_height=60) - } - - mock! { - FuelCoreLike {} - - #[async_trait::async_trait] - impl FuelCoreLike for FuelCoreLike { - fn get_latest_block_height(&self) -> anyhow::Result; - fn get_sealed_block_by_height(&self, height: u32) -> FuelCoreSealedBlock; - fn blocks_subscription(&self) -> broadcast::Receiver; - async fn start(&self) -> anyhow::Result<()>; - fn is_started(&self) -> bool; - async fn await_synced_at_least_once(&self, historical: bool) -> anyhow::Result<()>; - async fn stop(&self); - fn base_asset_id(&self) -> &FuelCoreAssetId; - fn chain_id(&self) -> &FuelCoreChainId; - fn database(&self) -> &CombinedDatabase; - async fn await_offchain_db_sync( - &self, - block_id: &FuelCoreBlockId, - ) -> anyhow::Result<()>; - fn get_receipts( - &self, - tx_id: &FuelCoreBytes32, - ) -> anyhow::Result>>; - } - } - - mock! { - FuelStreams {} - - #[async_trait::async_trait] - impl FuelStreamsExt for FuelStreams { - async fn get_last_published_block(&self) -> anyhow::Result>; - fn blocks(&self) -> &Stream; - fn transactions(&self) -> &Stream; - fn inputs(&self) -> &Stream; - fn outputs(&self) -> &Stream; - fn receipts(&self) -> &Stream; - fn utxos(&self) -> &Stream; - fn logs(&self) -> &Stream; - async fn get_consumers_and_state( - &self, - ) -> Result, StreamState)>, RequestErrorKind> ; - #[cfg(feature = "test-helpers")] - async fn is_empty(&self) -> bool; - } - } - - fn create_mock_importer_result(height: u64) -> FuelCoreImporterResult { - FuelCoreImporterResult { - shared_result: Arc::new(FuelCoreImportResult { - sealed_block: create_mock_fuel_core_sealed_block(height), - ..Default::default() - }), - #[cfg(feature = "test-helpers")] - changes: Arc::new(std::collections::HashMap::new()), - } - } - - fn create_mock_block(height: u64) -> Block { - Block::new( - &create_mock_fuel_core_sealed_block(height).entity, - FuelCoreConsensus::default().into(), - vec![], - ) - } - - fn create_mock_fuel_core_sealed_block(height: u64) -> FuelCoreSealedBlock { - let mut block = FuelCoreSealedBlock::default(); - - block.entity.header_mut().consensus_mut().height = - FuelCoreBlockHeight::new(height as u32); - - block - } -} diff --git a/crates/fuel-streams-publisher/src/publisher/fuel_streams.rs b/crates/fuel-streams-publisher/src/publisher/fuel_streams.rs deleted file mode 100644 index 05f9fd0f..00000000 --- a/crates/fuel-streams-publisher/src/publisher/fuel_streams.rs +++ /dev/null @@ -1,137 +0,0 @@ -use std::sync::Arc; - -use async_nats::{jetstream::stream::State as StreamState, RequestErrorKind}; -use fuel_streams::types::Log; -use fuel_streams_core::prelude::*; - -#[derive(Clone, Debug)] -/// Streams we currently support publishing to. -pub struct FuelStreams { - pub transactions: Stream, - pub blocks: Stream, - pub inputs: Stream, - pub outputs: Stream, - pub receipts: Stream, - pub utxos: Stream, - pub logs: Stream, -} - -impl FuelStreams { - pub async fn new( - nats_client: &NatsClient, - s3_client: &Arc, - ) -> Self { - Self { - transactions: Stream::::new(nats_client, s3_client) - .await, - blocks: Stream::::new(nats_client, s3_client).await, - inputs: Stream::::new(nats_client, s3_client).await, - outputs: Stream::::new(nats_client, s3_client).await, - receipts: Stream::::new(nats_client, s3_client).await, - utxos: Stream::::new(nats_client, s3_client).await, - logs: Stream::::new(nats_client, s3_client).await, - } - } -} - -#[async_trait::async_trait] -pub trait FuelStreamsExt: Sync + Send { - fn blocks(&self) -> &Stream; - fn transactions(&self) -> &Stream; - fn inputs(&self) -> &Stream; - fn outputs(&self) -> &Stream; - fn receipts(&self) -> &Stream; - fn utxos(&self) -> &Stream; - fn logs(&self) -> &Stream; - - async fn get_last_published_block(&self) -> anyhow::Result>; - - fn subjects_wildcards(&self) -> &[&'static str] { - &[ - TransactionsSubject::WILDCARD, - BlocksSubject::WILDCARD, - InputsByIdSubject::WILDCARD, - InputsCoinSubject::WILDCARD, - InputsMessageSubject::WILDCARD, - InputsContractSubject::WILDCARD, - ReceiptsLogSubject::WILDCARD, - ReceiptsBurnSubject::WILDCARD, - ReceiptsByIdSubject::WILDCARD, - ReceiptsCallSubject::WILDCARD, - ReceiptsMintSubject::WILDCARD, - ReceiptsPanicSubject::WILDCARD, - ReceiptsReturnSubject::WILDCARD, - ReceiptsRevertSubject::WILDCARD, - ReceiptsLogDataSubject::WILDCARD, - ReceiptsTransferSubject::WILDCARD, - ReceiptsMessageOutSubject::WILDCARD, - ReceiptsReturnDataSubject::WILDCARD, - ReceiptsTransferOutSubject::WILDCARD, - ReceiptsScriptResultSubject::WILDCARD, - UtxosSubject::WILDCARD, - LogsSubject::WILDCARD, - ] - } - - async fn get_consumers_and_state( - &self, - ) -> Result, StreamState)>, RequestErrorKind>; - - #[cfg(feature = "test-helpers")] - async fn is_empty(&self) -> bool; -} - -#[async_trait::async_trait] -impl FuelStreamsExt for FuelStreams { - fn blocks(&self) -> &Stream { - &self.blocks - } - fn transactions(&self) -> &Stream { - &self.transactions - } - fn inputs(&self) -> &Stream { - &self.inputs - } - fn outputs(&self) -> &Stream { - &self.outputs - } - fn receipts(&self) -> &Stream { - &self.receipts - } - fn utxos(&self) -> &Stream { - &self.utxos - } - fn logs(&self) -> &Stream { - &self.logs - } - - async fn get_last_published_block(&self) -> anyhow::Result> { - Ok(self - .blocks - .get_last_published(BlocksSubject::WILDCARD) - .await?) - } - - async fn get_consumers_and_state( - &self, - ) -> Result, StreamState)>, RequestErrorKind> { - Ok(vec![ - self.transactions.get_consumers_and_state().await?, - self.blocks.get_consumers_and_state().await?, - self.inputs.get_consumers_and_state().await?, - self.outputs.get_consumers_and_state().await?, - self.receipts.get_consumers_and_state().await?, - self.utxos.get_consumers_and_state().await?, - self.logs.get_consumers_and_state().await?, - ]) - } - - #[cfg(feature = "test-helpers")] - async fn is_empty(&self) -> bool { - self.blocks.is_empty(BlocksSubject::WILDCARD).await - && self - .transactions - .is_empty(TransactionsSubject::WILDCARD) - .await - } -} diff --git a/crates/fuel-streams-publisher/src/publisher/mod.rs b/crates/fuel-streams-publisher/src/publisher/mod.rs deleted file mode 100644 index 2873857c..00000000 --- a/crates/fuel-streams-publisher/src/publisher/mod.rs +++ /dev/null @@ -1,300 +0,0 @@ -pub mod fuel_core_like; -pub mod fuel_streams; -pub mod payloads; -pub mod shutdown; - -mod blocks_streams; - -use std::sync::Arc; - -use anyhow::Context; -use blocks_streams::build_blocks_stream; -pub use fuel_core_like::{FuelCore, FuelCoreLike}; -pub use fuel_streams::{FuelStreams, FuelStreamsExt}; -use fuel_streams_core::prelude::*; -use fuel_streams_storage::S3Client; -use futures::{future::try_join_all, stream::FuturesUnordered, StreamExt}; -use tokio::sync::Semaphore; - -use super::{ - payloads::blocks, - shutdown::{ShutdownToken, GRACEFUL_SHUTDOWN_TIMEOUT}, - telemetry::Telemetry, - PUBLISHER_MAX_THREADS, -}; - -#[derive(Clone)] -pub struct Publisher { - pub fuel_core: Arc, - pub nats_client: NatsClient, - pub fuel_streams: Arc, - pub telemetry: Arc, - pub s3_client: Arc, -} - -impl Publisher { - pub async fn new( - fuel_core: Arc, - telemetry: Arc, - ) -> anyhow::Result { - let nats_client_opts = NatsClientOpts::admin_opts(); - let nats_client = NatsClient::connect(&nats_client_opts).await?; - - let s3_client_opts = S3ClientOpts::admin_opts(); - let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); - if let Err(e) = s3_client.create_bucket().await { - tracing::error!("Failed to create S3 bucket: {:?}", e); - } - - let fuel_streams = - Arc::new(FuelStreams::new(&nats_client, &s3_client).await); - - telemetry.record_streams_count( - fuel_core.chain_id(), - fuel_streams.subjects_wildcards().len(), - ); - - Ok(Publisher { - fuel_core, - fuel_streams, - nats_client, - telemetry, - s3_client, - }) - } - - pub fn is_healthy(&self) -> bool { - // TODO: Update this condition to include more health checks - self.fuel_core.is_started() && self.nats_client.is_connected() - } - - #[cfg(feature = "test-helpers")] - pub async fn new_for_testing( - nats_client: &NatsClient, - s3_client: &Arc, - fuel_core: Arc, - ) -> anyhow::Result { - Ok(Publisher { - fuel_core, - fuel_streams: Arc::new( - FuelStreams::new(nats_client, s3_client).await, - ), - nats_client: nats_client.clone(), - telemetry: Telemetry::new().await?, - s3_client: Arc::clone(s3_client), - }) - } - - #[cfg(feature = "test-helpers")] - pub fn get_fuel_streams(&self) -> &Arc<(dyn FuelStreamsExt + 'static)> { - &self.fuel_streams - } - - async fn shutdown_services_with_timeout(&self) -> anyhow::Result<()> { - tokio::time::timeout(GRACEFUL_SHUTDOWN_TIMEOUT, async { - Publisher::flush_await_all_streams(&self.nats_client).await; - self.fuel_core.stop().await; - }) - .await?; - - Ok(()) - } - - async fn flush_await_all_streams(nats_client: &NatsClient) { - tracing::info!("Flushing in-flight messages to nats ..."); - match nats_client.nats_client.flush().await { - Ok(_) => { - tracing::info!("Flushed all streams successfully!"); - } - Err(e) => { - tracing::error!("Failed to flush all streams: {:?}", e); - } - } - } - - const MAX_RETAINED_BLOCKS: u64 = 100; - pub async fn run( - &self, - mut shutdown_token: ShutdownToken, - historical: bool, - ) -> anyhow::Result<()> { - tracing::info!("Awaiting FuelCore Sync..."); - - self.fuel_core - .await_synced_at_least_once(historical) - .await?; - - tracing::info!("FuelCore has synced successfully!"); - - tracing::info!("Publishing started..."); - - let mut blocks_stream = build_blocks_stream( - &self.fuel_streams, - &self.fuel_core, - Self::MAX_RETAINED_BLOCKS, - ); - - loop { - tokio::select! { - Some(sealed_block) = blocks_stream.next() => { - let sealed_block = sealed_block.context("block streams failed to produce sealed block")?; - - tracing::info!("Processing blocks stream"); - - let fuel_core = &self.fuel_core; - let (block, block_producer) = - fuel_core.get_block_and_producer(sealed_block); - - // TODO: Avoid awaiting Offchain DB sync for all streams by grouping in their own service - fuel_core - .await_offchain_db_sync(&block.id()) - .await - .context("Failed to await Offchain DB sync")?; - - if let Err(err) = self.publish(&block, &block_producer).await { - tracing::error!("Failed to publish block data: {}", err); - self.telemetry.record_failed_publishing(self.fuel_core.chain_id(), &block_producer); - } - - }, - shutdown = shutdown_token.wait_for_shutdown() => { - if shutdown { - tracing::info!("Shutdown signal received. Stopping services ..."); - self.shutdown_services_with_timeout().await?; - break; - } - }, - }; - } - - tracing::info!("Publishing stopped successfully!"); - - Ok(()) - } - - async fn publish( - &self, - block: &FuelCoreBlock, - block_producer: &Address, - ) -> anyhow::Result<()> { - let start_time = std::time::Instant::now(); - - let semaphore = Arc::new(Semaphore::new(*PUBLISHER_MAX_THREADS)); - let chain_id = Arc::new(*self.fuel_core.chain_id()); - let base_asset_id = Arc::new(*self.fuel_core.base_asset_id()); - let block_producer = Arc::new(block_producer.clone()); - let block_height = block.header().consensus().height; - let txs = block.transactions(); - let transaction_ids = txs - .iter() - .map(|tx| tx.id(&chain_id).into()) - .collect::>(); - - let consensus: Consensus = - self.fuel_core.get_consensus(&block_height)?.into(); - - let fuel_core = &*self.fuel_core; - let offchain_database = fuel_core.offchain_database()?; - - let fuel_streams = &*self.fuel_streams; - let blocks_stream = Arc::new(fuel_streams.blocks().to_owned()); - - let opts = &Arc::new(PublishOpts { - semaphore, - chain_id, - base_asset_id, - block_producer: Arc::clone(&block_producer), - block_height: Arc::new(block_height.into()), - telemetry: self.telemetry.clone(), - consensus: Arc::new(consensus), - offchain_database, - }); - - let publish_tasks = payloads::transactions::publish_all_tasks( - txs, - fuel_streams, - opts, - fuel_core, - )? - .into_iter() - .chain(std::iter::once(blocks::publish_task( - block, - blocks_stream, - opts, - transaction_ids, - ))) - .collect::>(); - - try_join_all(publish_tasks).await?; - - let elapsed = start_time.elapsed(); - tracing::info!( - "Published streams for BlockHeight: {} in {:?}", - *block_height, - elapsed - ); - - Ok(()) - } -} - -use tokio::task::JoinHandle; - -use crate::fuel_core_like::OffchainDatabase; - -#[derive(Clone)] -pub struct PublishOpts { - pub semaphore: Arc, - pub chain_id: Arc, - pub base_asset_id: Arc, - pub block_producer: Arc
, - pub block_height: Arc, - pub telemetry: Arc, - pub consensus: Arc, - pub offchain_database: Arc, -} - -pub fn publish( - packet: &PublishPacket, - stream: Arc>, - opts: &Arc, -) -> JoinHandle> { - let opts = Arc::clone(opts); - let packet = packet.clone(); - let telemetry = Arc::clone(&opts.telemetry); - let wildcard = packet.subject.wildcard(); - - tokio::spawn(async move { - let _permit = opts.semaphore.acquire().await?; - - // Publish to NATS - match stream.publish(&packet).await { - Ok(published_data_size) => { - telemetry.log_info(&format!( - "Successfully published for stream: {}", - wildcard - )); - telemetry.update_publisher_success_metrics( - wildcard, - published_data_size, - &opts.chain_id, - &opts.block_producer, - ); - - Ok(()) - } - Err(e) => { - tracing::error!("Failed to publish: {:?}", e); - telemetry.log_error(&e.to_string()); - telemetry.update_publisher_error_metrics( - wildcard, - &opts.chain_id, - &opts.block_producer, - &e.to_string(), - ); - - anyhow::bail!("Failed to publish: {}", e.to_string()) - } - } - }) -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/blocks.rs b/crates/fuel-streams-publisher/src/publisher/payloads/blocks.rs deleted file mode 100644 index 5e59f728..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/blocks.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; -use tokio::task::JoinHandle; - -use crate::{publish, PublishOpts}; - -pub fn publish_task( - block: &FuelCoreBlock, - stream: Arc>, - opts: &Arc, - transaction_ids: Vec, -) -> JoinHandle> { - let block_height = (*opts.block_height).clone(); - let block_producer = (*opts.block_producer).clone(); - let consensus = (*opts.consensus).clone(); - - let block = Block::new(block, consensus, transaction_ids); - let packet = PublishPacket::new( - block, - BlocksSubject { - height: Some(block_height), - producer: Some(block_producer), - } - .arc(), - ); - - publish(&packet, stream, opts) -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/inputs.rs b/crates/fuel-streams-publisher/src/publisher/payloads/inputs.rs deleted file mode 100644 index 88048d54..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/inputs.rs +++ /dev/null @@ -1,178 +0,0 @@ -use std::sync::Arc; - -use fuel_core_types::fuel_tx::input::{ - coin::{CoinPredicate, CoinSigned}, - message::{ - MessageCoinPredicate, - MessageCoinSigned, - MessageDataPredicate, - MessageDataSigned, - }, -}; -use fuel_streams_core::prelude::*; -use rayon::prelude::*; -use tokio::task::JoinHandle; - -use crate::{publish, PublishOpts}; - -pub fn publish_tasks( - tx: &FuelCoreTransaction, - tx_id: &Bytes32, - stream: &Stream, - opts: &Arc, -) -> Vec>> { - let packets = tx - .inputs() - .par_iter() - .enumerate() - .flat_map(move |(index, input)| { - let main_subject = main_subject(input, tx_id.clone(), index); - let identifier_subjects = identifiers(input, tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: InputsByIdSubject| subject.arc()) - .collect::>(); - - let input: Input = input.into(); - - let mut packets = vec![input.to_packet(main_subject)]; - packets.extend( - identifier_subjects - .into_iter() - .map(|subject| input.to_packet(subject)), - ); - - packets - }) - .collect::>(); - - packets - .iter() - .map(|packet| publish(packet, Arc::new(stream.to_owned()), opts)) - .collect() -} - -fn main_subject( - input: &FuelCoreInput, - tx_id: Bytes32, - index: usize, -) -> Arc { - match input { - FuelCoreInput::Contract(contract) => { - let contract_id = contract.contract_id; - - InputsContractSubject { - tx_id: Some(tx_id), - index: Some(index), - contract_id: Some(contract_id.into()), - } - .arc() - } - FuelCoreInput::CoinSigned(CoinSigned { - owner, asset_id, .. - }) - | FuelCoreInput::CoinPredicate(CoinPredicate { - owner, asset_id, .. - }) => InputsCoinSubject { - tx_id: Some(tx_id), - index: Some(index), - owner: Some(owner.into()), - asset_id: Some(asset_id.into()), - } - .arc(), - FuelCoreInput::MessageCoinSigned(MessageCoinSigned { - sender, - recipient, - .. - }) - | FuelCoreInput::MessageCoinPredicate(MessageCoinPredicate { - sender, - recipient, - .. - }) - | FuelCoreInput::MessageDataSigned(MessageDataSigned { - sender, - recipient, - .. - }) - | FuelCoreInput::MessageDataPredicate(MessageDataPredicate { - sender, - recipient, - .. - }) => InputsMessageSubject { - tx_id: Some(tx_id), - index: Some(index), - sender: Some(sender.into()), - recipient: Some(recipient.into()), - } - .arc(), - } -} - -pub fn identifiers( - input: &FuelCoreInput, - tx_id: &Bytes32, - index: u8, -) -> Vec { - let mut identifiers = match input { - FuelCoreInput::CoinSigned(CoinSigned { - owner, asset_id, .. - }) => { - vec![ - Identifier::Address(tx_id.to_owned(), index, owner.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - FuelCoreInput::CoinPredicate(CoinPredicate { - owner, asset_id, .. - }) => { - vec![ - Identifier::Address(tx_id.to_owned(), index, owner.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - FuelCoreInput::MessageCoinSigned(MessageCoinSigned { - sender, - recipient, - .. - }) - | FuelCoreInput::MessageCoinPredicate(MessageCoinPredicate { - sender, - recipient, - .. - }) - | FuelCoreInput::MessageDataSigned(MessageDataSigned { - sender, - recipient, - .. - }) - | FuelCoreInput::MessageDataPredicate(MessageDataPredicate { - sender, - recipient, - .. - }) => { - vec![ - Identifier::Address(tx_id.to_owned(), index, sender.into()), - Identifier::Address(tx_id.to_owned(), index, recipient.into()), - ] - } - FuelCoreInput::Contract(contract) => { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract.contract_id.into(), - )] - } - }; - - if let Some((predicate_bytecode, _, _)) = input.predicate() { - let predicate_tag = super::sha256(predicate_bytecode); - identifiers.push(Identifier::PredicateID( - tx_id.to_owned(), - index, - predicate_tag, - )); - } - - identifiers -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/logs.rs b/crates/fuel-streams-publisher/src/publisher/payloads/logs.rs deleted file mode 100644 index d443c1fc..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/logs.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; -use rayon::prelude::*; -use tokio::task::JoinHandle; - -use crate::{publish, PublishOpts}; - -pub fn publish_tasks( - tx_id: &Bytes32, - stream: &Stream, - opts: &Arc, - receipts: &Vec, -) -> Vec>> { - let block_height = (*opts.block_height).clone(); - let packets = receipts - .par_iter() - .enumerate() - .filter_map(|(index, receipt)| match receipt { - FuelCoreReceipt::Log { id, .. } - | FuelCoreReceipt::LogData { id, .. } => Some(PublishPacket::new( - receipt.to_owned().into(), - LogsSubject { - block_height: Some(block_height.clone()), - tx_id: Some(tx_id.to_owned()), - receipt_index: Some(index), - log_id: Some((*id).into()), - } - .arc(), - )), - _ => None, - }) - .collect::>(); - - packets - .iter() - .map(|packet| publish(packet, Arc::new(stream.to_owned()), opts)) - .collect() -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/mod.rs b/crates/fuel-streams-publisher/src/publisher/payloads/mod.rs deleted file mode 100644 index ac0ce74a..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub mod blocks; -pub mod inputs; -pub mod logs; -pub mod outputs; -pub mod receipts; -pub mod transactions; -pub mod utxos; - -use fuel_streams_core::prelude::Bytes32; -use sha2::{Digest, Sha256}; - -pub fn sha256(bytes: &[u8]) -> Bytes32 { - let mut sha256 = Sha256::new(); - sha256.update(bytes); - let bytes: [u8; 32] = sha256 - .finalize() - .as_slice() - .try_into() - .expect("Must be 32 bytes"); - - bytes.into() -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/outputs.rs b/crates/fuel-streams-publisher/src/publisher/payloads/outputs.rs deleted file mode 100644 index 0a61a974..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/outputs.rs +++ /dev/null @@ -1,161 +0,0 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; -use rayon::prelude::*; -use tokio::task::JoinHandle; - -use crate::{publish, PublishOpts}; - -pub fn publish_tasks( - tx: &FuelCoreTransaction, - tx_id: &Bytes32, - stream: &Stream, - opts: &Arc, -) -> Vec>> { - let packets: Vec> = tx - .outputs() - .par_iter() - .enumerate() - .flat_map(|(index, output)| { - let main_subject = main_subject(output, tx, tx_id, index); - let identifier_subjects = - identifiers(output, tx, tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: OutputsByIdSubject| subject.arc()) - .collect::>(); - - let output: Output = output.into(); - - let mut packets = vec![output.to_packet(main_subject)]; - packets.extend( - identifier_subjects - .into_iter() - .map(|subject| output.to_packet(subject)), - ); - - packets - }) - .collect(); - - packets - .iter() - .map(|packet| publish(packet, Arc::new(stream.to_owned()), opts)) - .collect() -} - -fn main_subject( - output: &FuelCoreOutput, - transaction: &FuelCoreTransaction, - tx_id: &Bytes32, - index: usize, -) -> Arc { - match output { - FuelCoreOutput::Coin { to, asset_id, .. } => OutputsCoinSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - to: Some((*to).into()), - asset_id: Some((*asset_id).into()), - } - .arc(), - - FuelCoreOutput::Contract(contract) => { - let contract_id = - match find_output_contract_id(transaction, contract) { - Some(contract_id) => contract_id, - None => { - tracing::warn!( - "Contract ID not found for output: {:?}", - output - ); - - Default::default() - } - }; - - OutputsContractSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - contract_id: Some(contract_id.into()), - } - .arc() - } - - FuelCoreOutput::Change { to, asset_id, .. } => OutputsChangeSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - to: Some((*to).into()), - asset_id: Some((*asset_id).into()), - } - .arc(), - - FuelCoreOutput::Variable { to, asset_id, .. } => { - OutputsVariableSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - to: Some((*to).into()), - asset_id: Some((*asset_id).into()), - } - .arc() - } - - FuelCoreOutput::ContractCreated { contract_id, .. } => { - OutputsContractCreatedSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index as u16), - contract_id: Some((*contract_id).into()), - } - .arc() - } - } -} - -pub fn identifiers( - output: &FuelCoreOutput, - tx: &FuelCoreTransaction, - tx_id: &Bytes32, - index: u8, -) -> Vec { - match output { - FuelCoreOutput::Change { to, asset_id, .. } - | FuelCoreOutput::Variable { to, asset_id, .. } - | FuelCoreOutput::Coin { to, asset_id, .. } => { - vec![ - Identifier::Address(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - FuelCoreOutput::Contract(contract) => { - find_output_contract_id(tx, contract) - .map(|contract_id| { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract_id.into(), - )] - }) - .unwrap_or_default() - } - FuelCoreOutput::ContractCreated { contract_id, .. } => { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract_id.into(), - )] - } - } -} - -pub fn find_output_contract_id( - tx: &FuelCoreTransaction, - contract: &FuelCoreOutputContract, -) -> Option { - let input_index = contract.input_index as usize; - tx.inputs().get(input_index).and_then(|input| { - if let FuelCoreInput::Contract(input_contract) = input { - Some(input_contract.contract_id) - } else { - None - } - }) -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/receipts.rs b/crates/fuel-streams-publisher/src/publisher/payloads/receipts.rs deleted file mode 100644 index e0936f5f..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/receipts.rs +++ /dev/null @@ -1,235 +0,0 @@ -use std::sync::Arc; - -use fuel_streams_core::prelude::*; -use rayon::prelude::*; -use tokio::task::JoinHandle; - -use crate::{publish, PublishOpts}; - -pub fn publish_tasks( - tx_id: &Bytes32, - stream: &Stream, - opts: &Arc, - receipts: &Vec, -) -> Vec>> { - let packets: Vec> = receipts - .par_iter() - .enumerate() - .flat_map(|(index, receipt)| { - let main_subject = main_subject(receipt, tx_id, index); - let identifier_subjects = identifiers(receipt, tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: ReceiptsByIdSubject| subject.arc()) - .collect::>(); - - let receipt: Receipt = receipt.into(); - - let mut packets = vec![receipt.to_packet(main_subject)]; - packets.extend( - identifier_subjects - .into_iter() - .map(|subject| receipt.to_packet(subject)), - ); - - packets - }) - .collect(); - - packets - .iter() - .map(|packet| publish(packet, Arc::new(stream.to_owned()), opts)) - .collect() -} - -fn main_subject( - receipt: &FuelCoreReceipt, - tx_id: &Bytes32, - index: usize, -) -> Arc { - match receipt { - FuelCoreReceipt::Call { - id: from, - to, - asset_id, - .. - } => ReceiptsCallSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - from: Some(from.into()), - to: Some(to.into()), - asset_id: Some(asset_id.into()), - } - .arc(), - FuelCoreReceipt::Return { id, .. } => ReceiptsReturnSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.into()), - } - .arc(), - FuelCoreReceipt::ReturnData { id, .. } => ReceiptsReturnDataSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.into()), - } - .arc(), - FuelCoreReceipt::Panic { id, .. } => ReceiptsPanicSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.into()), - } - .arc(), - FuelCoreReceipt::Revert { id, .. } => ReceiptsRevertSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.into()), - } - .arc(), - FuelCoreReceipt::Log { id, .. } => ReceiptsLogSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.into()), - } - .arc(), - FuelCoreReceipt::LogData { id, .. } => ReceiptsLogDataSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - id: Some(id.into()), - } - .arc(), - FuelCoreReceipt::Transfer { - id: from, - to, - asset_id, - .. - } => ReceiptsTransferSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - from: Some(from.into()), - to: Some(to.into()), - asset_id: Some(asset_id.into()), - } - .arc(), - - FuelCoreReceipt::TransferOut { - id: from, - to, - asset_id, - .. - } => ReceiptsTransferOutSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - from: Some(from.into()), - to: Some(to.into()), - asset_id: Some(asset_id.into()), - } - .arc(), - - FuelCoreReceipt::ScriptResult { .. } => ReceiptsScriptResultSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - } - .arc(), - FuelCoreReceipt::MessageOut { - sender, recipient, .. - } => ReceiptsMessageOutSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - sender: Some(sender.into()), - recipient: Some(recipient.into()), - } - .arc(), - FuelCoreReceipt::Mint { - contract_id, - sub_id, - .. - } => ReceiptsMintSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - contract_id: Some(contract_id.into()), - sub_id: Some((*sub_id).into()), - } - .arc(), - FuelCoreReceipt::Burn { - contract_id, - sub_id, - .. - } => ReceiptsBurnSubject { - tx_id: Some(tx_id.to_owned()), - index: Some(index), - contract_id: Some(contract_id.into()), - sub_id: Some((*sub_id).into()), - } - .arc(), - } -} - -pub fn identifiers( - receipt: &FuelCoreReceipt, - tx_id: &Bytes32, - index: u8, -) -> Vec { - match receipt { - FuelCoreReceipt::Call { - id: from, - to, - asset_id, - .. - } => { - vec![ - Identifier::ContractID(tx_id.to_owned(), index, from.into()), - Identifier::ContractID(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - FuelCoreReceipt::Return { id, .. } - | FuelCoreReceipt::ReturnData { id, .. } - | FuelCoreReceipt::Panic { id, .. } - | FuelCoreReceipt::Revert { id, .. } - | FuelCoreReceipt::Log { id, .. } - | FuelCoreReceipt::LogData { id, .. } => { - vec![Identifier::ContractID(tx_id.to_owned(), index, id.into())] - } - FuelCoreReceipt::Transfer { - id: from, - to, - asset_id, - .. - } => { - vec![ - Identifier::ContractID(tx_id.to_owned(), index, from.into()), - Identifier::ContractID(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - FuelCoreReceipt::TransferOut { - id: from, - to, - asset_id, - .. - } => { - vec![ - Identifier::ContractID(tx_id.to_owned(), index, from.into()), - Identifier::ContractID(tx_id.to_owned(), index, to.into()), - Identifier::AssetID(tx_id.to_owned(), index, asset_id.into()), - ] - } - FuelCoreReceipt::MessageOut { - sender, recipient, .. - } => { - vec![ - Identifier::Address(tx_id.to_owned(), index, sender.into()), - Identifier::Address(tx_id.to_owned(), index, recipient.into()), - ] - } - FuelCoreReceipt::Mint { contract_id, .. } - | FuelCoreReceipt::Burn { contract_id, .. } => { - vec![Identifier::ContractID( - tx_id.to_owned(), - index, - contract_id.into(), - )] - } - _ => Vec::new(), - } -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/transactions.rs b/crates/fuel-streams-publisher/src/publisher/payloads/transactions.rs deleted file mode 100644 index fc97edc6..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/transactions.rs +++ /dev/null @@ -1,177 +0,0 @@ -use std::sync::Arc; - -use fuel_core_types::fuel_tx::field::ScriptData; -use fuel_streams_core::prelude::*; -use rayon::prelude::*; -use tokio::task::JoinHandle; - -use super::{ - inputs::{self, publish_tasks as publish_inputs}, - logs::publish_tasks as publish_logs, - outputs::{self, publish_tasks as publish_outputs}, - receipts::{self, publish_tasks as publish_receipts}, - sha256, - utxos::publish_tasks as publish_utxos, -}; -use crate::{publish, FuelCoreLike, FuelStreamsExt, PublishOpts}; - -pub fn publish_all_tasks( - transactions: &[FuelCoreTransaction], - fuel_streams: &dyn FuelStreamsExt, - opts: &Arc, - fuel_core: &dyn FuelCoreLike, -) -> anyhow::Result>>> { - let offchain_database = Arc::clone(&opts.offchain_database); - let mut tasks = vec![]; - - for tx_item @ (_, tx) in transactions.iter().enumerate() { - let tx_id = tx.id(&opts.chain_id); - let tx_status: TransactionStatus = offchain_database - .get_tx_status(&tx_id)? - .map(|status| (&status).into()) - .unwrap_or_default(); - - let receipts = fuel_core.get_receipts(&tx_id)?.unwrap_or_default(); - - let tx_id = tx_id.into(); - - tasks.extend(publish_tasks( - tx_item, - &tx_id, - &tx_status, - fuel_streams.transactions(), - opts, - &receipts, - )); - tasks.extend(publish_inputs(tx, &tx_id, fuel_streams.inputs(), opts)); - tasks.extend(publish_outputs(tx, &tx_id, fuel_streams.outputs(), opts)); - tasks.extend(publish_receipts( - &tx_id, - fuel_streams.receipts(), - opts, - &receipts, - )); - tasks.extend(publish_outputs(tx, &tx_id, fuel_streams.outputs(), opts)); - tasks.extend(publish_logs( - &tx_id, - fuel_streams.logs(), - opts, - &receipts, - )); - tasks.extend(publish_utxos(tx, &tx_id, fuel_streams.utxos(), opts)); - } - - Ok(tasks) -} - -fn publish_tasks( - tx_item: (usize, &FuelCoreTransaction), - tx_id: &Bytes32, - tx_status: &TransactionStatus, - stream: &Stream, - opts: &Arc, - receipts: &Vec, -) -> Vec>> { - let block_height = &opts.block_height; - let base_asset_id = &opts.base_asset_id; - - packets_from_tx( - tx_item, - tx_id, - tx_status, - base_asset_id, - block_height, - receipts, - ) - .iter() - .map(|packet| publish(packet, Arc::new(stream.to_owned()), opts)) - .collect() -} - -fn packets_from_tx( - (index, tx): (usize, &FuelCoreTransaction), - tx_id: &Bytes32, - tx_status: &TransactionStatus, - base_asset_id: &FuelCoreAssetId, - block_height: &BlockHeight, - receipts: &Vec, -) -> Vec> { - let main_subject = TransactionsSubject { - block_height: Some(block_height.to_owned()), - index: Some(index), - tx_id: Some(tx_id.to_owned()), - status: Some(tx_status.to_owned()), - kind: Some(tx.into()), - } - .arc(); - - let transaction = - Transaction::new(tx_id, tx, tx_status, base_asset_id, receipts); - let mut packets = vec![transaction.to_packet(main_subject)]; - - packets.extend( - identifiers(tx, tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: TransactionsByIdSubject| subject.arc()) - .map(|subject| transaction.to_packet(subject)) - .collect::>(), - ); - - let packets_from_inputs: Vec> = tx - .inputs() - .par_iter() - .flat_map(|input| { - inputs::identifiers(input, tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: TransactionsByIdSubject| subject.arc()) - .map(|subject| transaction.to_packet(subject)) - }) - .collect(); - - packets.extend(packets_from_inputs); - - let packets_from_outputs: Vec> = tx - .outputs() - .par_iter() - .flat_map(|output| { - outputs::identifiers(output, tx, tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: TransactionsByIdSubject| subject.arc()) - .map(|subject| transaction.to_packet(subject)) - }) - .collect(); - - packets.extend(packets_from_outputs); - - let packets_from_receipts: Vec> = receipts - .par_iter() - .flat_map(|receipt| { - receipts::identifiers(receipt, tx_id, index as u8) - .into_par_iter() - .map(|identifier| identifier.into()) - .map(|subject: TransactionsByIdSubject| subject.arc()) - .map(|subject| transaction.to_packet(subject)) - }) - .collect(); - - packets.extend(packets_from_receipts); - - packets -} - -fn identifiers( - tx: &FuelCoreTransaction, - tx_id: &Bytes32, - index: u8, -) -> Vec { - match tx { - FuelCoreTransaction::Script(tx) => { - let script_tag = sha256(tx.script_data()); - vec![Identifier::ScriptID(tx_id.to_owned(), index, script_tag)] - } - _ => Vec::new(), - } -} diff --git a/crates/fuel-streams-publisher/src/publisher/payloads/utxos.rs b/crates/fuel-streams-publisher/src/publisher/payloads/utxos.rs deleted file mode 100644 index 0e4d1f89..00000000 --- a/crates/fuel-streams-publisher/src/publisher/payloads/utxos.rs +++ /dev/null @@ -1,145 +0,0 @@ -use std::sync::Arc; - -use fuel_core_types::fuel_tx::{ - input::{ - coin::{CoinPredicate, CoinSigned}, - contract::Contract, - message::{ - compute_message_id, - MessageCoinPredicate, - MessageCoinSigned, - MessageDataPredicate, - MessageDataSigned, - }, - }, - UtxoId, -}; -use fuel_streams_core::prelude::*; -use rayon::prelude::*; -use tokio::task::JoinHandle; - -use crate::{publish, PublishOpts}; - -pub fn publish_tasks( - tx: &FuelCoreTransaction, - tx_id: &Bytes32, - stream: &Stream, - opts: &Arc, -) -> Vec>> { - let packets = tx - .inputs() - .par_iter() - .filter_map(|input| utxo_packet(input, tx_id, input.utxo_id().cloned())) - .collect::>(); - - packets - .into_iter() - .map(|packet| publish(&packet, Arc::new(stream.to_owned()), opts)) - .collect() -} - -fn utxo_packet( - input: &FuelCoreInput, - tx_id: &Bytes32, - utxo_id: Option, -) -> Option> { - utxo_id?; - let utxo_id = utxo_id.expect("safe to unwrap utxo"); - - match input { - FuelCoreInput::Contract(Contract { utxo_id, .. }) => { - let utxo = Utxo { - utxo_id: utxo_id.into(), - tx_id: tx_id.to_owned(), - ..Default::default() - }; - - let subject = UtxosSubject { - utxo_type: Some(UtxoType::Contract), - hash: Some(tx_id.to_owned().into()), - } - .arc(); - - Some(utxo.to_packet(subject)) - } - FuelCoreInput::CoinSigned(CoinSigned { - utxo_id, amount, .. - }) - | FuelCoreInput::CoinPredicate(CoinPredicate { - utxo_id, amount, .. - }) => { - let utxo = Utxo { - utxo_id: utxo_id.into(), - amount: Some(*amount), - tx_id: tx_id.to_owned(), - ..Default::default() - }; - - let subject = UtxosSubject { - utxo_type: Some(UtxoType::Coin), - hash: Some(tx_id.to_owned().into()), - } - .arc(); - - Some(utxo.to_packet(subject)) - } - message @ (FuelCoreInput::MessageCoinSigned(MessageCoinSigned { - amount, - nonce, - recipient, - sender, - .. - }) - | FuelCoreInput::MessageCoinPredicate( - MessageCoinPredicate { - amount, - nonce, - recipient, - sender, - .. - }, - ) - | FuelCoreInput::MessageDataSigned(MessageDataSigned { - amount, - nonce, - recipient, - sender, - .. - }) - | FuelCoreInput::MessageDataPredicate( - MessageDataPredicate { - amount, - nonce, - recipient, - sender, - .. - }, - )) => { - let (data, hash) = if let Some(data) = message.input_data() { - let hash: MessageId = - compute_message_id(sender, recipient, nonce, *amount, data) - .into(); - (Some(data.to_vec()), hash) - } else { - (None, tx_id.to_owned().into()) - }; - - let utxo = Utxo { - utxo_id: utxo_id.into(), - sender: Some(sender.into()), - recipient: Some(recipient.into()), - nonce: Some(nonce.into()), - amount: Some(*amount), - tx_id: tx_id.to_owned(), - data, - }; - let subject = UtxosSubject { - utxo_type: Some(UtxoType::Message), - hash: Some(hash), - } - .arc(); - - Some(utxo.to_packet(subject)) - } - } -} diff --git a/crates/fuel-streams-publisher/src/publisher/shutdown.rs b/crates/fuel-streams-publisher/src/publisher/shutdown.rs deleted file mode 100644 index 89963fff..00000000 --- a/crates/fuel-streams-publisher/src/publisher/shutdown.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::time::Duration; - -use tokio::{ - signal::unix::{signal, SignalKind}, - sync::{broadcast, OnceCell}, -}; - -// TODO: move into publisher module along with subjects - -pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(90); - -// First, let's create a ShutdownToken that can be shared -#[derive(Debug)] -pub struct ShutdownToken { - receiver: broadcast::Receiver<()>, -} - -impl ShutdownToken { - pub async fn wait_for_shutdown(&mut self) -> bool { - self.receiver.recv().await.is_ok() - } -} - -#[derive(Debug, Clone)] -pub struct ShutdownController { - sender: broadcast::Sender<()>, - shutdown_initiated: OnceCell<()>, -} - -impl ShutdownController { - pub fn spawn_signal_listener(&self) { - let sender = self.sender.clone(); - tokio::spawn(async move { - let mut sigint = - signal(SignalKind::interrupt()).expect("shutdown_listener"); - let mut sigterm = - signal(SignalKind::terminate()).expect("shutdown_listener"); - - tokio::select! { - _ = sigint.recv() => { - tracing::info!("Received SIGINT ..."); - let _ = sender.send(()); - } - _ = sigterm.recv() => { - tracing::info!("Received SIGTERM ..."); - let _ = sender.send(()); - } - } - }); - } - - pub fn initiate_shutdown( - &self, - ) -> Result> { - if self.shutdown_initiated.set(()).is_ok() { - self.sender.send(()) - } else { - Ok(0) // Shutdown already initiated - } - } -} - -pub fn get_controller_and_token() -> (ShutdownController, ShutdownToken) { - let (sender, receiver) = broadcast::channel(1); - - ( - ShutdownController { - sender, - shutdown_initiated: OnceCell::new(), - }, - ShutdownToken { receiver }, - ) -} diff --git a/crates/fuel-streams-publisher/src/server/http.rs b/crates/fuel-streams-publisher/src/server/http.rs deleted file mode 100644 index 50933e3d..00000000 --- a/crates/fuel-streams-publisher/src/server/http.rs +++ /dev/null @@ -1,117 +0,0 @@ -use std::net::SocketAddr; - -use actix_cors::Cors; -use actix_server::Server; -use actix_web::{http, web, App, HttpResponse, HttpServer}; -use tracing_actix_web::TracingLogger; - -use super::state::ServerState; - -// We are keeping this low to give room for more -// Publishing processing power. This is fine since the -// the latency tolerance when fetching /health and /metrics -// is trivial -const MAX_WORKERS: usize = 2; - -pub fn create_web_server( - state: ServerState, - actix_server_addr: SocketAddr, -) -> anyhow::Result { - let server = HttpServer::new(move || { - // create cors - let cors = Cors::default() - .allow_any_origin() - .allowed_methods(vec!["GET", "POST"]) - .allowed_headers(vec![ - http::header::AUTHORIZATION, - http::header::ACCEPT, - ]) - .allowed_header(http::header::CONTENT_TYPE) - .max_age(3600); - - App::new() - .app_data(web::Data::new(state.clone())) - .wrap(TracingLogger::default()) - .wrap(cors) - .service(web::resource("/health").route(web::get().to( - |state: web::Data| async move { - if !state.is_healthy() { - return HttpResponse::ServiceUnavailable() - .body("Service Unavailable"); - } - HttpResponse::Ok().json(state.get_health().await) - }, - ))) - .service(web::resource("/metrics").route(web::get().to( - |state: web::Data| async move { - HttpResponse::Ok() - .body(state.publisher.telemetry.get_metrics().await) - }, - ))) - }) - .bind(actix_server_addr)? - .workers(MAX_WORKERS) - .shutdown_timeout(20) - .run(); - - Ok(server) -} - -#[cfg(test)] -#[cfg(feature = "test-helpers")] -mod tests { - use std::time::Duration; - - use actix_web::{http, test, web, App, HttpResponse}; - use fuel_core::service::Config; - use fuel_core_bin::FuelService; - use fuel_core_services::State; - - use crate::{ - server::state::{HealthResponse, ServerState}, - telemetry::Telemetry, - FuelCore, - Publisher, - }; - - #[actix_web::test] - async fn test_health_check() { - let fuel_service = - FuelService::new_node(Config::local_node()).await.unwrap(); - assert_eq!(fuel_service.state(), State::Started); - - let telemetry = Telemetry::new().await.unwrap(); - - let fuel_core = FuelCore::from(fuel_service); - let publisher = - Publisher::new(fuel_core.arc(), telemetry).await.unwrap(); - let state = ServerState::new(publisher).await; - assert!(state.publisher.nats_client.is_connected()); - - let app = test::init_service( - App::new().app_data(web::Data::new(state.clone())).route( - "/health", - web::get().to(|state: web::Data| async move { - if !state.is_healthy() { - return HttpResponse::ServiceUnavailable() - .body("Service Unavailable"); - } - HttpResponse::Ok().json(state.get_health().await) - }), - ), - ) - .await; - - let uptime = Duration::from_secs(2); - tokio::time::sleep(uptime).await; - - let req = test::TestRequest::get().uri("/health").to_request(); - let resp = test::call_service(&app, req).await; - - assert_eq!(resp.status(), http::StatusCode::OK); - - let result: HealthResponse = test::read_body_json(resp).await; - assert!(result.uptime >= uptime.as_secs()); - assert!(!result.streams_info.is_empty()); - } -} diff --git a/crates/fuel-streams-publisher/src/server/mod.rs b/crates/fuel-streams-publisher/src/server/mod.rs deleted file mode 100644 index 8ebfa9ca..00000000 --- a/crates/fuel-streams-publisher/src/server/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod http; -pub mod state; diff --git a/crates/fuel-streams-publisher/src/server/state.rs b/crates/fuel-streams-publisher/src/server/state.rs deleted file mode 100644 index c15660b9..00000000 --- a/crates/fuel-streams-publisher/src/server/state.rs +++ /dev/null @@ -1,106 +0,0 @@ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; - -use async_nats::jetstream::stream::State; -use parking_lot::RwLock; -use serde::{Deserialize, Serialize}; - -use crate::Publisher; - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct StreamInfo { - consumers: Vec, - state: StreamState, - stream_name: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] -pub struct StreamState { - /// The number of messages contained in this stream - pub messages: u64, - /// The number of bytes of all messages contained in this stream - pub bytes: u64, - /// The lowest sequence number still present in this stream - #[serde(rename = "first_seq")] - pub first_sequence: u64, - /// The time associated with the oldest message still present in this stream - #[serde(rename = "first_ts")] - pub first_timestamp: i64, - /// The last sequence number assigned to a message in this stream - #[serde(rename = "last_seq")] - pub last_sequence: u64, - /// The time that the last message was received by this stream - #[serde(rename = "last_ts")] - pub last_timestamp: i64, - /// The number of consumers configured to consume this stream - pub consumer_count: usize, -} - -impl From for StreamState { - fn from(state: State) -> Self { - StreamState { - messages: state.messages, - bytes: state.bytes, - first_sequence: state.first_sequence, - first_timestamp: state.first_timestamp.unix_timestamp(), - last_sequence: state.last_sequence, - last_timestamp: state.last_timestamp.unix_timestamp(), - consumer_count: state.consumer_count, - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct HealthResponse { - pub uptime: u64, - pub streams_info: Vec, -} - -#[derive(Clone)] -pub struct ServerState { - pub publisher: Publisher, - pub start_time: Instant, - pub connection_count: Arc>, -} - -impl ServerState { - pub async fn new(publisher: Publisher) -> Self { - Self { - publisher, - start_time: Instant::now(), - connection_count: Arc::new(RwLock::new(0)), - } - } -} - -impl ServerState { - pub fn is_healthy(&self) -> bool { - self.publisher.is_healthy() - } - - pub async fn get_health(&self) -> HealthResponse { - let streams_info = self - .publisher - .fuel_streams - .get_consumers_and_state() - .await - .unwrap_or_default() - .into_iter() - .map(|res| StreamInfo { - consumers: res.1, - state: res.2.into(), - stream_name: res.0, - }) - .collect::>(); - HealthResponse { - uptime: self.uptime().as_secs(), - streams_info, - } - } - - pub fn uptime(&self) -> Duration { - self.start_time.elapsed() - } -} diff --git a/crates/fuel-streams-publisher/src/telemetry/mod.rs b/crates/fuel-streams-publisher/src/telemetry/mod.rs deleted file mode 100644 index 17f26842..00000000 --- a/crates/fuel-streams-publisher/src/telemetry/mod.rs +++ /dev/null @@ -1,277 +0,0 @@ -mod elastic_search; -mod publisher; -mod runtime; -#[allow(clippy::needless_borrows_for_generic_args)] -mod system; - -use std::{sync::Arc, time::Duration}; - -use anyhow::Context; -use elastic_search::{ - new_elastic_search, - should_use_elasticsearch, - ElasticSearch, - LogEntry, -}; -use fuel_streams_core::prelude::*; -// TODO: Consider using tokio's Rwlock instead -use parking_lot::RwLock; -use publisher::PublisherMetrics; -use runtime::Runtime; -use system::{System, SystemMetricsWrapper}; - -#[derive(Clone)] -pub struct Telemetry { - runtime: Arc, - system: Arc>, - publisher_metrics: Option>, - elastic_search: Option>, -} - -impl Telemetry { - const DEDICATED_THREADS: usize = 2; - - pub async fn new() -> anyhow::Result> { - let runtime = - Runtime::new(Self::DEDICATED_THREADS, Duration::from_secs(20)); - let system = Arc::new(RwLock::new(System::new().await)); - - let publisher_metrics = if should_use_publisher_metrics() { - Some(Arc::new(PublisherMetrics::default())) - } else { - None - }; - - let elastic_search = if should_use_elasticsearch() { - Some(Arc::new(new_elastic_search().await?)) - } else { - None - }; - - Ok(Arc::new(Self { - runtime: Arc::new(runtime), - system, - publisher_metrics, - elastic_search, - })) - } - - pub async fn start(&self) -> anyhow::Result<()> { - let system = Arc::clone(&self.system); - - if let Some(elastic_search) = self.elastic_search.as_ref() { - tracing::info!( - "Elastic Search connection live? {:?}", - elastic_search.get_conn().check_alive().unwrap_or_default() - ); - elastic_search - .get_conn() - .ping() - .await - .context("Error pinging elastisearch connection")?; - tracing::info!("Elastic logger pinged successfully!"); - }; - - self.runtime.start(move || { - system.write().refresh(); - }); - - Ok(()) - } - - pub fn log_info(&self, message: &str) { - let entry = LogEntry::new("INFO", message); - self.maybe_elog(entry); - tracing::info!("{}", message); - } - - pub fn log_error(&self, message: &str) { - let entry = LogEntry::new("ERROR", message); - self.maybe_elog(entry); - tracing::error!("{}", message); - } - - fn maybe_elog(&self, entry: LogEntry) { - if let Some(elastic_search) = &self.elastic_search { - self.runtime - .spawn(elastic_search::log(elastic_search.clone(), entry)); - } - } - - pub fn update_publisher_success_metrics( - &self, - subject: &str, - published_data_size: usize, - chain_id: &FuelCoreChainId, - block_producer: &Address, - ) { - self.maybe_use_metrics(|metrics| { - // Update message size histogram - metrics - .message_size_histogram - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - subject, - ]) - .observe(published_data_size as f64); - - // Increment total published messages - metrics - .total_published_messages - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - ]) - .inc(); - - // Increment throughput for the published messages - metrics - .published_messages_throughput - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - subject, - ]) - .inc(); - }); - } - - pub fn update_publisher_error_metrics( - &self, - subject: &str, - chain_id: &FuelCoreChainId, - block_producer: &Address, - error: &str, - ) { - self.maybe_use_metrics(|metrics| { - metrics - .error_rates - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - subject, - error, - ]) - .inc(); - }); - } - - pub fn record_streams_count( - &self, - chain_id: &FuelCoreChainId, - count: usize, - ) { - self.maybe_use_metrics(|metrics| { - metrics - .total_subs - .with_label_values(&[&chain_id.to_string()]) - .set(count as i64); - }); - } - - pub fn record_failed_publishing( - &self, - chain_id: &FuelCoreChainId, - block_producer: &Address, - ) { - self.maybe_use_metrics(|metrics| { - metrics - .total_failed_messages - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - ]) - .inc(); - }); - } - - pub fn maybe_use_metrics(&self, f: F) - where - F: Fn(&PublisherMetrics), - { - if let Some(metrics) = &self.publisher_metrics { - f(metrics); - } - } - - // TODO: Break into smaller functions - pub async fn get_metrics(&self) -> String { - use prometheus::Encoder; - let encoder = prometheus::TextEncoder::new(); - - if self.publisher_metrics.is_none() { - return "".to_string(); - } - - // fetch all measured metrics - let mut buffer = Vec::new(); - if let Err(e) = encoder.encode( - &self.publisher_metrics.as_ref().unwrap().registry.gather(), - &mut buffer, - ) { - tracing::error!("could not encode custom metrics: {}", e); - }; - let mut res = match String::from_utf8(buffer.clone()) { - Ok(v) => v, - Err(e) => { - tracing::error!( - "custom metrics could not be from_utf8'd: {}", - e - ); - String::default() - } - }; - buffer.clear(); - - let mut buffer = Vec::new(); - if let Err(e) = encoder.encode(&prometheus::gather(), &mut buffer) { - tracing::error!("could not encode prometheus metrics: {}", e); - }; - let res_custom = match String::from_utf8(buffer.clone()) { - Ok(v) => v, - Err(e) => { - tracing::error!( - "prometheus metrics could not be from_utf8'd: {}", - e - ); - String::default() - } - }; - buffer.clear(); - - res.push_str(&res_custom); - - // now fetch and add system metrics - let system_metrics = match self.system.read().metrics() { - Ok(m) => { - let metrics = SystemMetricsWrapper::from(m); - let labels: Vec<(&str, &str)> = vec![]; - match serde_prometheus::to_string(&metrics, None, labels) { - Ok(m) => m, - Err(err) => { - tracing::error!( - "could not encode system metrics: {:?}", - err - ); - String::default() - } - } - } - Err(err) => { - tracing::error!( - "prometheus system metrics could not be stringified: {:?}", - err - ); - String::default() - } - }; - res.push_str(&system_metrics); - - res - } -} - -pub fn should_use_publisher_metrics() -> bool { - dotenvy::var("USE_METRICS").is_ok_and(|val| val == "true") -} diff --git a/crates/fuel-streams-publisher/src/telemetry/publisher.rs b/crates/fuel-streams-publisher/src/telemetry/publisher.rs deleted file mode 100644 index 338d46b5..00000000 --- a/crates/fuel-streams-publisher/src/telemetry/publisher.rs +++ /dev/null @@ -1,423 +0,0 @@ -use std::sync::Arc; - -use chrono::Utc; -use fuel_core::database::database_description::DatabaseHeight; -use fuel_streams_core::prelude::*; -use prometheus::{ - register_histogram_vec, - register_int_counter_vec, - register_int_gauge_vec, - HistogramVec, - IntCounterVec, - IntGaugeVec, - Registry, -}; - -#[derive(Clone, Debug)] -pub struct PublisherMetrics { - pub registry: Registry, - pub total_subs: IntGaugeVec, - pub total_published_messages: IntCounterVec, - pub total_failed_messages: IntCounterVec, - pub last_published_block_height: IntGaugeVec, - pub last_published_block_timestamp: IntGaugeVec, - pub published_messages_throughput: IntCounterVec, - pub publishing_latency_histogram: HistogramVec, - pub message_size_histogram: HistogramVec, - pub error_rates: IntCounterVec, -} - -impl Default for PublisherMetrics { - fn default() -> Self { - PublisherMetrics::new(None) - .expect("Failed to create default PublisherMetrics") - } -} - -impl PublisherMetrics { - pub fn new(prefix: Option) -> anyhow::Result { - let metric_prefix = prefix - .clone() - .map(|p| format!("{}_", p)) - .unwrap_or_default(); - - let total_subs = register_int_gauge_vec!( - format!("{}publisher_metrics_total_subscriptions", metric_prefix), - "A metric counting the number of active subscriptions", - &["chain_id"], - ) - .expect("metric must be created"); - - let total_published_messages = register_int_counter_vec!( - format!( - "{}publisher_metrics_total_published_messages", - metric_prefix - ), - "A metric counting the number of published messages", - &["chain_id", "block_producer"], - ) - .expect("metric must be created"); - - let total_failed_messages = register_int_counter_vec!( - format!("{}publisher_metrics_total_failed_messages", metric_prefix), - "A metric counting the number of unpublished and failed messages", - &["chain_id", "block_producer"], - ) - .expect("metric must be created"); - - let last_published_block_height = register_int_gauge_vec!( - format!( - "{}publisher_metrics_last_published_block_height", - metric_prefix - ), - "A metric that represents the last published block height", - &["chain_id", "block_producer"], - ) - .expect("metric must be created"); - - let last_published_block_timestamp = register_int_gauge_vec!( - format!( - "{}publisher_metrics_last_published_block_timestamp", - metric_prefix - ), - "A metric that represents the last published transaction timestamp", - &["chain_id", "block_producer"], - ) - .expect("metric must be created"); - - let published_messages_throughput = register_int_counter_vec!( - format!("{}publisher_metrics_published_messages_throughput", metric_prefix), - "A metric counting the number of published messages per subject wildcard", - &["chain_id", "block_producer", "subject_wildcard"], - ) - .expect("metric must be created"); - - // New histogram metric for block latency - let publishing_latency_histogram = register_histogram_vec!( - format!("{}publisher_metrics_block_latency_seconds", metric_prefix), - "Histogram of latencies between receiving and publishing a block", - &["chain_id", "block_producer", "subject_wildcard"], - // buckets for latency measurement (e.g., 0.1s, 0.5s, 1s, 5s, 10s) - vec![0.1, 0.5, 1.0, 5.0, 10.0], - ) - .expect("metric must be created"); - - let message_size_histogram = register_histogram_vec!( - format!("{}publisher_metrics_message_size_bytes", metric_prefix), - "Histogram of message sizes in bytes", - &["chain_id", "block_producer", "subject_wildcard"], - vec![100.0, 500.0, 1000.0, 5000.0, 10000.0, 100000.0, 1000000.0] - ) - .expect("metric must be created"); - - let error_rates = - register_int_counter_vec!( - format!("{}publisher_metrics_error_rates", metric_prefix), - "A metric counting errors or failures during message processing", - &["chain_id", "block_producer", "subject_wildcard", "error_type"], - ) - .expect("metric must be created"); - - let registry = - Registry::new_custom(prefix, None).expect("registry to be created"); - registry.register(Box::new(total_subs.clone()))?; - registry.register(Box::new(total_published_messages.clone()))?; - registry.register(Box::new(total_failed_messages.clone()))?; - registry.register(Box::new(last_published_block_height.clone()))?; - registry.register(Box::new(last_published_block_timestamp.clone()))?; - registry.register(Box::new(published_messages_throughput.clone()))?; - registry.register(Box::new(publishing_latency_histogram.clone()))?; - registry.register(Box::new(message_size_histogram.clone()))?; - registry.register(Box::new(error_rates.clone()))?; - - Ok(Self { - registry, - total_subs, - total_published_messages, - total_failed_messages, - last_published_block_height, - last_published_block_timestamp, - published_messages_throughput, - publishing_latency_histogram, - message_size_histogram, - error_rates, - }) - } -} - -#[allow(dead_code)] -// TODO: Will this be useful in the future? -pub fn add_block_metrics( - chain_id: &FuelCoreChainId, - block: &FuelCoreBlock, - block_producer: &Address, - metrics: &Arc, -) -> anyhow::Result> { - let latency = Utc::now().timestamp() - block.header().time().to_unix(); - - metrics - .publishing_latency_histogram - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - BlocksSubject::WILDCARD, - ]) - .observe(latency as f64); - - metrics - .last_published_block_timestamp - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - ]) - .set(block.header().time().to_unix()); - - metrics - .last_published_block_height - .with_label_values(&[ - &chain_id.to_string(), - &block_producer.to_string(), - ]) - .set(block.header().consensus().height.as_u64() as i64); - - Ok(metrics.to_owned()) -} - -#[cfg(test)] -mod tests { - use prometheus::{gather, Encoder, TextEncoder}; - - use super::*; - - impl PublisherMetrics { - pub fn random() -> Self { - use rand::{distributions::Alphanumeric, Rng}; - - let prefix = rand::thread_rng() - .sample_iter(&Alphanumeric) - .filter(|c| c.is_ascii_alphabetic()) - .take(6) - .map(char::from) - .collect(); - - PublisherMetrics::new(Some(prefix)) - .expect("Failed to create random PublisherMetrics") - } - } - - #[test] - fn test_total_published_messages_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .total_published_messages - .with_label_values(&["chain_id_1", "block_producer_1"]) - .inc_by(5); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!(output.contains("publisher_metrics_total_published_messages")); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("5")); - } - - #[test] - fn test_latency_histogram_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .publishing_latency_histogram - .with_label_values(&["chain_id_1", "block_producer_1", "topic_1"]) - .observe(0.75); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!(output.contains("publisher_metrics_block_latency_seconds")); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("topic_1")); - assert!(output.contains("0.75")); - } - - #[test] - fn test_message_size_histogram_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .message_size_histogram - .with_label_values(&["chain_id_1", "block_producer_1", "topic_1"]) - .observe(1500.1); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!(output.contains("publisher_metrics_message_size_bytes")); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("topic_1")); - assert!(output.contains("1500.1")); - } - - #[test] - fn test_total_failed_messages_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .total_failed_messages - .with_label_values(&["chain_id_1", "block_producer_1"]) - .inc_by(3); - - // Gather all the metrics - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - // Convert the gathered output to a string - let output = String::from_utf8(buffer.clone()).unwrap(); - - // Assert that the output contains the correct failed message metric - assert!(output.contains("publisher_metrics_total_failed_messages")); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("3")); - } - - #[test] - fn test_total_subs_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .total_subs - .with_label_values(&["chain_id_1"]) - .set(10); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!(output.contains("publisher_metrics_total_subscriptions")); - assert!(output.contains("chain_id_1")); - assert!(output.contains("10")); - } - - #[test] - fn test_last_published_block_height_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .last_published_block_height - .with_label_values(&["chain_id_1", "block_producer_1"]) - .set(1234); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!( - output.contains("publisher_metrics_last_published_block_height") - ); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("1234")); - } - - #[test] - fn test_last_published_block_timestamp_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .last_published_block_timestamp - .with_label_values(&["chain_id_1", "block_producer_1"]) - .set(1633046400); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!( - output.contains("publisher_metrics_last_published_block_timestamp") - ); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("1633046400")); - } - - #[test] - fn test_published_messages_throughput_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .published_messages_throughput - .with_label_values(&["chain_id_1", "block_producer_1", "topic_1"]) - .inc_by(10); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!( - output.contains("publisher_metrics_published_messages_throughput") - ); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("topic_1")); - assert!(output.contains("10")); - } - - #[test] - fn test_error_rates_metric() { - let metrics = PublisherMetrics::random(); - - metrics - .error_rates - .with_label_values(&[ - "chain_id_1", - "block_producer_1", - "topic_1", - "timeout", - ]) - .inc_by(1); - - let metric_families = gather(); - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let output = String::from_utf8(buffer.clone()).unwrap(); - - assert!(output.contains("publisher_metrics_error_rates")); - assert!(output.contains("chain_id_1")); - assert!(output.contains("block_producer_1")); - assert!(output.contains("topic_1")); - assert!(output.contains("timeout")); - assert!(output.contains("1")); - } -} diff --git a/crates/fuel-streams-publisher/src/telemetry/system.rs b/crates/fuel-streams-publisher/src/telemetry/system.rs deleted file mode 100644 index ec0f14eb..00000000 --- a/crates/fuel-streams-publisher/src/telemetry/system.rs +++ /dev/null @@ -1,634 +0,0 @@ -use std::{ - collections::HashMap, - convert::TryFrom, - hash::Hash, - path::PathBuf, - time::Duration, -}; - -use derive_more::Deref; -use rust_decimal::{ - prelude::{FromPrimitive, ToPrimitive}, - Decimal, -}; -use serde::{ser::SerializeStruct, Serialize, Serializer}; -use sysinfo::{ - CpuExt, - CpuRefreshKind, - DiskExt, - Pid, - PidExt, - ProcessExt, - RefreshKind, - SystemExt, -}; -use thiserror::Error; -use tokio::time; - -// TODO: move this to web interface as `SystemsMetricsResponse` ? -#[derive(Serialize)] -pub struct SystemMetricsWrapper { - system: SystemMetrics, -} - -impl From for SystemMetricsWrapper { - fn from(system: SystemMetrics) -> Self { - Self { system } - } -} - -#[derive(Debug, Error)] -pub enum Error { - #[error("The process {0} could not be found")] - ProcessNotFound(Pid), -} - -#[derive(Debug, Deref)] -pub struct System { - /// System information from the `sysinfo` crate. - #[deref] - system: sysinfo::System, - /// Refresh settings. - specifics: RefreshKind, - /// Cached physical CPU core count. - cpu_physical_core_count: Option, - /// Process ID. - pid: Pid, -} - -impl System { - pub async fn new() -> Self { - let specifics = RefreshKind::new().with_disks_list().with_memory(); - // Gathering CPU information takes about 150ms+ extra. - let specifics = specifics.with_cpu(CpuRefreshKind::everything()); - - let mut system = sysinfo::System::new_with_specifics(specifics); - - // We're only interested in the current process. - // NOTE: This ::expect can never fail on Linux! - let pid = sysinfo::get_current_pid().expect("Unable to get PID"); - system.refresh_process(pid); - - // We have to refresh the CPU statistics once on startup. - time::sleep(Duration::from_millis(100)).await; - system.refresh_process(pid); - - // Only retrieve the physical CPU core count once (while - // hotplug CPUs exist on virtual and physical platforms, we - // just assume that it is usually not changing on runtime). - let cpu_physical_core_count = system.physical_core_count(); - - Self { - system, - specifics, - cpu_physical_core_count, - pid, - } - } - - pub fn refresh(&mut self) { - self.system.refresh_process(self.pid); - self.system.refresh_specifics(self.specifics); - } - - pub fn metrics(&self) -> Result { - SystemMetrics::try_from(self) - } - - fn pid(&self) -> Pid { - self.pid - } -} - -/// Accumulated system status information. -#[derive(Debug, Default, Serialize)] -pub struct SystemMetrics { - /// Parent process of the application. - pub application: Process, - /// System memory information. - pub memory: SystemMemory, - /// Load averages - pub load_average: LoadAverage, - /// Host and operation system information. - pub host: Host, - /// Disk information and usage. - pub disk: HashMap, - /// CPU physical core count. - #[serde(serialize_with = "format_value")] - pub cpu_physical_core_count: usize, - /// CPU count. - #[serde(serialize_with = "format_value")] - pub cpu_count: usize, - /// CPU information. - pub cpu: HashMap, -} - -impl TryFrom<&System> for SystemMetrics { - type Error = Error; - - fn try_from(system: &System) -> Result { - // Get current pid. - let pid = system.pid(); - - let disk = system - .disks() - .iter() - .map(|v| { - let path = v.mount_point().to_path_buf(); - let disk = Disk::from(v); - (path, disk) - }) - .collect(); - - let cpu = system - .cpus() - .iter() - .enumerate() - .map(|(i, v)| (i, v.into())) - .collect::>(); - // Total number of CPUs (including CPU threads). - let cpu_count = cpu.len(); - - // Use cached number of CPU physical cores, if set. - let cpu_physical_core_count = system - .cpu_physical_core_count - .unwrap_or_else(|| system.physical_core_count().unwrap_or(1)); - - Ok(Self { - application: TryFrom::try_from((system.deref(), pid))?, - memory: system.deref().into(), - load_average: system.deref().into(), - host: system.deref().into(), - disk, - cpu_count, - cpu_physical_core_count, - cpu, - }) - } -} - -/// System memory usage information. -#[derive(Debug, Clone, Default)] -pub struct Memory { - /// Total memory. - size: u64, - /// Used memory. - free: Option, - /// Memory usage in percent. - usage: Decimal, -} - -impl serde::Serialize for Memory { - fn serialize( - &self, - serializer: S, - ) -> Result { - if let Some(free) = self.free { - let mut s = serializer.serialize_struct("Memory", 3)?; - s.serialize_field("size", &Format::Memory(self.size))?; - s.serialize_field("free", &Format::Memory(free))?; - s.serialize_field("usage", &Format::Memory(AsF64(self.usage)))?; - s.end() - } else { - let mut s = serializer.serialize_struct("Memory", 2)?; - s.serialize_field("size", &Format::Memory2(self.size))?; - s.serialize_field("usage", &Format::Memory2(AsF64(self.usage)))?; - s.end() - } - } -} - -/// System memory usage information. -#[derive(Debug, Default, Serialize)] -pub struct SystemMemory { - /// System memory. - system: Memory, - /// Swap memory. - swap: Memory, -} - -impl From<&sysinfo::System> for SystemMemory { - fn from(system: &sysinfo::System) -> Self { - let size = system.total_memory(); - let used = system.used_memory(); - let free = Some(size.saturating_sub(used)); - let usage = percent_usage(used, size); - - let swap_size = system.total_swap(); - let swap_used = system.used_swap(); - let swap_free = Some(swap_size.saturating_sub(swap_used)); - let swap_usage = percent_usage(swap_used, swap_size); - - Self { - system: Memory { size, free, usage }, - swap: Memory { - size: swap_size, - free: swap_free, - usage: swap_usage, - }, - } - } -} - -/// Process information and metrics. -#[derive(Debug)] -pub struct Process { - pid: Pid, - name: String, - cpu_usage: Decimal, - memory: Memory, -} - -impl Default for Process { - fn default() -> Self { - Self { - pid: Pid::from(0), - name: Default::default(), - cpu_usage: Default::default(), - memory: Default::default(), - } - } -} - -impl serde::Serialize for Process { - fn serialize( - &self, - serializer: S, - ) -> Result { - let mut s = serializer.serialize_struct("Process", 4)?; - s.serialize_field( - "pid", - &Format::::Process(self.pid.as_u32() as i32), - )?; - s.serialize_field("name", &FormatKey(&self.name))?; - s.serialize_field( - "cpu_usage", - &Format::Process(AsF64(self.cpu_usage)), - )?; - s.serialize_field("memory", &self.memory)?; - s.end() - } -} - -impl TryFrom<(&sysinfo::System, Pid)> for Process { - type Error = Error; - - fn try_from( - (system, pid): (&sysinfo::System, Pid), - ) -> Result { - let process = system.process(pid).ok_or(Error::ProcessNotFound(pid))?; - - let total = system.total_memory(); - let size = process.memory(); - let usage = percent_usage(size, total); - - Ok(Self { - memory: Memory { - size, - free: None, - usage, - }, - ..Self::from(process) - }) - } -} - -impl From<&sysinfo::Process> for Process { - fn from(process: &sysinfo::Process) -> Self { - Self { - name: process.name().to_string(), - pid: process.pid(), - cpu_usage: decimal(process.cpu_usage()), - memory: Default::default(), - } - } -} - -/// Disk information and usage. -#[derive(Debug)] -pub struct Disk { - size: u64, - free: u64, - usage: Decimal, -} - -impl serde::Serialize for Disk { - fn serialize( - &self, - serializer: S, - ) -> Result { - let mut s = serializer.serialize_struct("Disk", 2)?; - s.serialize_field("size", &Format::Disk(self.size))?; - s.serialize_field("free", &Format::Disk(self.free))?; - s.serialize_field("usage", &Format::Disk(AsF64(self.usage)))?; - s.end() - } -} - -impl From<&sysinfo::Disk> for Disk { - fn from(disk: &sysinfo::Disk) -> Self { - let size = disk.total_space(); - let free = disk.available_space(); - let used = size.saturating_sub(free); - - // Calculate the disk usage in percent. - let usage = percent_usage(used, size); - - Self { size, free, usage } - } -} - -/// System memory usage information. -#[derive(Debug, Default)] -pub struct LoadAverage(f64, f64, f64); - -impl serde::Serialize for LoadAverage { - fn serialize( - &self, - serializer: S, - ) -> Result { - let mut s = serializer.serialize_struct("LoadAverage", 3)?; - s.serialize_field("1", &Format::LoadAverage(self.0))?; - s.serialize_field("5", &Format::LoadAverage(self.1))?; - s.serialize_field("15", &Format::LoadAverage(self.2))?; - s.end() - } -} - -impl From<&sysinfo::System> for LoadAverage { - fn from(system: &sysinfo::System) -> Self { - let load_avg = system.load_average(); - Self(load_avg.one, load_avg.five, load_avg.fifteen) - } -} - -/// System memory usage information. -#[derive(Debug, Default)] -pub struct Cpu { - #[allow(dead_code)] - name: String, - frequency: u64, - usage: Decimal, -} - -impl serde::Serialize for Cpu { - fn serialize( - &self, - serializer: S, - ) -> Result { - let mut s = serializer.serialize_struct("Cpu", 2)?; - s.serialize_field("frequency", &Format::Cpu(self.frequency))?; - s.serialize_field("usage", &Format::Cpu(AsF64(self.usage)))?; - s.end() - } -} - -impl From<&sysinfo::Cpu> for Cpu { - fn from(cpu: &sysinfo::Cpu) -> Self { - Self { - name: cpu.brand().to_string(), - frequency: cpu.frequency(), - usage: decimal(cpu.cpu_usage()), - } - } -} - -/// System memory usage information. -#[derive(Debug, Default)] -pub struct Host { - os_version: String, - kernel_version: String, - uptime: u64, -} - -impl serde::Serialize for Host { - fn serialize( - &self, - serializer: S, - ) -> Result { - let mut s = serializer.serialize_struct("Host", 3)?; - s.serialize_field("os_version", &FormatKey(&self.os_version))?; - s.serialize_field("kernel_version", &FormatKey(&self.kernel_version))?; - s.serialize_field("uptime", &Format::Host(self.uptime))?; - s.end() - } -} - -impl From<&sysinfo::System> for Host { - fn from(system: &sysinfo::System) -> Self { - Self { - os_version: system.long_os_version().unwrap_or_default(), - kernel_version: system.kernel_version().unwrap_or_default(), - uptime: system.uptime(), - } - } -} - -struct AsF64(Decimal); - -impl Serialize for AsF64 { - fn serialize( - &self, - serializer: S, - ) -> Result { - use serde::ser::Error; - let value = self.0.to_f64().ok_or_else(|| { - S::Error::custom(format!( - "Failed to convert a Decimal value into a f64: {:?}", - self.0 - )) - })?; - value.serialize(serializer) - } -} - -pub enum Format { - Cpu(T), - Disk(T), - Host(T), - LoadAverage(T), - Memory(T), - Memory2(T), - Process(T), -} - -impl serde::Serialize for Format { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - // https://en.wikipedia.org/wiki/Brainfuck light - let (code, v) = match &self { - Self::Host(v) => ("<<-|", v), - Self::Cpu(v) => (" (" ("<<-|", v), - Self::Memory(v) => (" ("< ("<<<|", v), - }; - - serializer.serialize_newtype_struct(code, v) - } -} - -pub struct FormatKey(T); - -impl serde::Serialize for FormatKey { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut hashmap = HashMap::new(); - hashmap.insert(&self.0, 1); - serializer.serialize_newtype_struct(".<<<|", &hashmap) - } -} - -fn format_value(value: &usize, serializer: S) -> Result -where - S: Serializer, -{ - serializer.serialize_newtype_struct("<<-|", value) -} - -const DECIMAL_PRECISION: u32 = 4; - -#[inline] -fn percent_usage(current: u64, max: u64) -> Decimal { - Decimal::from(current) - .checked_div(Decimal::from(max)) - .unwrap_or_default() - .checked_mul(100.into()) - .unwrap_or_default() - .round_dp(DECIMAL_PRECISION) -} - -#[inline] -fn decimal(current: f32) -> Decimal { - Decimal::from_f32(current) - .unwrap_or_default() - .round_dp(DECIMAL_PRECISION) -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use rust_decimal::Decimal; - use serde::Serialize; - - use super::*; - - #[derive(Serialize)] - pub struct Metrics { - system: super::SystemMetrics, - } - - impl From<&System> for Metrics { - fn from(system: &System) -> Self { - Self { - system: system.metrics().expect("metrics"), - } - } - } - - #[tokio::test] - async fn test_metrics_system_values() { - let system = System::new().await; - let metrics = Metrics::from(&system); - - // NOTE: This ::expect can never fail on Linux! - let pid = sysinfo::get_current_pid().expect("Unable to get PID"); - assert_eq!(metrics.system.application.pid, pid); - assert!(metrics.system.host.uptime > 0); - assert!(!metrics.system.cpu.is_empty()); - assert!(!metrics.system.disk.is_empty()); - } - - #[tokio::test] - async fn test_metrics_system_prometheus_full() { - let memory = Memory { - size: 1000, - free: Some(877), - usage: Decimal::new(1234, 2), - }; - - let metrics = Metrics { - system: SystemMetrics { - application: Process { - pid: Pid::from(0), - name: "process".to_string(), - cpu_usage: Decimal::new(1234, 2), - memory: memory.clone(), - }, - memory: SystemMemory { - system: memory.clone(), - swap: memory, - }, - load_average: LoadAverage(1.2, 2.3, 3.4), - host: Host { - os_version: "os-version".to_string(), - kernel_version: "kernel-version".to_string(), - uptime: 123456, - }, - disk: vec![( - PathBuf::from("disk1"), - Disk { - size: 1000, - free: 877, - usage: Decimal::new(1234, 2), - }, - )] - .into_iter() - .collect(), - cpu_physical_core_count: 1, - cpu_count: 1, - cpu: vec![( - 1, - Cpu { - name: "cpu1".to_string(), - frequency: 12345, - usage: Decimal::new(1234, 2), - }, - )] - .into_iter() - .collect(), - }, - }; - - let output = serde_prometheus::to_string(&metrics, None, &[]) - .expect("prometheus"); - - assert_eq!( - output.trim_end().split('\n').collect::>(), - vec![ - r#"system_application_pid 0"#, - r#"system_application_name{path = "process"} 1"#, - r#"system_application_cpu_usage 12.34"#, - r#"system_application_size{type = "memory"} 1000"#, - r#"system_application_free{type = "memory"} 877"#, - r#"system_application_usage{type = "memory"} 12.34"#, - r#"system_memory_size{type = "system"} 1000"#, - r#"system_memory_free{type = "system"} 877"#, - r#"system_memory_usage{type = "system"} 12.34"#, - r#"system_memory_size{type = "swap"} 1000"#, - r#"system_memory_free{type = "swap"} 877"#, - r#"system_memory_usage{type = "swap"} 12.34"#, - r#"system_load_average_1 1.2"#, - r#"system_load_average_5 2.3"#, - r#"system_load_average_15 3.4"#, - r#"system_host_os_version{path = "os-version"} 1"#, - r#"system_host_kernel_version{path = "kernel-version"} 1"#, - r#"system_host_uptime 123456"#, - r#"system_disk_size{path = "disk1"} 1000"#, - r#"system_disk_free{path = "disk1"} 877"#, - r#"system_disk_usage{path = "disk1"} 12.34"#, - r#"system_cpu_physical_core_count 1"#, - r#"system_cpu_count 1"#, - r#"system_cpu_frequency{id = "1"} 12345"#, - r#"system_cpu_usage{id = "1"} 12.34"#, - ] - ) - } -} diff --git a/crates/fuel-streams-storage/Cargo.toml b/crates/fuel-streams-storage/Cargo.toml index c8478563..2aefacfb 100644 --- a/crates/fuel-streams-storage/Cargo.toml +++ b/crates/fuel-streams-storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fuel-streams-storage" -description = "strategies and adapters for storing fuel streams in transient and file storage systems (i.e. NATS and S3)" +description = "Srategies and adapters for storing fuel streams in transient and file storage systems (i.e. NATS and S3)" authors = { workspace = true } keywords = { workspace = true } edition = { workspace = true } @@ -11,14 +11,11 @@ version = { workspace = true } rust-version = { workspace = true } [dependencies] -async-nats = { workspace = true } aws-config = { version = "1.5.10", features = ["behavior-version-latest"] } aws-sdk-s3 = "1.65.0" aws-smithy-runtime-api = "1.7.3" aws-smithy-types = "=1.2.9" -displaydoc = { workspace = true } dotenvy = { workspace = true } -fuel-networks = { workspace = true } rand = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } diff --git a/crates/fuel-streams-storage/src/lib.rs b/crates/fuel-streams-storage/src/lib.rs index 15f3b0ec..f5cf85aa 100644 --- a/crates/fuel-streams-storage/src/lib.rs +++ b/crates/fuel-streams-storage/src/lib.rs @@ -1,7 +1,3 @@ // TODO: Introduce Adapters for Transient and FileStorage (NATS and S3 clients would implement those) - -pub mod nats; pub mod s3; - -pub use nats::*; pub use s3::*; diff --git a/crates/fuel-streams-storage/src/nats/mod.rs b/crates/fuel-streams-storage/src/nats/mod.rs deleted file mode 100644 index 63359fd2..00000000 --- a/crates/fuel-streams-storage/src/nats/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -/// Houses shared APIs for interacting with NATS for fuel-streams-publisher and fuel-streams crates -/// As much as possible, the public interface/APIS should be agnostic of NATS. These can then be extended -/// in the fuel-streams-publisher and fuel-streams crates to provide a more opinionated API towards -/// their specific use-cases. -mod error; -mod nats_client; -mod nats_client_opts; -mod nats_namespace; - -pub mod types; - -pub use error::*; -pub use nats_client::*; -pub use nats_client_opts::*; -pub use nats_namespace::*; -pub use types::*; diff --git a/crates/fuel-streams-storage/src/nats/nats_client_opts.rs b/crates/fuel-streams-storage/src/nats/nats_client_opts.rs deleted file mode 100644 index c43833d1..00000000 --- a/crates/fuel-streams-storage/src/nats/nats_client_opts.rs +++ /dev/null @@ -1,147 +0,0 @@ -use std::time::Duration; - -use async_nats::ConnectOptions; -use fuel_networks::{FuelNetwork, FuelNetworkUserRole}; - -use super::NatsNamespace; - -/// Represents options for configuring a NATS client. -/// -/// # Examples -/// -/// Creating a new `NatsClientOpts` instance: -/// -/// ``` -/// use fuel_streams_storage::nats::NatsClientOpts; -/// use fuel_networks::FuelNetwork; -/// -/// let opts = NatsClientOpts::new(FuelNetwork::Local); -/// ``` -/// -/// Creating a public `NatsClientOpts`: -/// -/// ``` -/// use fuel_streams_storage::nats::NatsClientOpts; -/// use fuel_networks::FuelNetwork; -/// -/// let opts = NatsClientOpts::new(FuelNetwork::Local); -/// ``` -/// -/// Modifying `NatsClientOpts`: -/// -/// ``` -/// use fuel_streams_storage::nats::NatsClientOpts; -/// use fuel_networks::{FuelNetwork, FuelNetworkUserRole}; -/// -/// let opts = NatsClientOpts::new(FuelNetwork::Local) -/// .with_role(FuelNetworkUserRole::Admin) -/// .with_timeout(10); -/// ``` -#[derive(Debug, Clone)] -pub struct NatsClientOpts { - pub network: FuelNetwork, - /// The role of the user connecting to the NATS server (Admin or Public). - pub(crate) role: FuelNetworkUserRole, - /// The namespace used as a prefix for NATS streams, consumers, and subject names. - pub(crate) namespace: NatsNamespace, - /// The timeout in seconds for NATS operations. - pub(crate) timeout_secs: u64, - /// URL of the NATS server. - pub(crate) url: Option, -} - -impl NatsClientOpts { - pub fn new(network: FuelNetwork) -> Self { - Self { - network, - role: FuelNetworkUserRole::default(), - namespace: NatsNamespace::default(), - timeout_secs: 5, - url: None, - } - } - - pub fn admin_opts() -> Self { - Self::new(FuelNetwork::load_from_env()) - .with_role(FuelNetworkUserRole::Admin) - } - - pub fn with_role(self, role: FuelNetworkUserRole) -> Self { - Self { role, ..self } - } - - pub fn with_url(self, url: String) -> Self { - Self { - url: Some(url), - ..self - } - } - - pub fn get_url(&self) -> String { - match self.url.clone() { - Some(url) => url, - None => match self.role { - FuelNetworkUserRole::Admin => dotenvy::var("NATS_URL") - .expect("NATS_URL must be set for admin role"), - FuelNetworkUserRole::Default => self.network.to_nats_url(), - }, - } - } - - #[cfg(any(test, feature = "test-helpers"))] - pub fn with_rdn_namespace(self) -> Self { - let namespace = format!(r"namespace-{}", Self::random_int()); - self.with_namespace(&namespace) - } - - #[cfg(any(test, feature = "test-helpers"))] - pub fn with_namespace(self, namespace: &str) -> Self { - let namespace = NatsNamespace::Custom(namespace.to_string()); - Self { namespace, ..self } - } - - pub fn with_timeout(self, secs: u64) -> Self { - Self { - timeout_secs: secs, - ..self - } - } - - pub(super) fn connect_opts(&self) -> ConnectOptions { - let (user, pass) = match self.role { - FuelNetworkUserRole::Admin => ( - Some("admin".to_string()), - Some( - dotenvy::var("NATS_ADMIN_PASS") - .expect("`NATS_ADMIN_PASS` env must be set"), - ), - ), - FuelNetworkUserRole::Default => { - (Some("default_user".to_string()), Some("".to_string())) - } - }; - - match (user, pass) { - (Some(user), Some(pass)) => { - ConnectOptions::with_user_and_password(user, pass) - .connection_timeout(Duration::from_secs(self.timeout_secs)) - .max_reconnects(1) - .name(Self::conn_id()) - } - _ => ConnectOptions::new() - .connection_timeout(Duration::from_secs(self.timeout_secs)) - .max_reconnects(1) - .name(Self::conn_id()), - } - } - - // This will be useful for debugging and monitoring connections - fn conn_id() -> String { - format!(r"connection-{}", Self::random_int()) - } - - fn random_int() -> u32 { - use rand::Rng; - rand::thread_rng().gen() - } -} diff --git a/crates/fuel-streams-storage/src/s3/s3_client.rs b/crates/fuel-streams-storage/src/s3/s3_client.rs index ea07be30..dad2ec35 100644 --- a/crates/fuel-streams-storage/src/s3/s3_client.rs +++ b/crates/fuel-streams-storage/src/s3/s3_client.rs @@ -1,4 +1,4 @@ -use aws_config::{meta::region::RegionProviderChain, Region}; +use aws_config::{BehaviorVersion, Region}; use aws_sdk_s3::{ config::http::HttpResponse, operation::{ @@ -6,7 +6,9 @@ use aws_sdk_s3::{ delete_bucket::DeleteBucketError, delete_object::DeleteObjectError, get_object::GetObjectError, + put_bucket_policy::PutBucketPolicyError, put_object::PutObjectError, + put_public_access_block::PutPublicAccessBlockError, }, Client, }; @@ -34,6 +36,12 @@ pub enum S3ClientError { MissingEnvVar(String), #[error("Failed to stream objects because: {0}")] StreamingError(String), + #[error("Failed to put bucket policy: {0}")] + PutBucketPolicyError(#[from] SdkError), + #[error("Failed to put public access block: {0}")] + PutPublicAccessBlockError( + #[from] SdkError, + ), #[error("IO Error: {0}")] IoError(#[from] std::io::Error), } @@ -46,52 +54,121 @@ pub struct S3Client { impl S3Client { pub async fn new(opts: &S3ClientOpts) -> Result { - // Load AWS configuration - let mut aws_config = aws_config::from_env(); - - if let Some(endpoint_url) = opts.endpoint_url() { - aws_config = aws_config.endpoint_url(endpoint_url); - } - - if let Some(region) = opts.region() { - let region_provider = - RegionProviderChain::first_try(Region::new(region)); - let region = region_provider.region().await.unwrap(); - - aws_config = aws_config.region(region); - } - - let s3_config = - aws_sdk_s3::config::Builder::from(&aws_config.load().await) - .force_path_style(true) - .build(); + let config = aws_config::defaults(BehaviorVersion::latest()) + .endpoint_url(opts.endpoint_url().to_string()) + .region(Region::new(opts.region().to_string())) + // TODO: Remove this once we have a proper S3 bucket created + // for now this is a workaround to avoid signing requests + .no_credentials() + .load() + .await; + + // Create S3 config without signing + let s3_config = aws_sdk_s3::config::Builder::from(&config) + .force_path_style(true) + .disable_s3_express_session_auth(true) + .build(); let client = aws_sdk_s3::Client::from_conf(s3_config); - - Ok(Self { + let s3_client = Self { client, bucket: opts.bucket(), - }) + }; + + Ok(s3_client) } pub fn arc(self) -> std::sync::Arc { std::sync::Arc::new(self) } + pub fn client(&self) -> &Client { + &self.client + } + + pub fn bucket(&self) -> &str { + &self.bucket + } + pub async fn put_object( &self, key: &str, object: Vec, ) -> Result<(), S3ClientError> { - self.client + match self + .client .put_object() .bucket(&self.bucket) .key(key) .body(object.into()) .send() - .await?; - - Ok(()) + .await + { + Ok(_) => Ok(()), + Err(error) => match error { + SdkError::ServiceError(error) => { + tracing::error!( + "Failed to put object in S3 bucket={} key={}: {}", + self.bucket, + key, + error.err() + ); + Err(S3ClientError::PutObjectError(SdkError::ServiceError( + error, + ))) + } + SdkError::ConstructionFailure(error) => { + tracing::error!( + "Failed to construct S3 request for bucket={} key={}", + self.bucket, + key, + ); + Err(S3ClientError::PutObjectError( + SdkError::ConstructionFailure(error), + )) + } + SdkError::TimeoutError(error) => { + tracing::error!( + "Timeout putting object in S3 bucket={} key={}", + self.bucket, + key, + ); + Err(S3ClientError::PutObjectError(SdkError::TimeoutError( + error, + ))) + } + SdkError::DispatchFailure(error) => { + tracing::error!( + "Failed to dispatch S3 request for bucket={} key={}: {}", + self.bucket, + key, + error.as_connector_error().unwrap() + ); + Err(S3ClientError::PutObjectError( + SdkError::DispatchFailure(error), + )) + } + SdkError::ResponseError(error) => { + tracing::error!( + "Invalid response from S3 for bucket={} key={}", + self.bucket, + key, + ); + Err(S3ClientError::PutObjectError(SdkError::ResponseError( + error, + ))) + } + _ => { + tracing::error!( + "Failed to put object in S3 bucket={} key={}: {:?}", + self.bucket, + key, + error + ); + Err(S3ClientError::PutObjectError(error)) + } + }, + } } pub async fn get_object( @@ -135,13 +212,12 @@ impl S3Client { #[cfg(any(test, feature = "test-helpers"))] pub async fn new_for_testing() -> Self { - use fuel_networks::FuelNetwork; - dotenvy::dotenv().expect(".env file not found"); - let s3_client = Self::new( - &S3ClientOpts::new(FuelNetwork::Local).with_random_namespace(), - ) + let s3_client = Self::new(&S3ClientOpts::new( + crate::S3Env::Local, + crate::S3Role::Admin, + )) .await .expect( "S3Client creation failed. Check AWS Env vars and Localstack setup", diff --git a/crates/fuel-streams-storage/src/s3/s3_client_opts.rs b/crates/fuel-streams-storage/src/s3/s3_client_opts.rs index a0d377a8..468efa30 100644 --- a/crates/fuel-streams-storage/src/s3/s3_client_opts.rs +++ b/crates/fuel-streams-storage/src/s3/s3_client_opts.rs @@ -1,49 +1,94 @@ -use fuel_networks::{FuelNetwork, FuelNetworkUserRole}; +use std::str::FromStr; + +#[derive(Debug, Clone, Default)] +pub enum S3Role { + Admin, + #[default] + Public, +} + +#[derive(Debug, Clone, Default)] +pub enum S3Env { + #[default] + Local, + Testnet, + Mainnet, +} + +impl FromStr for S3Env { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "local" => Ok(S3Env::Local), + "testnet" => Ok(S3Env::Testnet), + "mainnet" => Ok(S3Env::Mainnet), + _ => Err(format!("unknown S3 type: {}", s)), + } + } +} -// Introduced for consistency. -// TODO: make it more ergonomic by probably using FuelNetwork in S3Client directly #[derive(Debug, Clone, Default)] pub struct S3ClientOpts { - pub fuel_network: FuelNetwork, - pub role: FuelNetworkUserRole, + pub s3_env: S3Env, + pub role: S3Role, pub namespace: Option, } impl S3ClientOpts { - pub fn new(fuel_network: FuelNetwork) -> Self { + pub fn new(s3_env: S3Env, role: S3Role) -> Self { + Self { + s3_env, + role, + namespace: None, + } + } + + pub fn from_env(role: Option) -> Self { + let s3_env = std::env::var("NETWORK") + .map(|s| S3Env::from_str(&s).unwrap_or_default()) + .unwrap_or_default(); + Self { - fuel_network, - role: FuelNetworkUserRole::default(), + s3_env, + role: role.unwrap_or_default(), namespace: None, } } pub fn admin_opts() -> Self { - Self::new(FuelNetwork::load_from_env()) - .with_role(FuelNetworkUserRole::Admin) + Self::from_env(Some(S3Role::Admin)) } - pub fn with_role(self, role: FuelNetworkUserRole) -> Self { - Self { role, ..self } + pub fn public_opts() -> Self { + Self::from_env(Some(S3Role::Public)) } - pub fn endpoint_url(&self) -> Option { + pub fn endpoint_url(&self) -> String { match self.role { - FuelNetworkUserRole::Admin => dotenvy::var("AWS_ENDPOINT_URL").ok(), - FuelNetworkUserRole::Default => Some(self.fuel_network.to_s3_url()), + S3Role::Admin => dotenvy::var("AWS_ENDPOINT_URL") + .expect("AWS_ENDPOINT_URL must be set for admin role"), + S3Role::Public => { + match self.s3_env { + S3Env::Local => "http://localhost:4566".to_string(), + S3Env::Testnet | S3Env::Mainnet => { + let bucket = self.bucket(); + let region = self.region(); + format!("https://{bucket}.s3-website-{region}.amazonaws.com") + } + } + } } } - pub fn region(&self) -> Option { - match self.role { - FuelNetworkUserRole::Admin => dotenvy::var("AWS_S3_REGION").ok(), - FuelNetworkUserRole::Default => { - Some(self.fuel_network.to_s3_region()) - } + pub fn region(&self) -> String { + match &self.role { + S3Role::Admin => dotenvy::var("AWS_REGION") + .expect("AWS_REGION must be set for admin role"), + S3Role::Public => "us-east-1".to_string(), } } - // TODO: Consider revamping and reusing NATs' Namespace here #[cfg(any(test, feature = "test-helpers"))] pub fn with_random_namespace(mut self) -> Self { let random_namespace = { @@ -56,14 +101,20 @@ impl S3ClientOpts { } pub fn bucket(&self) -> String { - match self.role { - FuelNetworkUserRole::Admin => dotenvy::var("AWS_S3_BUCKET_NAME") - .expect("AWS_S3_BUCKET_NAME must be set for admin role"), - FuelNetworkUserRole::Default => format!( - "{}-{}", - self.fuel_network.to_s3_bucket(), - self.namespace.to_owned().unwrap_or_default() - ), + if matches!(self.role, S3Role::Admin) { + return dotenvy::var("AWS_S3_BUCKET_NAME") + .expect("AWS_S3_BUCKET_NAME must be set for admin role"); } + + let base_bucket = match self.s3_env { + S3Env::Local => "fuel-streams-local", + S3Env::Testnet => "fuel-streams-testnet", + S3Env::Mainnet => "fuel-streams", + }; + + self.namespace + .as_ref() + .map(|ns| format!("{base_bucket}-{ns}")) + .unwrap_or(base_bucket.to_string()) } } diff --git a/crates/fuel-streams-ws/src/server/ws/socket.rs b/crates/fuel-streams-ws/src/server/ws/socket.rs deleted file mode 100644 index a705b2cf..00000000 --- a/crates/fuel-streams-ws/src/server/ws/socket.rs +++ /dev/null @@ -1,397 +0,0 @@ -use std::sync::{atomic::AtomicUsize, Arc}; - -use actix_web::{ - web::{self, Bytes}, - HttpMessage, - HttpRequest, - Responder, -}; -use actix_ws::{Message, Session}; -use fuel_streams::{ - logs::Log, - types::{Block, Input, Output, Receipt, Transaction}, - utxos::Utxo, - StreamEncoder, - Streamable, -}; -use fuel_streams_core::SubscriptionConfig; -use fuel_streams_storage::DeliverPolicy; -use futures::StreamExt; -use uuid::Uuid; - -use super::{ - errors::WsSubscriptionError, - fuel_streams::FuelStreams, - models::ClientMessage, -}; -use crate::{ - server::{ - state::ServerState, - ws::{ - fuel_streams::FuelStreamsExt, - models::{ServerMessage, SubscriptionPayload}, - }, - }, - telemetry::Telemetry, -}; - -static _NEXT_USER_ID: AtomicUsize = AtomicUsize::new(1); - -pub async fn get_ws( - req: HttpRequest, - body: web::Payload, - state: web::Data, -) -> actix_web::Result { - // extract user id - let user_id = match req.extensions().get::() { - Some(user_id) => { - tracing::info!( - "Authenticated WebSocket connection for user: {:?}", - user_id.to_string() - ); - user_id.to_owned() - } - None => { - tracing::info!("Unauthenticated WebSocket connection"); - return Err(actix_web::error::ErrorUnauthorized( - "Missing or invalid JWT", - )); - } - }; - - // split the request into response, session, and message stream - let (response, session, mut msg_stream) = actix_ws::handle(&req, body)?; - - // record the new subscription - state.context.telemetry.increment_subscriptions_count(); - - // spawm an actor handling the ws connection - let streams = state.context.fuel_streams.clone(); - let telemetry = state.context.telemetry.clone(); - actix_web::rt::spawn(async move { - tracing::info!("Ws opened for user id {:?}", user_id.to_string()); - while let Some(Ok(msg)) = msg_stream.recv().await { - let mut session = session.clone(); - match msg { - Message::Ping(bytes) => { - tracing::info!("Received ping, {:?}", bytes); - if session.pong(&bytes).await.is_err() { - tracing::error!("Error sending pong, {:?}", bytes); - } - } - Message::Pong(bytes) => { - tracing::info!("Received pong, {:?}", bytes); - } - Message::Text(string) => { - tracing::info!("Received text, {string}"); - } - Message::Binary(bytes) => { - tracing::info!("Received binary {:?}", bytes); - let client_message = match parse_client_message(bytes) { - Ok(msg) => msg, - Err(e) => { - close_socket_with_error( - e, user_id, session, None, telemetry, - ) - .await; - return; - } - }; - - // handle the client message - match client_message { - ClientMessage::Subscribe(payload) => { - tracing::info!( - "Received subscribe message: {:?}", - payload - ); - let subject_wildcard = payload.wildcard; - let deliver_policy = payload.deliver_policy; - - // verify the subject name - let sub_subject = - match verify_and_extract_subject_name( - &subject_wildcard, - ) { - Ok(res) => res, - Err(e) => { - close_socket_with_error( - e, - user_id, - session, - Some(subject_wildcard.clone()), - telemetry, - ) - .await; - return; - } - }; - - // start the streamer async - let mut stream_session = session.clone(); - - // reply to socket with subscription - send_message_to_socket( - &mut session, - ServerMessage::Subscribed( - SubscriptionPayload { - wildcard: subject_wildcard.clone(), - deliver_policy, - }, - ), - ) - .await; - - // receive streaming in a background thread - let streams = streams.clone(); - let telemetry = telemetry.clone(); - actix_web::rt::spawn(async move { - // update metrics - telemetry.update_user_subscription_metrics( - user_id, - &subject_wildcard, - ); - - // subscribe to the stream - let config = SubscriptionConfig { - deliver_policy: DeliverPolicy::All, - filter_subjects: vec![ - subject_wildcard.clone() - ], - }; - let mut sub = match streams - .subscribe(&sub_subject, Some(config)) - .await - { - Ok(sub) => sub, - Err(e) => { - close_socket_with_error( - WsSubscriptionError::Stream(e), - user_id, - session, - Some(subject_wildcard.clone()), - telemetry, - ) - .await; - return; - } - }; - - // consume and forward to the ws - while let Some(s3_serialized_payload) = - sub.next().await - { - // decode and serialize back to ws payload - let serialized_ws_payload = match decode( - &subject_wildcard, - s3_serialized_payload, - ) - .await - { - Ok(res) => res, - Err(e) => { - telemetry.update_error_metrics( - &subject_wildcard, - &e.to_string(), - ); - tracing::error!("Error serializing received stream message: {:?}", e); - continue; - } - }; - - // send the payload over the stream - let _ = stream_session - .binary(serialized_ws_payload) - .await; - } - }); - } - ClientMessage::Unsubscribe(payload) => { - tracing::info!( - "Received unsubscribe message: {:?}", - payload - ); - let subject_wildcard = payload.wildcard; - - let deliver_policy = payload.deliver_policy; - - if let Err(e) = verify_and_extract_subject_name( - &subject_wildcard, - ) { - close_socket_with_error( - e, - user_id, - session, - Some(subject_wildcard.clone()), - telemetry, - ) - .await; - return; - } - - // TODO: implement session management for the same user_id - - // send a message to the client to confirm unsubscribing - send_message_to_socket( - &mut session, - ServerMessage::Unsubscribed( - SubscriptionPayload { - wildcard: subject_wildcard, - deliver_policy, - }, - ), - ) - .await; - return; - } - } - } - Message::Close(reason) => { - tracing::info!( - "Got close event, terminating session with reason {:?}", - reason - ); - let reason_str = - reason.and_then(|r| r.description).unwrap_or_default(); - close_socket_with_error( - WsSubscriptionError::ClosedWithReason( - reason_str.to_string(), - ), - user_id, - session, - None, - telemetry, - ) - .await; - return; - } - _ => { - tracing::error!("Received unknown message type"); - close_socket_with_error( - WsSubscriptionError::ClosedWithReason( - "Unknown message type".to_string(), - ), - user_id, - session, - None, - telemetry, - ) - .await; - return; - } - }; - } - }); - - Ok(response) -} - -fn parse_client_message( - msg: Bytes, -) -> Result { - let msg = serde_json::from_slice::(&msg) - .map_err(WsSubscriptionError::UnparsablePayload)?; - Ok(msg) -} - -fn stream_to_server_message( - msg: Vec, -) -> Result, WsSubscriptionError> { - let server_message = serde_json::to_vec(&ServerMessage::Update(msg)) - .map_err(WsSubscriptionError::UnserializableMessagePayload)?; - Ok(server_message) -} - -pub fn verify_and_extract_subject_name( - subject_wildcard: &str, -) -> Result { - let mut subject_parts = subject_wildcard.split('.'); - // TODO: more advanced checks here with Regex - if subject_parts.clone().count() == 1 { - return Err(WsSubscriptionError::UnsupportedWildcardPattern( - subject_wildcard.to_string(), - )); - } - let subject_name = subject_parts.next().unwrap_or_default(); - if !FuelStreams::is_within_subject_names(subject_name) { - return Err(WsSubscriptionError::UnknownSubjectName( - subject_wildcard.to_string(), - )); - } - Ok(subject_name.to_string()) -} - -async fn close_socket_with_error( - e: WsSubscriptionError, - user_id: uuid::Uuid, - mut session: Session, - subject_wildcard: Option, - telemetry: Arc, -) { - tracing::error!("ws subscription error: {:?}", e.to_string()); - if let Some(subject_wildcard) = subject_wildcard { - telemetry.update_error_metrics(&subject_wildcard, &e.to_string()); - telemetry.update_unsubscribed(user_id, &subject_wildcard); - } - telemetry.decrement_subscriptions_count(); - send_message_to_socket(&mut session, ServerMessage::Error(e.to_string())) - .await; - let _ = session.close(None).await; -} - -async fn send_message_to_socket(session: &mut Session, message: ServerMessage) { - let data = serde_json::to_vec(&message).ok().unwrap_or_default(); - let _ = session.binary(data).await; -} - -async fn decode( - subject_wildcard: &str, - s3_payload: Vec, -) -> Result, WsSubscriptionError> { - let subject = verify_and_extract_subject_name(subject_wildcard)?; - match subject.as_str() { - Transaction::NAME => { - let entity = Transaction::decode_or_panic(s3_payload); - let serialized_data = serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload)?; - stream_to_server_message(serialized_data) - } - Block::NAME => { - let entity = Block::decode_or_panic(s3_payload); - let serialized_data = serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload)?; - stream_to_server_message(serialized_data) - } - Input::NAME => { - let entity = Input::decode_or_panic(s3_payload); - let serialized_data = serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload)?; - stream_to_server_message(serialized_data) - } - Output::NAME => { - let entity = Output::decode_or_panic(s3_payload); - let serialized_data = serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload)?; - stream_to_server_message(serialized_data) - } - Receipt::NAME => { - let entity = Receipt::decode_or_panic(s3_payload); - let serialized_data = serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload)?; - stream_to_server_message(serialized_data) - } - Utxo::NAME => { - let entity = Utxo::decode_or_panic(s3_payload); - let serialized_data = serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload)?; - stream_to_server_message(serialized_data) - } - Log::NAME => { - let entity = Log::decode_or_panic(s3_payload); - let serialized_data = serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload)?; - stream_to_server_message(serialized_data) - } - _ => Err(WsSubscriptionError::UnknownSubjectName(subject.to_string())), - } -} diff --git a/crates/fuel-streams-ws/src/telemetry/elastic_search.rs b/crates/fuel-streams-ws/src/telemetry/elastic_search.rs deleted file mode 100755 index 905ef088..00000000 --- a/crates/fuel-streams-ws/src/telemetry/elastic_search.rs +++ /dev/null @@ -1,323 +0,0 @@ -// TODO: Consider using external lib for elasticsearch -// TODO: Consider modularizing this module further - -use std::{fs, io, path::PathBuf, sync::Arc}; - -use anyhow::Context; -use chrono::Utc; -use displaydoc::Display; -pub use elasticsearch::params::Refresh; -use elasticsearch::{ - self, - auth::{ClientCertificate, Credentials}, - cert::{Certificate, CertificateValidation}, - http::transport::{SingleNodeConnectionPool, Transport, TransportBuilder}, - params, - Elasticsearch, - IndexParts, -}; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use url::{self, Url}; - -pub const ELASTICSEARCH_PATH: &str = "fuel-data-systems"; - -/// LogEntry represents a log entry that will be stored in Elastic Search -/// for monitoring purposes. -/// TODO: Consider adding more useful optional fields to this struct -#[derive(Serialize, Deserialize)] -pub struct LogEntry { - timestamp: chrono::DateTime, - level: String, - message: String, -} - -impl LogEntry { - pub fn new(level: &str, message: &str) -> Self { - Self { - timestamp: Utc::now(), - level: level.to_string(), - message: message.to_string(), - } - } -} - -pub async fn log(elastic_search: Arc, log_entry: LogEntry) { - if let Err(err) = elastic_search - .get_conn() - .index( - ELASTICSEARCH_PATH, - Some("publisher-logs"), - &log_entry, - Some(Refresh::WaitFor), - ) - .await - { - tracing::error!("Failed to log to ElasticSearch: {}", err); - } -} - -pub fn should_use_elasticsearch() -> bool { - dotenvy::var("USE_ELASTIC_LOGGING").is_ok_and(|val| val == "true") -} - -pub async fn new_elastic_search() -> anyhow::Result { - let elasticsearch_url = dotenvy::var("ELASTICSEARCH_URL") - .expect("`ELASTICSEARCH_URL` env must be set"); - let elsaticsearch_username = dotenvy::var("ELASTICSEARCH_USERNAME") - .expect("`ELASTICSEARCH_USERNAME` env must be set"); - let elsaticsearch_password = dotenvy::var("ELASTICSEARCH_PASSWORD") - .expect("`ELASTICSEARCH_PASSWORD` env must be set"); - - let config = Config { - url: elasticsearch_url, - enabled: true, - pool_max_size: Some(2), - username: Some(elsaticsearch_username), - password: Some(elsaticsearch_password), - ..Default::default() - }; - let client = ElasticSearch::new(&config) - .await - .context("Failed to configure Elasticsearch connection")?; - Ok(client) -} - -/// Elasticsearch errors -#[derive(Debug, Display, Error)] -pub enum ElasticSearchError { - /// ElasticSearchConfigError: `{0}` - Config(#[from] elasticsearch::http::transport::BuildError), - /// ElasticSearchDisabled - Disabled, - /// ElasticSearchError: `{0}` - Generic(#[from] elasticsearch::Error), - /// IoError: `{0}` - Io(#[from] io::Error), - /// UrlParseError: `{0}` - UrlParse(#[from] url::ParseError), - /// CertificateError: `{0}`: `{0}` - Certificate(PathBuf, io::Error), - /// SerdeJsonError: `{0}` - SerdeJson(#[from] serde_json::Error), -} - -#[derive(Debug, Deserialize, Clone, Default)] -#[serde(rename_all = "kebab-case")] -#[serde(default)] -pub struct Config { - pub url: String, - pub enabled: bool, - pub username: Option, - pub password: Option, - pub api_key_id: Option, - pub api_key_value: Option, - pub pool_max_size: Option, - pub pool_min_size: Option, - pub tls: Option, -} - -/// TLS acceptor configuration. -#[derive(Debug, Deserialize, Clone, Default)] -#[serde(rename_all = "kebab-case")] -#[serde(default)] -pub struct TlsConfig { - /// Filename of CA certificates in PEM format. - pub ca: Option, - /// Filename of combined TLS client certificate and key in PKCS#12 format. - pub certificate: Option, - /// Optional passphrase to decode the TLS private key. - pub key_passphrase: Option, -} - -#[derive(Debug, Clone)] -pub struct ElasticSearch(ElasticConnection); - -impl ElasticSearch { - pub async fn new(config: &Config) -> Result { - if !config.enabled { - return Err(ElasticSearchError::Disabled); - } - let conn_info = ConnectionInfo::new(config)?; - let conn = conn_info - .get_connection() - .expect("connection must be created"); - Ok(Self(conn)) - } - - pub fn get_conn(&self) -> &ElasticConnection { - &self.0 - } -} - -#[derive(Clone, Debug, PartialEq, Deserialize)] -pub struct BulkResults { - pub errors: bool, - #[serde(rename = "items")] - pub results: Vec>, -} - -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum Operation { - Create(T), - Delete(T), - Index(T), - Update(T), -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize)] -pub struct OperationParams { - #[serde(rename = "_id", skip_serializing_if = "Option::is_none")] - id: Option, - #[serde(rename = "_index", skip_serializing_if = "Option::is_none")] - index: Option, - #[serde(skip_serializing_if = "Option::is_none")] - version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - version_type: Option, -} - -#[derive(Clone, Debug, PartialEq, Deserialize)] -pub struct OperationStatus { - #[serde(rename = "_id")] - pub id: Option, - #[serde(rename = "_index")] - pub index: Option, - #[serde(rename = "status")] - pub http_code: u32, - #[serde(flatten)] - pub result: OperationResult, -} - -#[derive(Clone, Debug, PartialEq, Deserialize)] -pub enum OperationResult { - #[serde(rename = "result")] - Ok(String), - #[serde(rename = "error")] - Error { - #[serde(rename = "type")] - kind: String, - reason: String, - }, -} - -#[derive(Clone, Debug)] -pub struct ConnectionInfo(Transport); - -impl ConnectionInfo { - pub fn new(config: &Config) -> Result { - let url = Url::parse(&config.url)?; - let pool = SingleNodeConnectionPool::new(url); - let transport = TransportBuilder::new(pool); - let tls = config.tls.clone().unwrap_or_default(); - let credentials = match ( - config.api_key_id.as_ref(), - config.api_key_value.as_ref(), - tls.certificate, - ) { - (Some(api_key_id), Some(api_key_value), _) => Some( - Credentials::ApiKey(api_key_id.into(), api_key_value.into()), - ), - (_, _, Some(certificate)) => { - Some(Credentials::Certificate(ClientCertificate::Pkcs12( - fs::read(&certificate).map_err(|err| { - ElasticSearchError::Certificate(certificate, err) - })?, - tls.key_passphrase, - ))) - } - _ => config.username.as_ref().map(|username| { - Credentials::Basic( - username.into(), - config.password.clone().unwrap_or_default(), - ) - }), - }; - let transport = if let Some(ca) = tls.ca { - transport.cert_validation(CertificateValidation::Full( - Certificate::from_pem(&fs::read(&ca).map_err(|err| { - ElasticSearchError::Certificate(ca.clone(), err) - })?) - .map_err(|err| { - ElasticSearchError::Certificate( - ca, - io::Error::new(io::ErrorKind::Other, err), - ) - })?, - )) - } else { - transport - }; - let transport = if let Some(credentials) = credentials { - transport.auth(credentials) - } else { - transport - }; - let inner = transport.build()?; - Ok(Self(inner)) - } - - pub fn get_connection( - &self, - ) -> Result { - let conn = Elasticsearch::new(self.0.clone()); - Ok(ElasticConnection(Some(conn))) - } -} - -#[derive(Debug, Clone)] -pub struct ElasticConnection(Option); - -impl ElasticConnection { - pub fn check_alive(&self) -> Option { - Some(self.0.is_some()) - } - - pub async fn ping(&self) -> Result<(), ElasticSearchError> { - let conn = self.0.as_ref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::ConnectionAborted, - "Connection to Elasticsearch is already closed", - ) - })?; - - let response = conn.ping().send().await?; - let _ = response.error_for_status_code()?; - Ok(()) - } -} - -impl ElasticConnection { - pub async fn index( - &self, - path: &str, - id: Option<&str>, - doc: B, - refresh: Option, - ) -> Result<(), ElasticSearchError> - where - B: Serialize, - { - let conn = self.0.as_ref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::ConnectionAborted, - "Connection to Elasticsearch is already closed", - ) - })?; - let index_parts = id - .map(|id| IndexParts::IndexId(path, id)) - .unwrap_or(IndexParts::Index(path)); - - let response = conn - .index(index_parts) - .body(doc) - .refresh(refresh.unwrap_or(Refresh::False)) - .send() - .await?; - response - .error_for_status_code() - .map(|_| ()) - .map_err(Into::into) - } -} diff --git a/crates/fuel-streams-ws/src/telemetry/runtime.rs b/crates/fuel-streams-ws/src/telemetry/runtime.rs deleted file mode 100644 index 6c3fc637..00000000 --- a/crates/fuel-streams-ws/src/telemetry/runtime.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::{ - collections::VecDeque, - pin::Pin, - sync::{Arc, Mutex}, -}; - -use futures::Future; -use tokio::time::{self, Duration}; - -// Task type: Each task is represented by a Boxed, pinned Future -type Task = Pin + Send + 'static>>; - -#[derive(Clone)] -pub struct Runtime { - task_queue: Arc>>, - max_capacity: usize, - interval: Duration, -} - -impl Runtime { - pub fn new(capacity: usize, interval: Duration) -> Self { - Self { - task_queue: Arc::new(Mutex::new(VecDeque::with_capacity(capacity))), - max_capacity: capacity, - interval, - } - } - - pub fn spawn(&self, task: F) - where - F: Future + Send + 'static, - { - let mut queue = self.task_queue.lock().unwrap(); - - // If the queue is at capacity, discard the oldest task - if queue.len() >= self.max_capacity { - queue.pop_front(); - } - - queue.push_back(Box::pin(task)); - } - - pub fn start(&self, blocking_task_executor: F) - where - F: FnOnce() + Send + 'static + Clone, - { - let interval = self.interval; - let task_queue = Arc::clone(&self.task_queue); - - tokio::spawn(async move { - let mut ticker = time::interval(interval); - - loop { - // Wait for the interval - ticker.tick().await; - - tokio::task::spawn_blocking(blocking_task_executor.clone()); - - // Lock the queue, drain tasks, and run them sequentially - let tasks: Vec<_> = { - let mut queue = task_queue.lock().unwrap(); - queue.drain(..).collect() - }; - - // Run each task sequentially - for task in tasks { - task.await; - } - } - }); - } -} diff --git a/crates/fuel-streams/src/client/client_impl.rs b/crates/fuel-streams/src/client/client_impl.rs index 620d1a7d..b2d9deb1 100644 --- a/crates/fuel-streams/src/client/client_impl.rs +++ b/crates/fuel-streams/src/client/client_impl.rs @@ -36,12 +36,24 @@ impl Client { /// # } /// ``` pub async fn connect(network: FuelNetwork) -> Result { - let nats_opts = NatsClientOpts::new(network); + let nats_opts = + NatsClientOpts::public_opts().with_url(network.to_nats_url()); let nats_client = NatsClient::connect(&nats_opts) .await .map_err(ClientError::NatsConnectionFailed)?; - let s3_client_opts = S3ClientOpts::new(network); + let s3_client_opts = match network { + FuelNetwork::Local => { + S3ClientOpts::new(S3Env::Local, S3Role::Admin) + } + FuelNetwork::Testnet => { + S3ClientOpts::new(S3Env::Testnet, S3Role::Public) + } + FuelNetwork::Mainnet => { + S3ClientOpts::new(S3Env::Mainnet, S3Role::Public) + } + }; + let s3_client = S3Client::new(&s3_client_opts) .await .map_err(ClientError::S3ConnectionFailed)?; @@ -67,11 +79,11 @@ impl Client { /// ```no_run /// use fuel_streams::client::{Client, FuelNetwork}; /// use fuel_streams_core::nats::NatsClientOpts; - /// use fuel_streams_core::s3::S3ClientOpts; + /// use fuel_streams_core::s3::{S3ClientOpts, S3Env, S3Role}; /// /// # async fn example() -> Result<(), fuel_streams::Error> { - /// let nats_opts = NatsClientOpts::new(FuelNetwork::Local); - /// let s3_opts = S3ClientOpts::new(FuelNetwork::Local); + /// let nats_opts = NatsClientOpts::public_opts().with_url("nats://localhost:4222"); + /// let s3_opts = S3ClientOpts::new(S3Env::Local, S3Role::Admin); /// /// let client = Client::with_opts(&nats_opts, &s3_opts).await?; /// # Ok(()) diff --git a/crates/sv-consumer/Cargo.toml b/crates/sv-consumer/Cargo.toml new file mode 100644 index 00000000..50d1e17e --- /dev/null +++ b/crates/sv-consumer/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "sv-consumer" +description = "Service that consumers new blocks from the emitter" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } +publish = false + +[[bin]] +name = "sv-consumer" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +async-nats = { workspace = true } +clap = { workspace = true } +dotenvy = { workspace = true } +fuel-core = { workspace = true, default-features = false, features = ["p2p", "relayer", "rocksdb"] } +fuel-streams-core = { workspace = true, features = ["test-helpers"] } +fuel-streams-executors = { workspace = true, features = ["test-helpers"] } +futures = { workspace = true } +num_cpus = { workspace = true } +serde_json = { workspace = true } +sv-publisher = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = "0.7.13" +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["local-time"] } + +[features] +default = [] +test-helpers = [] + +[target.x86_64-unknown-linux-gnu.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.x86_64-unknown-linux-musl.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.aarch64-unknown-linux-gnu.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.aarch64-unknown-linux-musl.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } diff --git a/crates/sv-consumer/src/cli.rs b/crates/sv-consumer/src/cli.rs new file mode 100644 index 00000000..6b51f6bf --- /dev/null +++ b/crates/sv-consumer/src/cli.rs @@ -0,0 +1,22 @@ +use clap::Parser; + +#[derive(Clone, Parser)] +pub struct Cli { + /// Fuel Network to connect to. + #[arg( + long, + value_name = "NATS_URL", + env = "NATS_URL", + default_value = "localhost:4222", + help = "NATS URL to connect to." + )] + pub nats_url: String, + #[arg( + long, + value_name = "NATS_PUBLISHER_URL", + env = "NATS_PUBLISHER_URL", + default_value = "localhost:4333", + help = "NATS Publisher URL to connect to." + )] + pub nats_publisher_url: String, +} diff --git a/crates/sv-consumer/src/lib.rs b/crates/sv-consumer/src/lib.rs new file mode 100644 index 00000000..1cddb9ed --- /dev/null +++ b/crates/sv-consumer/src/lib.rs @@ -0,0 +1,33 @@ +use std::sync::Arc; + +use fuel_streams_core::prelude::*; + +pub mod cli; + +#[derive(Debug, Clone, Default)] +pub enum Client { + #[default] + Core, + Publisher, +} + +impl Client { + pub fn url(&self, cli: &cli::Cli) -> String { + match self { + Client::Core => cli.nats_url.clone(), + Client::Publisher => cli.nats_publisher_url.clone(), + } + } + pub async fn create( + &self, + cli: &cli::Cli, + ) -> Result, NatsError> { + let url = self.url(cli); + let opts = NatsClientOpts::admin_opts() + .with_url(url) + .with_domain("CORE".to_string()) + .with_user("admin".to_string()) + .with_password("admin".to_string()); + Ok(Arc::new(NatsClient::connect(&opts).await?)) + } +} diff --git a/crates/sv-consumer/src/main.rs b/crates/sv-consumer/src/main.rs new file mode 100644 index 00000000..3e4fb6b2 --- /dev/null +++ b/crates/sv-consumer/src/main.rs @@ -0,0 +1,220 @@ +use std::{ + env, + sync::{Arc, LazyLock}, + time::Duration, +}; + +use async_nats::jetstream::{ + consumer::{ + pull::{BatchErrorKind, Config as ConsumerConfig}, + Consumer, + }, + context::CreateStreamErrorKind, + stream::{ConsumerErrorKind, RetentionPolicy}, +}; +use clap::Parser; +use fuel_streams_core::prelude::*; +use fuel_streams_executors::*; +use futures::{future::try_join_all, stream::FuturesUnordered, StreamExt}; +use sv_consumer::{cli::Cli, Client}; +use sv_publisher::shutdown::ShutdownController; +use tokio_util::sync::CancellationToken; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::fmt::time; + +#[derive(thiserror::Error, Debug)] +pub enum ConsumerError { + #[error("Failed to receive batch of messages from NATS: {0}")] + BatchStream(#[from] async_nats::error::Error), + + #[error("Failed to create stream: {0}")] + CreateStream(#[from] async_nats::error::Error), + + #[error("Failed to create consumer: {0}")] + CreateConsumer(#[from] async_nats::error::Error), + + #[error("Failed to connect to NATS client: {0}")] + NatsClient(#[from] NatsError), + + #[error("Failed to communicate with NATS server: {0}")] + Nats(#[from] async_nats::Error), + + #[error("Failed to deserialize block payload from message: {0}")] + Deserialization(#[from] serde_json::Error), + + #[error("Failed to decode UTF-8: {0}")] + Utf8(#[from] std::str::Utf8Error), + + #[error("Failed to execute executor tasks: {0}")] + Executor(#[from] ExecutorError), + + #[error("Failed to join tasks: {0}")] + JoinTasks(#[from] tokio::task::JoinError), + + #[error("Failed to acquire semaphore: {0}")] + Semaphore(#[from] tokio::sync::AcquireError), + + #[error("Failed to setup S3 client: {0}")] + S3(#[from] S3ClientError), +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Initialize tracing subscriber + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .with_timer(time::LocalTime::rfc_3339()) + .with_target(false) + .with_thread_ids(false) + .with_file(true) + .with_line_number(true) + .with_level(true) + .init(); + + if let Err(err) = dotenvy::dotenv() { + tracing::warn!("File .env not found: {:?}", err); + } + + let cli = Cli::parse(); + let shutdown = Arc::new(ShutdownController::new()); + shutdown.clone().spawn_signal_handler(); + + tracing::info!("Consumer started. Waiting for messages..."); + tokio::select! { + result = async { + process_messages(&cli, shutdown.token()) + .await + } => { + result?; + tracing::info!("Processing complete"); + } + _ = shutdown.wait_for_shutdown() => { + tracing::info!("Shutdown signal received"); + } + }; + + tracing::info!("Shutdown complete"); + Ok(()) +} + +async fn setup_s3() -> Result, ConsumerError> { + let s3_client_opts = S3ClientOpts::admin_opts(); + let s3_client = S3Client::new(&s3_client_opts).await?; + Ok(Arc::new(s3_client)) +} + +async fn setup_nats( + cli: &Cli, +) -> Result< + (Arc, Arc, Consumer), + ConsumerError, +> { + let core_client = Client::Core.create(cli).await?; + let publisher_client = Client::Publisher.create(cli).await?; + let stream_name = publisher_client.namespace.stream_name("block_importer"); + let stream = publisher_client + .jetstream + .get_or_create_stream(async_nats::jetstream::stream::Config { + name: stream_name, + subjects: vec!["block_submitted.>".to_string()], + retention: RetentionPolicy::WorkQueue, + duplicate_window: Duration::from_secs(1), + allow_rollup: true, + ..Default::default() + }) + .await?; + + let consumer = stream + .get_or_create_consumer("block_importer", ConsumerConfig { + durable_name: Some("block_importer".to_string()), + ack_policy: AckPolicy::Explicit, + ..Default::default() + }) + .await?; + + Ok((core_client, publisher_client, consumer)) +} + +pub static CONSUMER_MAX_THREADS: LazyLock = LazyLock::new(|| { + let available_cpus = num_cpus::get(); + env::var("CONSUMER_MAX_THREADS") + .ok() + .and_then(|val| val.parse().ok()) + .unwrap_or(available_cpus) +}); + +async fn process_messages( + cli: &Cli, + token: &CancellationToken, +) -> Result<(), ConsumerError> { + let (core_client, publisher_client, consumer) = setup_nats(cli).await?; + let s3_client = setup_s3().await?; + let (_, publisher_stream) = + FuelStreams::setup_all(&core_client, &publisher_client, &s3_client) + .await; + + let fuel_streams: Arc = publisher_stream.arc(); + let semaphore = Arc::new(tokio::sync::Semaphore::new(64)); + while !token.is_cancelled() { + let mut messages = + consumer.fetch().max_messages(100).messages().await?.fuse(); + let mut futs = FuturesUnordered::new(); + while let Some(msg) = messages.next().await { + let msg = msg?; + let fuel_streams = fuel_streams.clone(); + let semaphore = semaphore.clone(); + let future = async move { + let msg_str = std::str::from_utf8(&msg.payload)?; + let payload = Arc::new(BlockPayload::decode(msg_str)?); + let start_time = std::time::Instant::now(); + let futures = Executor::::process_all( + payload.clone(), + &fuel_streams, + &semaphore, + ); + let results = try_join_all(futures).await?; + let end_time = std::time::Instant::now(); + msg.ack().await?; + Ok::<_, ConsumerError>((results, start_time, end_time, payload)) + }; + futs.push(future); + } + while let Some(result) = futs.next().await { + let (results, start_time, end_time, payload) = result?; + log_task(results, start_time, end_time, payload); + } + } + Ok(()) +} + +fn log_task( + res: Vec>, + start_time: std::time::Instant, + end_time: std::time::Instant, + payload: Arc, +) { + let height = payload.metadata().clone().block_height; + let has_error = res.iter().any(|r| r.is_err()); + let errors = res + .iter() + .filter_map(|r| r.as_ref().err()) + .collect::>(); + + let elapsed = end_time.duration_since(start_time); + if has_error { + tracing::error!( + "Block {height} published with errors in {:?}", + elapsed + ); + tracing::debug!("Errors: {:?}", errors); + } else { + tracing::info!( + "Block {height} published successfully in {:?}", + elapsed + ); + } +} diff --git a/crates/sv-publisher/Cargo.toml b/crates/sv-publisher/Cargo.toml new file mode 100644 index 00000000..fdf306e6 --- /dev/null +++ b/crates/sv-publisher/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "sv-publisher" +description = "Service that emitts new blocks using fuel-core block subscription" +authors = { workspace = true } +keywords = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +rust-version = { workspace = true } +publish = false + +[[bin]] +name = "sv-publisher" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +async-nats = { workspace = true } +clap = { workspace = true } +fuel-core = { workspace = true, default-features = false, features = ["p2p", "relayer", "rocksdb"] } +fuel-core-bin = { workspace = true, default-features = false, features = [ + "p2p", + "relayer", + "rocksdb", +] } +fuel-core-types = { workspace = true, default-features = false, features = ["std", "serde"] } +fuel-streams-core = { workspace = true, features = ["test-helpers"] } +fuel-streams-executors = { workspace = true, features = ["test-helpers"] } +futures = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = "0.7.13" +tracing = { workspace = true } + +[features] +default = [] +test-helpers = [] + +[target.x86_64-unknown-linux-gnu.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.x86_64-unknown-linux-musl.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.aarch64-unknown-linux-gnu.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } + +[target.aarch64-unknown-linux-musl.dependencies] +openssl = { version = "0.10.68", features = ["vendored"] } diff --git a/crates/fuel-streams-publisher/src/cli.rs b/crates/sv-publisher/src/cli.rs similarity index 58% rename from crates/fuel-streams-publisher/src/cli.rs rename to crates/sv-publisher/src/cli.rs index 7363eb81..4573fedd 100644 --- a/crates/fuel-streams-publisher/src/cli.rs +++ b/crates/sv-publisher/src/cli.rs @@ -12,21 +12,13 @@ pub struct Cli { /// Flattened command structure for Fuel Core configuration. #[command(flatten)] pub fuel_core_config: fuel_core_bin::cli::run::Command, - /// Http server address + /// Fuel Network to connect to. #[arg( long, - value_name = "TPORT", - env = "TELEMETRY_PORT", - default_value = "8080", - help = "Port for the Actix Web server to bind telemetry to." + value_name = "NATS_URL", + env = "NATS_URL", + default_value = "localhost:4222", + help = "NATS URL to connect to." )] - pub telemetry_port: u16, - #[arg( - long, - value_name = "HISTORICAL", - env = "HISTORICAL", - default_value = "false", - help = "Whether to publish historical data to NATS" - )] - pub historical: bool, + pub nats_url: String, } diff --git a/crates/sv-publisher/src/lib.rs b/crates/sv-publisher/src/lib.rs new file mode 100644 index 00000000..5bf4a4b0 --- /dev/null +++ b/crates/sv-publisher/src/lib.rs @@ -0,0 +1,2 @@ +pub mod cli; +pub mod shutdown; diff --git a/crates/sv-publisher/src/main.rs b/crates/sv-publisher/src/main.rs new file mode 100644 index 00000000..38b1d491 --- /dev/null +++ b/crates/sv-publisher/src/main.rs @@ -0,0 +1,213 @@ +use std::{sync::Arc, time::Duration}; + +use async_nats::jetstream::{ + context::PublishErrorKind, + stream::RetentionPolicy, + Context, +}; +use clap::Parser; +use fuel_core_types::blockchain::SealedBlock; +use fuel_streams_core::prelude::*; +use fuel_streams_executors::*; +use futures::StreamExt; +use sv_publisher::{cli::Cli, shutdown::ShutdownController}; +use thiserror::Error; +use tokio_util::sync::CancellationToken; + +#[derive(Error, Debug)] +pub enum LiveBlockProcessingError { + #[error("Failed to publish block: {0}")] + PublishError(#[from] PublishError), + + #[error("Processing was cancelled")] + Cancelled, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + let config = cli.fuel_core_config; + let fuel_core: Arc = FuelCore::new(config).await?; + fuel_core.start().await?; + + let s3_client = setup_s3().await?; + let nats_client = setup_nats(&cli.nats_url).await?; + let last_block_height = Arc::new(fuel_core.get_latest_block_height()?); + let last_published = + Arc::new(find_last_published_height(&nats_client, &s3_client).await?); + + let shutdown = Arc::new(ShutdownController::new()); + shutdown.clone().spawn_signal_handler(); + + tracing::info!("Last published height: {}", last_published); + tracing::info!("Last block height: {}", last_block_height); + tokio::select! { + result = async { + let historical = process_historical_blocks( + &nats_client, + fuel_core.clone(), + last_block_height, + last_published, + shutdown.token().clone(), + ); + + let live = process_live_blocks( + &nats_client.jetstream, + fuel_core.clone(), + shutdown.token().clone(), + ); + + tokio::join!(historical, live) + } => { + result.0?; + result.1?; + } + _ = shutdown.wait_for_shutdown() => { + tracing::info!("Shutdown signal received, waiting for processing to complete..."); + fuel_core.stop().await + } + } + + tracing::info!("Shutdown complete"); + Ok(()) +} + +async fn setup_s3() -> anyhow::Result> { + let s3_client_opts = S3ClientOpts::admin_opts(); + let s3_client = S3Client::new(&s3_client_opts).await?; + Ok(Arc::new(s3_client)) +} + +async fn setup_nats(nats_url: &str) -> anyhow::Result { + let opts = NatsClientOpts::admin_opts() + .with_url(nats_url.to_string()) + .with_domain("CORE".to_string()); + let nats_client = NatsClient::connect(&opts).await?; + let stream_name = nats_client.namespace.stream_name("block_importer"); + nats_client + .jetstream + .get_or_create_stream(async_nats::jetstream::stream::Config { + name: stream_name, + subjects: vec!["block_submitted.>".to_string()], + retention: RetentionPolicy::WorkQueue, + duplicate_window: Duration::from_secs(1), + ..Default::default() + }) + .await?; + + Ok(nats_client) +} + +async fn find_last_published_height( + nats_client: &NatsClient, + s3_client: &Arc, +) -> anyhow::Result { + let block_stream = + Stream::::get_or_init(nats_client, s3_client).await; + let last_publish_height = block_stream + .get_last_published(BlocksSubject::WILDCARD) + .await?; + match last_publish_height { + Some(block) => Ok(block.height), + None => Ok(0), + } +} + +fn get_historical_block_range( + last_published_height: Arc, + last_block_height: Arc, +) -> Option> { + let last_published_height = *last_published_height; + let last_block_height = *last_block_height; + let start_height = last_published_height + 1; + let end_height = last_block_height; + if start_height > end_height { + tracing::info!("No historical blocks to process"); + return None; + } + let block_count = end_height - start_height + 1; + let heights: Vec = (start_height..=end_height).collect(); + tracing::info!( + "Processing {block_count} historical blocks from height {start_height} to {end_height}" + ); + Some(heights) +} + +fn process_historical_blocks( + nats_client: &NatsClient, + fuel_core: Arc, + last_block_height: Arc, + last_published_height: Arc, + token: CancellationToken, +) -> tokio::task::JoinHandle<()> { + let jetstream = nats_client.jetstream.clone(); + tokio::spawn(async move { + let Some(heights) = get_historical_block_range( + last_published_height, + last_block_height, + ) else { + return; + }; + futures::stream::iter(heights) + .map(|height| { + let jetstream = jetstream.clone(); + let fuel_core = fuel_core.clone(); + let sealed_block = fuel_core.get_sealed_block_by_height(height); + let sealed_block = Arc::new(sealed_block); + async move { + publish_block(&jetstream, &fuel_core, &sealed_block).await + } + }) + .buffer_unordered(100) + .take_until(token.cancelled()) + .collect::>() + .await; + }) +} + +async fn process_live_blocks( + jetstream: &Context, + fuel_core: Arc, + token: CancellationToken, +) -> Result<(), LiveBlockProcessingError> { + let mut subscription = fuel_core.blocks_subscription(); + while let Ok(data) = subscription.recv().await { + if token.is_cancelled() { + break; + } + let sealed_block = Arc::new(data.sealed_block.clone()); + publish_block(jetstream, &fuel_core, &sealed_block).await?; + } + Ok(()) +} + +#[derive(Error, Debug)] +pub enum PublishError { + #[error("Failed to publish block to NATS server: {0}")] + NatsPublish(#[from] async_nats::error::Error), + + #[error("Failed to create block payload due to: {0}")] + BlockPayload(#[from] ExecutorError), + + #[error("Failed to access offchain database: {0}")] + OffchainDatabase(String), +} + +async fn publish_block( + jetstream: &Context, + fuel_core: &Arc, + sealed_block: &Arc, +) -> Result<(), PublishError> { + let metadata = Metadata::new(fuel_core, sealed_block); + let fuel_core = Arc::clone(fuel_core); + let payload = BlockPayload::new(fuel_core, sealed_block, &metadata)?; + jetstream + .send_publish(payload.subject(), payload.to_owned().try_into()?) + .await + .map_err(PublishError::NatsPublish)? + .await + .map_err(PublishError::NatsPublish)?; + + tracing::info!("New block submitted: {}", payload.block_height()); + Ok(()) +} diff --git a/crates/sv-publisher/src/shutdown.rs b/crates/sv-publisher/src/shutdown.rs new file mode 100644 index 00000000..6d66e7b1 --- /dev/null +++ b/crates/sv-publisher/src/shutdown.rs @@ -0,0 +1,104 @@ +use std::sync::Arc; + +use tokio_util::sync::CancellationToken; + +#[derive(Clone)] +pub struct ShutdownController { + token: CancellationToken, +} + +impl Default for ShutdownController { + fn default() -> Self { + Self::new() + } +} + +impl ShutdownController { + pub fn new() -> Self { + Self { + token: CancellationToken::new(), + } + } + + pub fn token(&self) -> &CancellationToken { + &self.token + } + + pub fn spawn_signal_handler(self: Arc) -> Arc { + tokio::spawn({ + let shutdown = self.clone(); + async move { + tokio::signal::ctrl_c() + .await + .expect("Failed to listen for ctrl+c"); + tracing::info!("Received shutdown signal"); + shutdown.initiate_shutdown(); + } + }); + self + } + + pub fn initiate_shutdown(&self) { + tracing::info!("Initiating graceful shutdown..."); + self.token.cancel(); + } + + pub fn is_shutdown_initiated(&self) -> bool { + self.token.is_cancelled() + } + + pub async fn wait_for_shutdown(&self) { + self.token.cancelled().await; + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[tokio::test] + async fn test_manual_shutdown() { + let controller = ShutdownController::new(); + assert!( + !controller.is_shutdown_initiated(), + "Controller should not be shutdown initially" + ); + + controller.initiate_shutdown(); + assert!( + controller.is_shutdown_initiated(), + "Controller should be shutdown after initiation" + ); + } + + #[tokio::test] + async fn test_wait_for_shutdown_timeout() { + let controller = ShutdownController::new(); + + let timeout = Duration::from_millis(50); + let result = + tokio::time::timeout(timeout, controller.wait_for_shutdown()).await; + + assert!( + result.is_err(), + "wait_for_shutdown should not complete without initiation" + ); + } + + #[tokio::test] + async fn test_clone_behavior() { + let controller = ShutdownController::new(); + let cloned = controller.clone(); + + // Initiate shutdown from clone + cloned.initiate_shutdown(); + + assert!( + controller.is_shutdown_initiated(), + "Original should be shutdown" + ); + assert!(cloned.is_shutdown_initiated(), "Clone should be shutdown"); + } +} diff --git a/crates/fuel-streams-ws/Cargo.toml b/crates/sv-webserver/Cargo.toml similarity index 95% rename from crates/fuel-streams-ws/Cargo.toml rename to crates/sv-webserver/Cargo.toml index 86c66c2d..e1f77ddd 100644 --- a/crates/fuel-streams-ws/Cargo.toml +++ b/crates/sv-webserver/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "fuel-streams-ws" +name = "sv-webserver" description = "Fuel library for streaming data from nats and storage" authors = { workspace = true } keywords = { workspace = true } @@ -19,22 +19,21 @@ actix-web = { workspace = true } actix-ws = "0.3.0" anyhow = { workspace = true } async-nats = { workspace = true } -async-trait = { workspace = true } bytestring = "1.4.0" chrono = { workspace = true } clap = { workspace = true } -confy = "0.6" derive_more = { version = "1.0", features = ["full"] } displaydoc = { workspace = true } dotenvy = { workspace = true } elasticsearch = "8.15.0-alpha.1" fuel-streams = { workspace = true, features = ["test-helpers"] } fuel-streams-core = { workspace = true, features = ["test-helpers"] } +fuel-streams-nats = { workspace = true, features = ["test-helpers"] } fuel-streams-storage = { workspace = true, features = ["test-helpers"] } futures = { workspace = true } futures-util = { workspace = true } jsonwebtoken = "9.3.0" -num_cpus = "1.16" +num_cpus = { workspace = true } parking_lot = { version = "0.12", features = ["serde"] } prometheus = { version = "0.13", features = ["process"] } rand = { workspace = true } @@ -48,7 +47,6 @@ thiserror = "2.0" time = { version = "0.3", features = ["serde"] } tokio = { workspace = true } tokio-tungstenite = "0.24.0" -toml = "0.8.19" tracing = { workspace = true } tracing-actix-web = { workspace = true } tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } diff --git a/crates/fuel-streams-ws/README.md b/crates/sv-webserver/README.md similarity index 97% rename from crates/fuel-streams-ws/README.md rename to crates/sv-webserver/README.md index 5c60af56..cd717d7c 100644 --- a/crates/fuel-streams-ws/README.md +++ b/crates/sv-webserver/README.md @@ -16,7 +16,7 @@

- 📚 Documentation + 📚 Documentation   🐛 Report Bug   diff --git a/crates/fuel-streams-ws/src/cli.rs b/crates/sv-webserver/src/cli.rs similarity index 95% rename from crates/fuel-streams-ws/src/cli.rs rename to crates/sv-webserver/src/cli.rs index 4aed7400..37cad8bd 100644 --- a/crates/fuel-streams-ws/src/cli.rs +++ b/crates/sv-webserver/src/cli.rs @@ -7,11 +7,11 @@ pub struct Cli { #[arg( long, value_name = "PORT", - env = "API_PORT", + env = "PORT", default_value = "9003", help = "Port number for the API server" )] - pub api_port: u16, + pub port: u16, /// NATS URL #[arg( diff --git a/crates/fuel-streams-ws/src/client/mod.rs b/crates/sv-webserver/src/client/mod.rs similarity index 100% rename from crates/fuel-streams-ws/src/client/mod.rs rename to crates/sv-webserver/src/client/mod.rs diff --git a/crates/fuel-streams-ws/src/config.rs b/crates/sv-webserver/src/config.rs similarity index 97% rename from crates/fuel-streams-ws/src/config.rs rename to crates/sv-webserver/src/config.rs index eeb01ce0..eeaada1b 100644 --- a/crates/fuel-streams-ws/src/config.rs +++ b/crates/sv-webserver/src/config.rs @@ -54,7 +54,7 @@ impl Config { fn from_cli(cli: &crate::cli::Cli) -> Result { Ok(Config { api: ApiConfig { - port: cli.api_port, + port: cli.port, tls: None, }, auth: AuthConfig { diff --git a/crates/fuel-streams-ws/src/lib.rs b/crates/sv-webserver/src/lib.rs similarity index 100% rename from crates/fuel-streams-ws/src/lib.rs rename to crates/sv-webserver/src/lib.rs diff --git a/crates/fuel-streams-ws/src/main.rs b/crates/sv-webserver/src/main.rs similarity index 93% rename from crates/fuel-streams-ws/src/main.rs rename to crates/sv-webserver/src/main.rs index e629bd31..253f6fb3 100644 --- a/crates/fuel-streams-ws/src/main.rs +++ b/crates/sv-webserver/src/main.rs @@ -1,4 +1,4 @@ -use fuel_streams_ws::{ +use sv_webserver::{ config::Config, server::{api::create_api, context::Context, state::ServerState}, }; @@ -18,7 +18,7 @@ async fn main() -> anyhow::Result<()> { .init(); if let Err(err) = dotenvy::dotenv() { - tracing::error!("File .env not found: {:?}", err); + tracing::warn!("File .env not found: {:?}", err); } let config = Config::load()?; diff --git a/crates/fuel-streams-ws/src/server/api.rs b/crates/sv-webserver/src/server/api.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/api.rs rename to crates/sv-webserver/src/server/api.rs diff --git a/crates/fuel-streams-ws/src/server/auth.rs b/crates/sv-webserver/src/server/auth.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/auth.rs rename to crates/sv-webserver/src/server/auth.rs diff --git a/crates/fuel-streams-ws/src/server/context.rs b/crates/sv-webserver/src/server/context.rs similarity index 66% rename from crates/fuel-streams-ws/src/server/context.rs rename to crates/sv-webserver/src/server/context.rs index 7bda3220..c0af7476 100644 --- a/crates/fuel-streams-ws/src/server/context.rs +++ b/crates/sv-webserver/src/server/context.rs @@ -3,11 +3,7 @@ use std::{sync::Arc, time::Duration}; use fuel_streams_core::prelude::*; use fuel_streams_storage::S3Client; -use crate::{ - config::Config, - server::ws::fuel_streams::FuelStreams, - telemetry::Telemetry, -}; +use crate::{config::Config, telemetry::Telemetry}; pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(90); @@ -23,8 +19,9 @@ pub struct Context { impl Context { pub async fn new(config: &Config) -> anyhow::Result { - let nats_client_opts = - NatsClientOpts::admin_opts().with_url(config.nats.url.clone()); + let nats_client_opts = NatsClientOpts::admin_opts() + .with_url(config.nats.url.clone()) + .with_domain("CORE"); let nats_client = NatsClient::connect(&nats_client_opts).await?; let s3_client_opts = S3ClientOpts::admin_opts(); let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); @@ -47,28 +44,6 @@ impl Context { }) } - pub async fn new_for_testing( - fuel_network: FuelNetwork, - ) -> anyhow::Result { - let nats_client_opts = NatsClientOpts::new(fuel_network); - let nats_client = NatsClient::connect(&nats_client_opts).await?; - let s3_client_opts = S3ClientOpts::admin_opts(); - let s3_client = Arc::new(S3Client::new(&s3_client_opts).await?); - Ok(Context { - fuel_streams: Arc::new( - FuelStreams::new(&nats_client, &s3_client).await, - ), - nats_client: nats_client.clone(), - telemetry: Telemetry::new(None).await?, - s3_client: None, - jwt_secret: String::new(), - }) - } - - pub fn get_streams(&self) -> &FuelStreams { - &self.fuel_streams - } - #[allow(dead_code)] async fn shutdown_services_with_timeout(&self) -> anyhow::Result<()> { tokio::time::timeout(GRACEFUL_SHUTDOWN_TIMEOUT, async { diff --git a/crates/fuel-streams-ws/src/server/http/handlers.rs b/crates/sv-webserver/src/server/http/handlers.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/http/handlers.rs rename to crates/sv-webserver/src/server/http/handlers.rs diff --git a/crates/fuel-streams-ws/src/server/http/mod.rs b/crates/sv-webserver/src/server/http/mod.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/http/mod.rs rename to crates/sv-webserver/src/server/http/mod.rs diff --git a/crates/fuel-streams-ws/src/server/http/models.rs b/crates/sv-webserver/src/server/http/models.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/http/models.rs rename to crates/sv-webserver/src/server/http/models.rs diff --git a/crates/fuel-streams-ws/src/server/middlewares/auth.rs b/crates/sv-webserver/src/server/middlewares/auth.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/middlewares/auth.rs rename to crates/sv-webserver/src/server/middlewares/auth.rs diff --git a/crates/fuel-streams-ws/src/server/middlewares/mod.rs b/crates/sv-webserver/src/server/middlewares/mod.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/middlewares/mod.rs rename to crates/sv-webserver/src/server/middlewares/mod.rs diff --git a/crates/fuel-streams-ws/src/server/mod.rs b/crates/sv-webserver/src/server/mod.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/mod.rs rename to crates/sv-webserver/src/server/mod.rs diff --git a/crates/fuel-streams-ws/src/server/state.rs b/crates/sv-webserver/src/server/state.rs similarity index 98% rename from crates/fuel-streams-ws/src/server/state.rs rename to crates/sv-webserver/src/server/state.rs index 8a2a3609..19e379d2 100644 --- a/crates/fuel-streams-ws/src/server/state.rs +++ b/crates/sv-webserver/src/server/state.rs @@ -4,11 +4,11 @@ use std::{ }; use async_nats::jetstream::stream::State; +use fuel_streams_core::prelude::FuelStreamsExt; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use super::context::Context; -use crate::server::ws::fuel_streams::FuelStreamsExt; #[derive(Debug, Serialize, Deserialize, Clone)] pub struct StreamInfo { diff --git a/crates/fuel-streams-ws/src/server/ws/errors.rs b/crates/sv-webserver/src/server/ws/errors.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/ws/errors.rs rename to crates/sv-webserver/src/server/ws/errors.rs diff --git a/crates/fuel-streams-ws/src/server/ws/mod.rs b/crates/sv-webserver/src/server/ws/mod.rs similarity index 74% rename from crates/fuel-streams-ws/src/server/ws/mod.rs rename to crates/sv-webserver/src/server/ws/mod.rs index c1f05088..bb1b9404 100644 --- a/crates/fuel-streams-ws/src/server/ws/mod.rs +++ b/crates/sv-webserver/src/server/ws/mod.rs @@ -1,5 +1,4 @@ pub mod errors; -pub mod fuel_streams; pub mod models; pub mod socket; pub mod state; diff --git a/crates/fuel-streams-ws/src/server/ws/models.rs b/crates/sv-webserver/src/server/ws/models.rs similarity index 98% rename from crates/fuel-streams-ws/src/server/ws/models.rs rename to crates/sv-webserver/src/server/ws/models.rs index 3033e5f8..0dfcec38 100644 --- a/crates/fuel-streams-ws/src/server/ws/models.rs +++ b/crates/sv-webserver/src/server/ws/models.rs @@ -1,4 +1,4 @@ -use fuel_streams_storage::DeliverPolicy as NatsDeliverPolicy; +use fuel_streams_nats::DeliverPolicy as NatsDeliverPolicy; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] diff --git a/crates/sv-webserver/src/server/ws/socket.rs b/crates/sv-webserver/src/server/ws/socket.rs new file mode 100644 index 00000000..03512ed7 --- /dev/null +++ b/crates/sv-webserver/src/server/ws/socket.rs @@ -0,0 +1,377 @@ +use std::sync::{atomic::AtomicUsize, Arc}; + +use actix_web::{ + web::{self, Bytes}, + HttpMessage, + HttpRequest, + Responder, +}; +use actix_ws::{Message, Session}; +use fuel_streams::{ + logs::Log, + types::{Block, Input, Output, Receipt, Transaction}, + utxos::Utxo, + StreamEncoder, + Streamable, +}; +use fuel_streams_core::prelude::*; +use fuel_streams_nats::DeliverPolicy; +use futures::StreamExt; +use uuid::Uuid; + +use super::{errors::WsSubscriptionError, models::ClientMessage}; +use crate::{ + server::{ + state::ServerState, + ws::models::{ServerMessage, SubscriptionPayload}, + }, + telemetry::Telemetry, +}; + +static _NEXT_USER_ID: AtomicUsize = AtomicUsize::new(1); + +pub async fn get_ws( + req: HttpRequest, + body: web::Payload, + state: web::Data, +) -> actix_web::Result { + // extract user id + let user_id = match req.extensions().get::() { + Some(user_id) => { + tracing::info!( + "Authenticated WebSocket connection for user: {:?}", + user_id.to_string() + ); + user_id.to_owned() + } + None => { + tracing::info!("Unauthenticated WebSocket connection"); + return Err(actix_web::error::ErrorUnauthorized( + "Missing or invalid JWT", + )); + } + }; + + // split the request into response, session, and message stream + let (response, session, mut msg_stream) = actix_ws::handle(&req, body)?; + + // record the new subscription + state.context.telemetry.increment_subscriptions_count(); + + // spawm an actor handling the ws connection + let streams = state.context.fuel_streams.clone(); + let telemetry = state.context.telemetry.clone(); + actix_web::rt::spawn(async move { + tracing::info!("Ws opened for user id {:?}", user_id.to_string()); + while let Some(Ok(msg)) = msg_stream.recv().await { + let mut session = session.clone(); + match msg { + Message::Ping(bytes) => { + tracing::info!("Received ping, {:?}", bytes); + if session.pong(&bytes).await.is_err() { + tracing::error!("Error sending pong, {:?}", bytes); + } + } + Message::Pong(bytes) => { + tracing::info!("Received pong, {:?}", bytes); + } + Message::Text(string) => { + tracing::info!("Received text, {string}"); + let bytes = Bytes::from(string.as_bytes().to_vec()); + let _ = handle_binary_message( + bytes, + user_id, + session, + Arc::clone(&telemetry), + Arc::clone(&streams), + ) + .await; + } + Message::Binary(bytes) => { + let _ = handle_binary_message( + bytes, + user_id, + session, + Arc::clone(&telemetry), + Arc::clone(&streams), + ) + .await; + } + Message::Close(reason) => { + tracing::info!( + "Got close event, terminating session with reason {:?}", + reason + ); + let reason_str = + reason.and_then(|r| r.description).unwrap_or_default(); + close_socket_with_error( + WsSubscriptionError::ClosedWithReason( + reason_str.to_string(), + ), + user_id, + session, + None, + telemetry, + ) + .await; + return; + } + _ => { + tracing::error!("Received unknown message type"); + close_socket_with_error( + WsSubscriptionError::ClosedWithReason( + "Unknown message type".to_string(), + ), + user_id, + session, + None, + telemetry, + ) + .await; + return; + } + }; + } + }); + + Ok(response) +} + +async fn handle_binary_message( + msg: Bytes, + user_id: uuid::Uuid, + mut session: Session, + telemetry: Arc, + streams: Arc, +) -> Result<(), WsSubscriptionError> { + tracing::info!("Received binary {:?}", msg); + let client_message = match parse_client_message(msg) { + Ok(msg) => msg, + Err(e) => { + close_socket_with_error(e, user_id, session, None, telemetry).await; + return Ok(()); + } + }; + + tracing::info!("Message parsed: {:?}", client_message); + // handle the client message + match client_message { + ClientMessage::Subscribe(payload) => { + tracing::info!("Received subscribe message: {:?}", payload); + let subject_wildcard = payload.wildcard; + let deliver_policy = payload.deliver_policy; + + // verify the subject name + let sub_subject = + match verify_and_extract_subject_name(&subject_wildcard) { + Ok(res) => res, + Err(e) => { + close_socket_with_error( + e, + user_id, + session, + Some(subject_wildcard.clone()), + telemetry, + ) + .await; + return Ok(()); + } + }; + + // start the streamer async + let mut stream_session = session.clone(); + + // reply to socket with subscription + send_message_to_socket( + &mut session, + ServerMessage::Subscribed(SubscriptionPayload { + wildcard: subject_wildcard.clone(), + deliver_policy, + }), + ) + .await; + + // receive streaming in a background thread + let streams = streams.clone(); + let telemetry = telemetry.clone(); + actix_web::rt::spawn(async move { + // update metrics + telemetry.update_user_subscription_metrics( + user_id, + &subject_wildcard, + ); + + // subscribe to the stream + let config = SubscriptionConfig { + deliver_policy: DeliverPolicy::All, + filter_subjects: vec![subject_wildcard.clone()], + }; + let mut sub = + match streams.subscribe(&sub_subject, Some(config)).await { + Ok(sub) => sub, + Err(e) => { + close_socket_with_error( + WsSubscriptionError::Stream(e), + user_id, + session, + Some(subject_wildcard.clone()), + telemetry, + ) + .await; + return; + } + }; + + // consume and forward to the ws + while let Some(s3_serialized_payload) = sub.next().await { + // decode and serialize back to ws payload + let serialized_ws_payload = match decode( + &subject_wildcard, + s3_serialized_payload, + ) + .await + { + Ok(res) => res, + Err(e) => { + telemetry.update_error_metrics( + &subject_wildcard, + &e.to_string(), + ); + tracing::error!("Error serializing received stream message: {:?}", e); + continue; + } + }; + + // send the payload over the stream + let _ = stream_session.binary(serialized_ws_payload).await; + } + }); + Ok(()) + } + ClientMessage::Unsubscribe(payload) => { + tracing::info!("Received unsubscribe message: {:?}", payload); + let subject_wildcard = payload.wildcard; + + let deliver_policy = payload.deliver_policy; + + if let Err(e) = verify_and_extract_subject_name(&subject_wildcard) { + close_socket_with_error( + e, + user_id, + session, + Some(subject_wildcard.clone()), + telemetry, + ) + .await; + return Ok(()); + } + + // TODO: implement session management for the same user_id + // send a message to the client to confirm unsubscribing + send_message_to_socket( + &mut session, + ServerMessage::Unsubscribed(SubscriptionPayload { + wildcard: subject_wildcard, + deliver_policy, + }), + ) + .await; + Ok(()) + } + } +} + +fn parse_client_message( + msg: Bytes, +) -> Result { + let msg = serde_json::from_slice::(&msg) + .map_err(WsSubscriptionError::UnparsablePayload)?; + Ok(msg) +} + +pub fn verify_and_extract_subject_name( + subject_wildcard: &str, +) -> Result { + let mut subject_parts = subject_wildcard.split('.'); + // TODO: more advanced checks here with Regex + if subject_parts.clone().count() == 1 { + return Err(WsSubscriptionError::UnsupportedWildcardPattern( + subject_wildcard.to_string(), + )); + } + let subject_name = subject_parts.next().unwrap_or_default(); + if !FuelStreamsUtils::is_within_subject_names(subject_name) { + return Err(WsSubscriptionError::UnknownSubjectName( + subject_wildcard.to_string(), + )); + } + Ok(subject_name.to_string()) +} + +async fn close_socket_with_error( + e: WsSubscriptionError, + user_id: uuid::Uuid, + mut session: Session, + subject_wildcard: Option, + telemetry: Arc, +) { + tracing::error!("ws subscription error: {:?}", e.to_string()); + if let Some(subject_wildcard) = subject_wildcard { + telemetry.update_error_metrics(&subject_wildcard, &e.to_string()); + telemetry.update_unsubscribed(user_id, &subject_wildcard); + } + telemetry.decrement_subscriptions_count(); + send_message_to_socket(&mut session, ServerMessage::Error(e.to_string())) + .await; + let _ = session.close(None).await; +} + +async fn send_message_to_socket(session: &mut Session, message: ServerMessage) { + let data = serde_json::to_vec(&message).ok().unwrap_or_default(); + let _ = session.binary(data).await; +} + +async fn decode( + subject_wildcard: &str, + s3_payload: Vec, +) -> Result, WsSubscriptionError> { + let subject = verify_and_extract_subject_name(subject_wildcard)?; + match subject.as_str() { + Transaction::NAME => { + let entity = Transaction::decode_or_panic(s3_payload); + serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload) + } + Block::NAME => { + let entity = Block::decode_or_panic(s3_payload); + serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload) + } + Input::NAME => { + let entity = Input::decode_or_panic(s3_payload); + serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload) + } + Output::NAME => { + let entity = Output::decode_or_panic(s3_payload); + serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload) + } + Receipt::NAME => { + let entity = Receipt::decode_or_panic(s3_payload); + serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload) + } + Utxo::NAME => { + let entity = Utxo::decode_or_panic(s3_payload); + serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload) + } + Log::NAME => { + let entity = Log::decode_or_panic(s3_payload); + serde_json::to_vec(&entity) + .map_err(WsSubscriptionError::UnparsablePayload) + } + _ => Err(WsSubscriptionError::UnknownSubjectName(subject.to_string())), + } +} diff --git a/crates/fuel-streams-ws/src/server/ws/state.rs b/crates/sv-webserver/src/server/ws/state.rs similarity index 100% rename from crates/fuel-streams-ws/src/server/ws/state.rs rename to crates/sv-webserver/src/server/ws/state.rs diff --git a/crates/fuel-streams-publisher/src/telemetry/elastic_search.rs b/crates/sv-webserver/src/telemetry/elastic_search.rs similarity index 100% rename from crates/fuel-streams-publisher/src/telemetry/elastic_search.rs rename to crates/sv-webserver/src/telemetry/elastic_search.rs diff --git a/crates/fuel-streams-ws/src/telemetry/metrics.rs b/crates/sv-webserver/src/telemetry/metrics.rs similarity index 100% rename from crates/fuel-streams-ws/src/telemetry/metrics.rs rename to crates/sv-webserver/src/telemetry/metrics.rs diff --git a/crates/fuel-streams-ws/src/telemetry/mod.rs b/crates/sv-webserver/src/telemetry/mod.rs similarity index 100% rename from crates/fuel-streams-ws/src/telemetry/mod.rs rename to crates/sv-webserver/src/telemetry/mod.rs diff --git a/crates/fuel-streams-publisher/src/telemetry/runtime.rs b/crates/sv-webserver/src/telemetry/runtime.rs similarity index 100% rename from crates/fuel-streams-publisher/src/telemetry/runtime.rs rename to crates/sv-webserver/src/telemetry/runtime.rs diff --git a/crates/fuel-streams-ws/src/telemetry/system.rs b/crates/sv-webserver/src/telemetry/system.rs similarity index 88% rename from crates/fuel-streams-ws/src/telemetry/system.rs rename to crates/sv-webserver/src/telemetry/system.rs index ec0f14eb..bae499a0 100644 --- a/crates/fuel-streams-ws/src/telemetry/system.rs +++ b/crates/sv-webserver/src/telemetry/system.rs @@ -572,26 +572,20 @@ mod tests { kernel_version: "kernel-version".to_string(), uptime: 123456, }, - disk: vec![( - PathBuf::from("disk1"), - Disk { - size: 1000, - free: 877, - usage: Decimal::new(1234, 2), - }, - )] + disk: vec![(PathBuf::from("disk1"), Disk { + size: 1000, + free: 877, + usage: Decimal::new(1234, 2), + })] .into_iter() .collect(), cpu_physical_core_count: 1, cpu_count: 1, - cpu: vec![( - 1, - Cpu { - name: "cpu1".to_string(), - frequency: 12345, - usage: Decimal::new(1234, 2), - }, - )] + cpu: vec![(1, Cpu { + name: "cpu1".to_string(), + frequency: 12345, + usage: Decimal::new(1234, 2), + })] .into_iter() .collect(), }, @@ -600,35 +594,32 @@ mod tests { let output = serde_prometheus::to_string(&metrics, None, &[]) .expect("prometheus"); - assert_eq!( - output.trim_end().split('\n').collect::>(), - vec![ - r#"system_application_pid 0"#, - r#"system_application_name{path = "process"} 1"#, - r#"system_application_cpu_usage 12.34"#, - r#"system_application_size{type = "memory"} 1000"#, - r#"system_application_free{type = "memory"} 877"#, - r#"system_application_usage{type = "memory"} 12.34"#, - r#"system_memory_size{type = "system"} 1000"#, - r#"system_memory_free{type = "system"} 877"#, - r#"system_memory_usage{type = "system"} 12.34"#, - r#"system_memory_size{type = "swap"} 1000"#, - r#"system_memory_free{type = "swap"} 877"#, - r#"system_memory_usage{type = "swap"} 12.34"#, - r#"system_load_average_1 1.2"#, - r#"system_load_average_5 2.3"#, - r#"system_load_average_15 3.4"#, - r#"system_host_os_version{path = "os-version"} 1"#, - r#"system_host_kernel_version{path = "kernel-version"} 1"#, - r#"system_host_uptime 123456"#, - r#"system_disk_size{path = "disk1"} 1000"#, - r#"system_disk_free{path = "disk1"} 877"#, - r#"system_disk_usage{path = "disk1"} 12.34"#, - r#"system_cpu_physical_core_count 1"#, - r#"system_cpu_count 1"#, - r#"system_cpu_frequency{id = "1"} 12345"#, - r#"system_cpu_usage{id = "1"} 12.34"#, - ] - ) + assert_eq!(output.trim_end().split('\n').collect::>(), vec![ + r#"system_application_pid 0"#, + r#"system_application_name{path = "process"} 1"#, + r#"system_application_cpu_usage 12.34"#, + r#"system_application_size{type = "memory"} 1000"#, + r#"system_application_free{type = "memory"} 877"#, + r#"system_application_usage{type = "memory"} 12.34"#, + r#"system_memory_size{type = "system"} 1000"#, + r#"system_memory_free{type = "system"} 877"#, + r#"system_memory_usage{type = "system"} 12.34"#, + r#"system_memory_size{type = "swap"} 1000"#, + r#"system_memory_free{type = "swap"} 877"#, + r#"system_memory_usage{type = "swap"} 12.34"#, + r#"system_load_average_1 1.2"#, + r#"system_load_average_5 2.3"#, + r#"system_load_average_15 3.4"#, + r#"system_host_os_version{path = "os-version"} 1"#, + r#"system_host_kernel_version{path = "kernel-version"} 1"#, + r#"system_host_uptime 123456"#, + r#"system_disk_size{path = "disk1"} 1000"#, + r#"system_disk_free{path = "disk1"} 877"#, + r#"system_disk_usage{path = "disk1"} 12.34"#, + r#"system_cpu_physical_core_count 1"#, + r#"system_cpu_count 1"#, + r#"system_cpu_frequency{id = "1"} 12345"#, + r#"system_cpu_usage{id = "1"} 12.34"#, + ]) } } diff --git a/examples/Cargo.toml b/examples/Cargo.toml index cad5e612..c9a435e3 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -8,8 +8,8 @@ edition = "2021" anyhow = { workspace = true } fuel-core-types = { workspace = true } fuel-streams = { workspace = true, features = ["test-helpers"] } -fuel-streams-ws = { workspace = true } futures = { workspace = true } +sv-webserver = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } [[example]] diff --git a/examples/multiple-streams.rs b/examples/multiple-streams.rs index 1bb1d4ca..7f5508f3 100644 --- a/examples/multiple-streams.rs +++ b/examples/multiple-streams.rs @@ -240,6 +240,7 @@ async fn stream_contract( ) -> anyhow::Result<()> { let mut receipt_stream = fuel_streams::Stream::::new(client).await; + // Set up filters for all receipt types that can be associated with a contract receipt_stream.with_filter( ReceiptsBurnSubject::new().with_contract_id(Some(contract_id.into())), ); @@ -275,21 +276,37 @@ async fn stream_contract( ); let mut sub = receipt_stream.subscribe_raw().await?; - while let Some(bytes) = sub.next().await { let decoded_msg = Receipt::decode_raw(bytes).unwrap(); let receipt = decoded_msg.payload; - // Check if the receipt has a contract_id and if it matches our target - if let Some(receipt_contract_id) = &receipt.contract_id { - if *receipt_contract_id == contract_id.into() { - let receipt_subject = decoded_msg.subject; - let receipt_published_at = decoded_msg.timestamp; - println!( - "Received contract receipt: data={:?}, subject={}, published_at={}", - receipt, receipt_subject, receipt_published_at - ); + // Check if this is a contract-related receipt and matches our target + let should_process = match &receipt { + Receipt::Call(r) => { + r.id == contract_id.into() || r.to == contract_id.into() + } + Receipt::Return(r) => r.id == contract_id.into(), + Receipt::ReturnData(r) => r.id == contract_id.into(), + Receipt::Panic(r) => r.id == contract_id.into(), + Receipt::Revert(r) => r.id == contract_id.into(), + Receipt::Log(r) => r.id == contract_id.into(), + Receipt::LogData(r) => r.id == contract_id.into(), + Receipt::Transfer(r) => { + r.id == contract_id.into() || r.to == contract_id.into() } + Receipt::TransferOut(r) => r.id == contract_id.into(), + Receipt::Mint(r) => r.contract_id == contract_id.into(), + Receipt::Burn(r) => r.contract_id == contract_id.into(), + Receipt::ScriptResult(_) | Receipt::MessageOut(_) => false, + }; + + if should_process { + let receipt_subject = decoded_msg.subject; + let receipt_published_at = decoded_msg.timestamp; + println!( + "Received contract receipt: data={:?}, subject={}, published_at={}", + receipt, receipt_subject, receipt_published_at + ); } } diff --git a/examples/websockets.rs b/examples/websockets.rs index c05c8206..c8de232a 100755 --- a/examples/websockets.rs +++ b/examples/websockets.rs @@ -18,7 +18,7 @@ use fuel_streams::{ subjects::SubjectBuildable, types::FuelNetwork, }; -use fuel_streams_ws::{ +use sv_webserver::{ client::WebSocketClient, server::ws::models::DeliverPolicy, }; diff --git a/knope.toml b/knope.toml index 6f7f5d2d..57f4fc54 100644 --- a/knope.toml +++ b/knope.toml @@ -1,5 +1,5 @@ # ------------------------------------------------------------ -# Fuel-streams-publisher package +# Fuel-streams package # ------------------------------------------------------------ [packages.fuel-streams] versioned_files = ["crates/fuel-streams/Cargo.toml"] @@ -19,63 +19,6 @@ extra_changelog_sections = [ ], name = "📝 Notes" }, ] -[[packages.fuel-streams.assets]] -path = "artifacts/fuel-streams-publisher-Linux-aarch64-gnu.tar.gz" - -[[packages.fuel-streams.assets]] -path = "artifacts/fuel-streams-publisher-Linux-aarch64-musl.tar.gz" - -[[packages.fuel-streams.assets]] -path = "artifacts/fuel-streams-publisher-Linux-x86_64-gnu.tar.gz" - -[[packages.fuel-streams.assets]] -path = "artifacts/fuel-streams-publisher-Linux-x86_64-musl.tar.gz" - -[[packages.fuel-streams.assets]] -path = "artifacts/fuel-streams-publisher-macOS-aarch64.tar.gz" - -[[packages.fuel-streams.assets]] -path = "artifacts/fuel-streams-publisher-macOS-x86_64.tar.gz" - -# ------------------------------------------------------------ -# Fuel-streams-ws package -# ------------------------------------------------------------ -[packages.fuel-streams-ws] -versioned_files = ["crates/fuel-streams-ws/Cargo.toml"] -changelog = "CHANGELOG.md" -extra_changelog_sections = [ - { types = [ - "major", - ], name = "⚠️ Breaking Change" }, - { types = [ - "minor", - ], name = "🚀 Features" }, - { types = [ - "patch", - ], name = "🐛 Fixes" }, - { footers = [ - "Changelog-Note", - ], name = "📝 Notes" }, -] - -[[packages.fuel-streams-ws.assets]] -path = "artifacts/fuel-streams-ws-Linux-aarch64-gnu.tar.gz" - -[[packages.fuel-streams-ws.assets]] -path = "artifacts/fuel-streams-ws-Linux-aarch64-musl.tar.gz" - -[[packages.fuel-streams-ws.assets]] -path = "artifacts/fuel-streams-ws-Linux-x86_64-gnu.tar.gz" - -[[packages.fuel-streams-ws.assets]] -path = "artifacts/fuel-streams-ws-Linux-x86_64-musl.tar.gz" - -[[packages.fuel-streams-ws.assets]] -path = "artifacts/fuel-streams-ws-macOS-aarch64.tar.gz" - -[[packages.fuel-streams-ws.assets]] -path = "artifacts/fuel-streams-ws-macOS-x86_64.tar.gz" - # ------------------------------------------------------------ # Workflow to get the current version # ------------------------------------------------------------ diff --git a/scripts/run_publisher.sh b/scripts/run_publisher.sh index c2170dd8..3ffa1868 100755 --- a/scripts/run_publisher.sh +++ b/scripts/run_publisher.sh @@ -28,6 +28,12 @@ usage() { exit 1 } +# Set default values from environment variables with fallbacks +NETWORK=${NETWORK:-"testnet"} +MODE=${MODE:-"profiling"} +PORT=${PORT:-"4004"} +TELEMETRY_PORT=${TELEMETRY_PORT:-"8080"} + while [[ "$#" -gt 0 ]]; do case $1 in --network) @@ -63,7 +69,7 @@ done # ------------------------------ # Load Environment # ------------------------------ -source ./scripts/set_env.sh +source ./scripts/set_env.sh NETWORK=${NETWORK} # Print the configuration being used echo -e "\n==========================================" @@ -94,30 +100,35 @@ echo -e "==========================================\n" # Define common arguments COMMON_ARGS=( "--enable-relayer" + "--service-name" "fuel-${NETWORK}-node" "--keypair" "${KEYPAIR}" "--relayer" "${RELAYER}" "--ip=0.0.0.0" - "--service-name" "fuel-${NETWORK}-node" - "--db-path" "./cluster/docker/db-${NETWORK}" - "--snapshot" "./cluster/chain-config/${NETWORK}" "--port" "${PORT}" - "--telemetry-port" "${TELEMETRY_PORT}" "--peering-port" "30333" + "--db-path" "./cluster/docker/db-${NETWORK}" + "--snapshot" "./cluster/chain-config/${NETWORK}" "--utxo-validation" "--poa-instant" "false" "--enable-p2p" - "--sync-header-batch-size" "${SYNC_HEADER_BATCH_SIZE}" - "--relayer-log-page-size=${RELAYER_LOG_PAGE_SIZE}" - "--sync-block-stream-buffer-size" "30" - "--bootstrap-nodes" "${RESERVED_NODES}" + "--reserved-nodes" "${RESERVED_NODES}" "--relayer-v2-listening-contracts=${RELAYER_V2_LISTENING_CONTRACTS}" "--relayer-da-deploy-height=${RELAYER_DA_DEPLOY_HEIGHT}" + "--relayer-log-page-size=${RELAYER_LOG_PAGE_SIZE}" + "--sync-block-stream-buffer-size" "50" + "--max-database-cache-size" "17179869184" + "--state-rewind-duration" "136y" + "--request-timeout" "60" + "--graphql-max-complexity" "1000000000" + # Application specific + "--nats-url" "nats://localhost:4222" + # "--telemetry-port" "${TELEMETRY_PORT}" ) # Execute based on mode if [ "$MODE" == "dev" ]; then - cargo run -p fuel-streams-publisher -- "${COMMON_ARGS[@]}" ${EXTRA_ARGS} + cargo run -p sv-publisher -- "${COMMON_ARGS[@]}" ${EXTRA_ARGS} else - cargo build --profile profiling --package fuel-streams-publisher - samply record ./target/profiling/fuel-streams-publisher "${COMMON_ARGS[@]}" ${EXTRA_ARGS} + cargo build --profile profiling --package sv-publisher + samply record ./target/profiling/sv-publisher "${COMMON_ARGS[@]}" ${EXTRA_ARGS} fi diff --git a/scripts/run_streamer.sh b/scripts/run_streamer.sh deleted file mode 100755 index c2a109a6..00000000 --- a/scripts/run_streamer.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# Exit immediately if a command exits with a non-zero status -set -e - -# ------------------------------ -# Function to Display Usage -# ------------------------------ -usage() { - echo "Usage: $0 [options]" - echo "Options:" - echo " --mode : Specify the run mode (dev|profiling)" - echo " --extra-args : Optional additional arguments to append (in quotes)" - echo "" - echo "Examples:" - echo " $0 # Runs with all defaults" - echo " $0 --mode dev # Runs with dev mode" - echo " $0 --mode dev # Custom config toml path and mode" - exit 1 -} - -while [[ "$#" -gt 0 ]]; do - case $1 in - --mode) - MODE="$2" - shift 2 - ;; - --extra-args) - EXTRA_ARGS="$2" - shift 2 - ;; - --help) - usage - ;; - *) - echo "Error: Unknown parameter passed: $1" >&2 - usage - ;; - esac -done - -# ------------------------------ -# Load Environment -# ------------------------------ -source ./scripts/set_env.sh - -# Print the configuration being used -echo -e "\n==========================================" -echo "⚙️ Configuration" -echo -e "==========================================" - -# Runtime Configuration -echo "Runtime Settings:" -echo "→ Mode: $MODE" -if [ -n "$EXTRA_ARGS" ]; then - echo "→ Extra Arguments: $EXTRA_ARGS" -fi - -# Environment Variables -echo -e "\nEnvironment Variables:" -echo " → Use Metrics: ${USE_METRICS}" -echo " → Use Elastic Logging: $USE_ELASTIC_LOGGING" -echo " → AWS S3 Enabled: $AWS_S3_ENABLED" -echo " → AWS Access Key Id: $AWS_ACCESS_KEY_ID" -echo " → AWS Secret Access Key: $AWS_SECRET_ACCESS_KEY" -echo " → AWS Region: $AWS_REGION" -echo " → AWS Bucket: $AWS_S3_BUCKET_NAME" -echo " → AWS Endpoint: $AWS_ENDPOINT_URL" -echo " → Jwt Auth Secret: $JWT_AUTH_SECRET" -echo " → Nats Url: $NATS_URL" -echo -e "==========================================\n" - -# Define common arguments -COMMON_ARGS=() - -# Execute based on mode -if [ "$MODE" == "dev" ]; then - cargo run -p fuel-streams-ws -- "${COMMON_ARGS[@]}" ${EXTRA_ARGS} -else - cargo build --profile profiling --package fuel-streams-ws - samply record ./target/profiling/fuel-streams-ws "${COMMON_ARGS[@]}" ${EXTRA_ARGS} -fi diff --git a/scripts/run_webserver.sh b/scripts/run_webserver.sh new file mode 100755 index 00000000..91845a8a --- /dev/null +++ b/scripts/run_webserver.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status +set -e + +# ------------------------------ +# Function to Display Usage +# ------------------------------ +usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " --mode : Specify the run mode (dev|profiling)" + echo " --port : Port number for the API server (default: 9003)" + echo " --nats-url : NATS URL (default: nats://localhost:4222)" + echo " --extra-args : Optional additional arguments to append (in quotes)" + echo "" + echo "Examples:" + echo " $0 # Runs with all defaults" + echo " $0 --mode dev --port 8080 # Custom port" + echo " $0 --mode dev --extra-args '\"--use-metrics\"' # Enable metrics" + exit 1 +} + +while [[ "$#" -gt 0 ]]; do + case $1 in + --mode) + MODE="$2" + shift 2 + ;; + --port) + PORT="$2" + shift 2 + ;; + --nats-url) + NATS_URL="$2" + shift 2 + ;; + --extra-args) + EXTRA_ARGS="$2" + shift 2 + ;; + --help) + usage + ;; + *) + echo "Error: Unknown parameter passed: $1" >&2 + usage + ;; + esac +done + +# Load environment variables with defaults +PORT=${PORT:-9003} +NATS_URL=${NATS_URL:-nats://localhost:4222} +MODE=${MODE:-dev} +EXTRA_ARGS=${EXTRA_ARGS:-""} + +# ------------------------------ +# Load Environment +# ------------------------------ +source ./scripts/set_env.sh + +# Print the configuration being used +echo -e "\n==========================================" +echo "⚙️ Configuration" +echo -e "==========================================" + +# Runtime Configuration +echo "Runtime Settings:" +echo "→ Mode: ${MODE:-dev}" +echo "→ API Port: ${PORT:-9003}" +echo "→ NATS URL: ${NATS_URL:-"nats://localhost:4222"}" +if [ -n "$EXTRA_ARGS" ]; then + echo "→ Extra Arguments: $EXTRA_ARGS" +fi + +echo -e "==========================================\n" + +# Define common arguments +COMMON_ARGS=( + "--port" "${PORT:-9003}" + "--nats-url" "${NATS_URL:-"nats://localhost:4222"}" +) + +# Execute based on mode +if [ "${MODE:-dev}" == "dev" ]; then + cargo run -p sv-webserver -- "${COMMON_ARGS[@]}" ${EXTRA_ARGS} +else + cargo build --profile profiling --package sv-webserver + samply record ./target/profiling/sv-webserver "${COMMON_ARGS[@]}" ${EXTRA_ARGS} +fi diff --git a/scripts/set_env.sh b/scripts/set_env.sh index da21ea07..36efa113 100755 --- a/scripts/set_env.sh +++ b/scripts/set_env.sh @@ -40,13 +40,14 @@ cleanup_env() { fi } -# Load initial environment -load_env - # Clean up previous auto-generated content cleanup_env +# Load initial environment +load_env + # Set and export network-specific variables +export NETWORK=$NETWORK export RESERVED_NODES=$(eval echo "\$${NETWORK_UPPER}_RESERVED_NODES") export RELAYER_V2_LISTENING_CONTRACTS=$(eval echo "\$${NETWORK_UPPER}_RELAYER_V2_LISTENING_CONTRACTS") export RELAYER_DA_DEPLOY_HEIGHT=$(eval echo "\$${NETWORK_UPPER}_RELAYER_DA_DEPLOY_HEIGHT") @@ -54,17 +55,6 @@ export RELAYER=$(eval echo "\$${NETWORK_UPPER}_RELAYER") export SYNC_HEADER_BATCH_SIZE=$(eval echo "\$${NETWORK_UPPER}_SYNC_HEADER_BATCH_SIZE") export RELAYER_LOG_PAGE_SIZE=$(eval echo "\$${NETWORK_UPPER}_RELAYER_LOG_PAGE_SIZE") export CHAIN_CONFIG=$NETWORK -export NETWORK=$NETWORK -export USE_METRICS="$(echo "$USE_METRICS")" -export USE_ELASTIC_LOGGING="$(echo "$USE_ELASTIC_LOGGING")" -export AWS_S3_ENABLED="$(echo "$AWS_S3_ENABLED")" -export AWS_ACCESS_KEY_ID="$(echo "$AWS_ACCESS_KEY_ID")" -export AWS_SECRET_ACCESS_KEY="$(echo "$AWS_SECRET_ACCESS_KEY")" -export AWS_REGION="$(echo "$AWS_REGION")" -export AWS_ENDPOINT_URL="$(echo "$AWS_ENDPOINT_URL")" -export AWS_S3_BUCKET_NAME="$(echo "$AWS_S3_BUCKET_NAME")" -export JWT_AUTH_SECRET="$(echo "$JWT_AUTH_SECRET")" -export NATS_URL="$(echo "$NATS_URL")" # Append network-specific variables to .env file { @@ -79,14 +69,4 @@ export NATS_URL="$(echo "$NATS_URL")" echo "SYNC_HEADER_BATCH_SIZE=$SYNC_HEADER_BATCH_SIZE" echo "RELAYER_LOG_PAGE_SIZE=$RELAYER_LOG_PAGE_SIZE" echo "CHAIN_CONFIG=$CHAIN_CONFIG" - echo "USE_METRICS=$USE_METRICS" - echo "USE_ELASTIC_LOGGING=$USE_ELASTIC_LOGGING" - echo "AWS_S3_ENABLED=$AWS_S3_ENABLED" - echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" - echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" - echo "AWS_REGION=$AWS_REGION" - echo "AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL" - echo "AWS_S3_BUCKET_NAME=$AWS_S3_BUCKET_NAME" - echo "JWT_AUTH_SECRET=$JWT_AUTH_SECRET" - echo "NATS_URL=$NATS_URL" } >> .env diff --git a/tarpaulin.toml b/tarpaulin.toml index 1bc0cf6d..ce0c18cd 100644 --- a/tarpaulin.toml +++ b/tarpaulin.toml @@ -63,10 +63,10 @@ engine = "Llvm" # ========================================== # ignore due to wasm incompatibility -# [cov_fuel_streams_publisher] +# [cov_sv_publisher] # name = "Fuel Streams Publisher Coverage Analysis" # packages = [ -# "fuel-streams-publisher" +# "sv-publisher" # ] # all-features = true # run-types = [ diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 9f1c81b8..747eb520 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -22,14 +22,9 @@ name = "special_integration_tests" path = "src/main.rs" [dependencies] -anyhow = { workspace = true } -async-trait = { workspace = true } fuel-core = { workspace = true, features = ["test-helpers"] } -fuel-core-importer = { workspace = true, features = ["test-helpers"] } -fuel-core-types = { workspace = true } fuel-streams = { workspace = true, features = ["test-helpers"] } fuel-streams-core = { workspace = true, features = ["test-helpers"] } -fuel-streams-publisher = { workspace = true, features = ["test-helpers"] } futures = { workspace = true } rand = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } diff --git a/tests/tests/client.rs b/tests/tests/client.rs index a0178a7c..29ffcb5d 100644 --- a/tests/tests/client.rs +++ b/tests/tests/client.rs @@ -73,7 +73,9 @@ async fn multiple_client_connections() -> BoxedResult<()> { #[tokio::test] async fn public_user_cannot_create_streams() -> BoxedResult<()> { - let opts = NatsClientOpts::new(FuelNetwork::Local) + let network = FuelNetwork::Local; + let opts = NatsClientOpts::public_opts() + .with_url(network.to_nats_url()) .with_rdn_namespace() .with_timeout(1); let client = NatsClient::connect(&opts).await?; @@ -95,7 +97,9 @@ async fn public_user_cannot_create_streams() -> BoxedResult<()> { #[tokio::test] async fn public_user_cannot_create_stores() -> BoxedResult<()> { - let opts = NatsClientOpts::new(FuelNetwork::Local) + let network = FuelNetwork::Local; + let opts = NatsClientOpts::public_opts() + .with_url(network.to_nats_url()) .with_rdn_namespace() .with_timeout(1); @@ -116,7 +120,9 @@ async fn public_user_cannot_create_stores() -> BoxedResult<()> { #[tokio::test] async fn public_user_cannot_delete_stores() -> BoxedResult<()> { + let network = FuelNetwork::Local; let opts = NatsClientOpts::admin_opts() + .with_url(network.to_nats_url()) .with_rdn_namespace() .with_timeout(1); @@ -131,7 +137,8 @@ async fn public_user_cannot_delete_stores() -> BoxedResult<()> { }) .await?; - let opts = NatsClientOpts::new(FuelNetwork::Local) + let opts = NatsClientOpts::public_opts() + .with_url(network.to_nats_url()) .with_rdn_namespace() .with_timeout(1); let client = NatsClient::connect(&opts).await?; @@ -164,7 +171,9 @@ async fn public_user_cannot_delete_stream() -> BoxedResult<()> { }) .await?; - let public_opts = opts.clone().with_role(FuelNetworkUserRole::Default); + let network = FuelNetwork::Local; + let public_opts = + NatsClientOpts::public_opts().with_url(network.to_nats_url()); let public_client = NatsClient::connect(&public_opts).await?; assert!( @@ -181,21 +190,29 @@ async fn public_user_cannot_delete_stream() -> BoxedResult<()> { #[tokio::test] async fn public_user_can_access_streams_after_created() { - let opts = NatsClientOpts::new(FuelNetwork::Local) + let network = FuelNetwork::Local; + let admin_opts = NatsClientOpts::admin_opts() + .with_url(network.to_nats_url()) .with_rdn_namespace() .with_timeout(1); - let admin_opts = opts.clone().with_role(FuelNetworkUserRole::Admin); - assert!(NatsClient::connect(&admin_opts).await.is_ok()); + let public_opts = NatsClientOpts::public_opts() + .with_url(network.to_nats_url()) + .with_rdn_namespace() + .with_timeout(1); - let public_opts = opts.clone().with_role(FuelNetworkUserRole::Default); + assert!(NatsClient::connect(&admin_opts).await.is_ok()); assert!(NatsClient::connect(&public_opts).await.is_ok()); } #[tokio::test] async fn public_and_admin_user_can_access_streams_after_created( ) -> BoxedResult<()> { - let admin_opts = NatsClientOpts::admin_opts(); + let network = FuelNetwork::Local; + let admin_opts = NatsClientOpts::admin_opts() + .with_url(network.to_nats_url()) + .with_rdn_namespace() + .with_timeout(1); let s3_opts = Arc::new(S3ClientOpts::admin_opts()); let admin_tasks: Vec>> = (0..100) .map(|_| { @@ -210,8 +227,12 @@ async fn public_and_admin_user_can_access_streams_after_created( }) .collect(); - let public_opts = NatsClientOpts::new(FuelNetwork::Local); - let s3_public_opts = Arc::new(S3ClientOpts::new(FuelNetwork::Local)); + let public_opts = NatsClientOpts::public_opts() + .with_url(network.to_nats_url()) + .with_rdn_namespace() + .with_timeout(1); + let s3_public_opts = + Arc::new(S3ClientOpts::new(S3Env::Local, S3Role::Public)); let public_tasks: Vec>> = (0..100) .map(|_| { let opts: NatsClientOpts = public_opts.clone(); diff --git a/tests/tests/publisher.rs b/tests/tests/publisher.rs index ecd8bcf6..19eb20a2 100644 --- a/tests/tests/publisher.rs +++ b/tests/tests/publisher.rs @@ -1,346 +1,416 @@ -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; - -use fuel_core::combined_database::CombinedDatabase; -use fuel_core_importer::ImporterResult; -use fuel_core_types::blockchain::SealedBlock; -use fuel_streams_core::prelude::*; -use fuel_streams_publisher::{ - publisher::shutdown::ShutdownController, - shutdown::get_controller_and_token, - FuelCoreLike, - Publisher, -}; -use futures::StreamExt; -use tokio::sync::broadcast::{self, Receiver, Sender}; - -// TODO - Re-implement with `mockall` and `mock` macros -struct TestFuelCore { - chain_id: FuelCoreChainId, - base_asset_id: FuelCoreAssetId, - database: CombinedDatabase, - blocks_broadcaster: Sender, - receipts: Option>, -} - -impl TestFuelCore { - fn default( - blocks_broadcaster: Sender, - ) -> Self { - Self { - chain_id: FuelCoreChainId::default(), - base_asset_id: FuelCoreAssetId::zeroed(), - database: CombinedDatabase::default(), - blocks_broadcaster, - receipts: None, - } - } - fn with_receipts(mut self, receipts: Vec) -> Self { - self.receipts = Some(receipts); - self - } - fn arc(self) -> Arc { - Arc::new(self) - } -} - -#[async_trait::async_trait] -impl FuelCoreLike for TestFuelCore { - async fn start(&self) -> anyhow::Result<()> { - Ok(()) - } - fn is_started(&self) -> bool { - true - } - async fn await_synced_at_least_once( - &self, - _historical: bool, - ) -> anyhow::Result<()> { - Ok(()) - } - async fn stop(&self) {} - - async fn await_offchain_db_sync( - &self, - _block_id: &FuelCoreBlockId, - ) -> anyhow::Result<()> { - Ok(()) - } - - fn base_asset_id(&self) -> &FuelCoreAssetId { - &self.base_asset_id - } - fn chain_id(&self) -> &FuelCoreChainId { - &self.chain_id - } - - fn database(&self) -> &CombinedDatabase { - &self.database - } - - fn blocks_subscription( - &self, - ) -> Receiver { - self.blocks_broadcaster.subscribe() - } - - fn get_receipts( - &self, - _tx_id: &FuelCoreBytes32, - ) -> anyhow::Result>> { - Ok(self.receipts.clone()) - } -} - -#[tokio::test(flavor = "multi_thread")] -async fn doesnt_publish_any_message_when_no_block_has_been_mined() { - let (blocks_broadcaster, _) = broadcast::channel::(1); - let s3_client = Arc::new(S3Client::new_for_testing().await); - let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; - - let shutdown_controller = start_publisher(&publisher).await; - stop_publisher(shutdown_controller).await; - - assert!(publisher.get_fuel_streams().is_empty().await); -} - -#[tokio::test(flavor = "multi_thread")] -async fn publishes_a_block_message_when_a_single_block_has_been_mined() { - let (blocks_broadcaster, _) = broadcast::channel::(1); - let s3_client = Arc::new(S3Client::new_for_testing().await); - let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; - - publish_block(&publisher, &blocks_broadcaster).await; - - assert!(publisher - .get_fuel_streams() - .blocks() - .get_last_published(BlocksSubject::WILDCARD) - .await - .is_ok_and(|result| result.is_some())); - s3_client.cleanup_after_testing().await; -} - -#[tokio::test(flavor = "multi_thread")] -async fn publishes_transaction_for_each_published_block() { - let (blocks_broadcaster, _) = broadcast::channel::(1); - let s3_client = Arc::new(S3Client::new_for_testing().await); - let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; - - publish_block(&publisher, &blocks_broadcaster).await; - - assert!(publisher - .get_fuel_streams() - .transactions() - .get_last_published(TransactionsSubject::WILDCARD) - .await - .is_ok_and(|result| result.is_some())); - s3_client.cleanup_after_testing().await; -} - -#[tokio::test(flavor = "multi_thread")] -async fn publishes_receipts() { - let (blocks_broadcaster, _) = broadcast::channel::(1); - - let receipts = [ - FuelCoreReceipt::Call { - id: FuelCoreContractId::default(), - to: Default::default(), - amount: 0, - asset_id: Default::default(), - gas: 0, - param1: 0, - param2: 0, - pc: 0, - is: 0, - }, - FuelCoreReceipt::Return { - id: FuelCoreContractId::default(), - val: 0, - pc: 0, - is: 0, - }, - FuelCoreReceipt::ReturnData { - id: FuelCoreContractId::default(), - ptr: 0, - len: 0, - digest: FuelCoreBytes32::default(), - pc: 0, - is: 0, - data: None, - }, - FuelCoreReceipt::Revert { - id: FuelCoreContractId::default(), - ra: 0, - pc: 0, - is: 0, - }, - FuelCoreReceipt::Log { - id: FuelCoreContractId::default(), - ra: 0, - rb: 0, - rc: 0, - rd: 0, - pc: 0, - is: 0, - }, - FuelCoreReceipt::LogData { - id: FuelCoreContractId::default(), - ra: 0, - rb: 0, - ptr: 0, - len: 0, - digest: FuelCoreBytes32::default(), - pc: 0, - is: 0, - data: None, - }, - FuelCoreReceipt::Transfer { - id: FuelCoreContractId::default(), - to: FuelCoreContractId::default(), - amount: 0, - asset_id: FuelCoreAssetId::default(), - pc: 0, - is: 0, - }, - FuelCoreReceipt::TransferOut { - id: FuelCoreContractId::default(), - to: FuelCoreAddress::default(), - amount: 0, - asset_id: FuelCoreAssetId::default(), - pc: 0, - is: 0, - }, - FuelCoreReceipt::Mint { - sub_id: FuelCoreBytes32::default(), - contract_id: FuelCoreContractId::default(), - val: 0, - pc: 0, - is: 0, - }, - FuelCoreReceipt::Burn { - sub_id: FuelCoreBytes32::default(), - contract_id: FuelCoreContractId::default(), - val: 0, - pc: 0, - is: 0, - }, - ]; - - let fuel_core = TestFuelCore::default(blocks_broadcaster.clone()) - .with_receipts(receipts.to_vec()) - .arc(); - - let s3_client = Arc::new(S3Client::new_for_testing().await); - let publisher = - Publisher::new_for_testing(&nats_client().await, &s3_client, fuel_core) - .await - .unwrap(); - - publish_block(&publisher, &blocks_broadcaster).await; - - let mut receipts_stream = publisher - .get_fuel_streams() - .receipts() - .catchup(10) - .await - .unwrap(); - - let receipts: HashSet = receipts.iter().map(Into::into).collect(); - while let Some(Some(receipt)) = receipts_stream.next().await { - assert!(receipts.contains(&receipt)); - } - - s3_client.cleanup_after_testing().await; -} - -#[tokio::test(flavor = "multi_thread")] -async fn publishes_inputs() { - let (blocks_broadcaster, _) = broadcast::channel::(1); - let s3_client = Arc::new(S3Client::new_for_testing().await); - let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; - - publish_block(&publisher, &blocks_broadcaster).await; - - assert!(publisher - .get_fuel_streams() - .inputs() - .get_last_published(InputsByIdSubject::WILDCARD) - .await - .is_ok_and(|result| result.is_some())); - s3_client.cleanup_after_testing().await; -} - -async fn new_publisher( - broadcaster: Sender, - s3_client: &Arc, -) -> Publisher { - let fuel_core = TestFuelCore::default(broadcaster).arc(); - Publisher::new_for_testing(&nats_client().await, s3_client, fuel_core) - .await - .unwrap() -} - -async fn publish_block( - publisher: &Publisher, - blocks_broadcaster: &Sender, -) { - let shutdown_controller = start_publisher(publisher).await; - send_block(blocks_broadcaster); - stop_publisher(shutdown_controller).await; -} - -async fn start_publisher(publisher: &Publisher) -> ShutdownController { - let (shutdown_controller, shutdown_token) = get_controller_and_token(); - tokio::spawn({ - let publisher = publisher.clone(); - async move { - publisher.run(shutdown_token, true).await.unwrap(); - } - }); - wait_for_publisher_to_start().await; - shutdown_controller -} -async fn stop_publisher(shutdown_controller: ShutdownController) { - wait_for_publisher_to_process_block().await; - - assert!(shutdown_controller.initiate_shutdown().is_ok()); -} - -async fn wait_for_publisher_to_start() { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; -} -async fn wait_for_publisher_to_process_block() { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; -} - -fn send_block(broadcaster: &Sender) { - let block = create_test_block(); - assert!(broadcaster.send(block).is_ok()); -} -fn create_test_block() -> ImporterResult { - let mut block_entity = FuelCoreBlock::default(); - let tx = FuelCoreTransaction::default_test_tx(); - - *block_entity.transactions_mut() = vec![tx]; - - ImporterResult { - shared_result: Arc::new(FuelCoreImportResult { - sealed_block: SealedBlock { - entity: block_entity, - ..Default::default() - }, - ..Default::default() - }), - changes: Arc::new(HashMap::new()), - } -} - -async fn nats_client() -> NatsClient { - let opts = NatsClientOpts::admin_opts().with_rdn_namespace(); - NatsClient::connect(&opts) - .await - .expect("NATS connection failed") -} +// use std::{collections::HashMap, sync::Arc}; + +// use fuel_core::{ +// combined_database::CombinedDatabase, +// service::{Config, FuelService}, +// ShutdownListener, +// }; +// use fuel_core_importer::ImporterResult; +// use fuel_core_types::blockchain::SealedBlock; +// use fuel_streams_core::prelude::*; +// use tokio::sync::broadcast::{self, Receiver, Sender}; + +// // TODO - Re-implement with `mockall` and `mock` macros +// struct TestFuelCore { +// fuel_service: FuelService, +// chain_id: FuelCoreChainId, +// base_asset_id: FuelCoreAssetId, +// database: CombinedDatabase, +// blocks_broadcaster: Sender, +// receipts: Option>, +// } + +// impl TestFuelCore { +// fn default( +// blocks_broadcaster: Sender, +// ) -> Self { +// let mut shutdown = ShutdownListener::spawn(); +// let service = FuelService::new( +// Default::default(), +// Config::local_node(), +// &mut shutdown, +// ) +// .unwrap(); +// Self { +// fuel_service: service, +// chain_id: FuelCoreChainId::default(), +// base_asset_id: FuelCoreAssetId::zeroed(), +// database: CombinedDatabase::default(), +// blocks_broadcaster, +// receipts: None, +// } +// } +// fn with_receipts(mut self, receipts: Vec) -> Self { +// self.receipts = Some(receipts); +// self +// } +// fn arc(self) -> Arc { +// Arc::new(self) +// } +// } + +// #[async_trait::async_trait] +// impl FuelCoreLike for TestFuelCore { +// async fn start(&self) -> anyhow::Result<()> { +// Ok(()) +// } +// fn is_started(&self) -> bool { +// true +// } +// fn fuel_service(&self) -> &FuelService { +// &self.fuel_service +// } +// async fn await_synced_at_least_once( +// &self, +// _historical: bool, +// ) -> anyhow::Result<()> { +// Ok(()) +// } +// async fn stop(&self) {} + +// async fn await_offchain_db_sync( +// &self, +// _block_id: &FuelCoreBlockId, +// ) -> anyhow::Result<()> { +// Ok(()) +// } + +// fn base_asset_id(&self) -> &FuelCoreAssetId { +// &self.base_asset_id +// } +// fn chain_id(&self) -> &FuelCoreChainId { +// &self.chain_id +// } + +// fn database(&self) -> &CombinedDatabase { +// &self.database +// } + +// fn blocks_subscription( +// &self, +// ) -> Receiver { +// self.blocks_broadcaster.subscribe() +// } + +// fn get_receipts( +// &self, +// _tx_id: &FuelCoreBytes32, +// ) -> anyhow::Result>> { +// Ok(self.receipts.clone()) +// } + +// fn get_tx_status( +// &self, +// _tx_id: &FuelCoreBytes32, +// ) -> anyhow::Result> { +// Ok(Some(FuelCoreTransactionStatus::Success { +// receipts: self.receipts.clone().unwrap_or_default(), +// block_height: 0.into(), +// result: None, +// time: FuelCoreTai64::now(), +// total_gas: 0, +// total_fee: 0, +// })) +// } +// } + +// #[tokio::test(flavor = "multi_thread")] +// async fn doesnt_publish_any_message_when_no_block_has_been_mined() { +// let (blocks_broadcaster, _) = broadcast::channel::(1); +// let s3_client = Arc::new(S3Client::new_for_testing().await); +// let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; + +// let shutdown_controller = start_publisher(&publisher).await; +// stop_publisher(shutdown_controller).await; + +// assert!(publisher.get_fuel_streams().is_empty().await); +// } + +// #[tokio::test(flavor = "multi_thread")] +// async fn publishes_a_block_message_when_a_single_block_has_been_mined() { +// let (blocks_broadcaster, _) = broadcast::channel::(1); +// let s3_client = Arc::new(S3Client::new_for_testing().await); +// let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; + +// publish_block(&publisher, &blocks_broadcaster).await; + +// assert!(publisher +// .get_fuel_streams() +// .blocks() +// .get_last_published(BlocksSubject::WILDCARD) +// .await +// .is_ok_and(|result| result.is_some())); +// s3_client.cleanup_after_testing().await; +// } + +// #[tokio::test(flavor = "multi_thread")] +// async fn publishes_transaction_for_each_published_block() { +// let (blocks_broadcaster, _) = broadcast::channel::(1); +// let s3_client = Arc::new(S3Client::new_for_testing().await); +// let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; + +// publish_block(&publisher, &blocks_broadcaster).await; + +// assert!(publisher +// .get_fuel_streams() +// .transactions() +// .get_last_published(TransactionsSubject::WILDCARD) +// .await +// .is_ok_and(|result| result.is_some())); +// s3_client.cleanup_after_testing().await; +// } + +// #[tokio::test(flavor = "multi_thread")] +// async fn publishes_receipts() { +// let (blocks_broadcaster, _) = broadcast::channel::(1); + +// let receipts = [ +// FuelCoreReceipt::Call { +// id: FuelCoreContractId::default(), +// to: Default::default(), +// amount: 0, +// asset_id: Default::default(), +// gas: 0, +// param1: 0, +// param2: 0, +// pc: 0, +// is: 0, +// }, +// FuelCoreReceipt::Return { +// id: FuelCoreContractId::default(), +// val: 0, +// pc: 0, +// is: 0, +// }, +// FuelCoreReceipt::ReturnData { +// id: FuelCoreContractId::default(), +// ptr: 0, +// len: 0, +// digest: FuelCoreBytes32::default(), +// pc: 0, +// is: 0, +// data: None, +// }, +// FuelCoreReceipt::Revert { +// id: FuelCoreContractId::default(), +// ra: 0, +// pc: 0, +// is: 0, +// }, +// FuelCoreReceipt::Log { +// id: FuelCoreContractId::default(), +// ra: 0, +// rb: 0, +// rc: 0, +// rd: 0, +// pc: 0, +// is: 0, +// }, +// FuelCoreReceipt::LogData { +// id: FuelCoreContractId::default(), +// ra: 0, +// rb: 0, +// ptr: 0, +// len: 0, +// digest: FuelCoreBytes32::default(), +// pc: 0, +// is: 0, +// data: None, +// }, +// FuelCoreReceipt::Transfer { +// id: FuelCoreContractId::default(), +// to: FuelCoreContractId::default(), +// amount: 0, +// asset_id: FuelCoreAssetId::default(), +// pc: 0, +// is: 0, +// }, +// FuelCoreReceipt::TransferOut { +// id: FuelCoreContractId::default(), +// to: FuelCoreAddress::default(), +// amount: 0, +// asset_id: FuelCoreAssetId::default(), +// pc: 0, +// is: 0, +// }, +// FuelCoreReceipt::Mint { +// sub_id: FuelCoreBytes32::default(), +// contract_id: FuelCoreContractId::default(), +// val: 0, +// pc: 0, +// is: 0, +// }, +// FuelCoreReceipt::Burn { +// sub_id: FuelCoreBytes32::default(), +// contract_id: FuelCoreContractId::default(), +// val: 0, +// pc: 0, +// is: 0, +// }, +// ]; + +// let fuel_core = TestFuelCore::default(blocks_broadcaster.clone()) +// .with_receipts(receipts.to_vec()) +// .arc(); + +// let s3_client = Arc::new(S3Client::new_for_testing().await); +// let publisher = +// Publisher::new_for_testing(&nats_client().await, &s3_client, fuel_core) +// .await +// .unwrap(); + +// publish_block(&publisher, &blocks_broadcaster).await; + +// let mut receipts_stream = publisher +// .get_fuel_streams() +// .receipts() +// .catchup(10) +// .await +// .unwrap(); + +// let expected_receipts: Vec = +// receipts.iter().map(Into::into).collect(); +// let mut found_receipts = Vec::new(); + +// while let Some(Some(receipt)) = receipts_stream.next().await { +// found_receipts.push(receipt); +// } +// assert_eq!( +// found_receipts.len(), +// expected_receipts.len(), +// "Number of receipts doesn't match" +// ); + +// // Create sets of receipt identifiers +// let found_ids: std::collections::HashSet<_> = found_receipts +// .into_iter() +// .map(|r| match r { +// Receipt::Call(r) => r.id, +// Receipt::Return(r) => r.id, +// Receipt::ReturnData(r) => r.id, +// Receipt::Revert(r) => r.id, +// Receipt::Log(r) => r.id, +// Receipt::LogData(r) => r.id, +// Receipt::Transfer(r) => r.id, +// Receipt::TransferOut(r) => r.id, +// Receipt::Mint(r) => r.contract_id, +// Receipt::Burn(r) => r.contract_id, +// Receipt::Panic(r) => r.id, +// _ => unreachable!(), +// }) +// .collect(); + +// let expected_ids: std::collections::HashSet<_> = expected_receipts +// .into_iter() +// .map(|r| match r { +// Receipt::Call(r) => r.id, +// Receipt::Return(r) => r.id, +// Receipt::ReturnData(r) => r.id, +// Receipt::Revert(r) => r.id, +// Receipt::Log(r) => r.id, +// Receipt::LogData(r) => r.id, +// Receipt::Transfer(r) => r.id, +// Receipt::TransferOut(r) => r.id, +// Receipt::Mint(r) => r.contract_id, +// Receipt::Burn(r) => r.contract_id, +// Receipt::Panic(r) => r.id, +// _ => unreachable!(), +// }) +// .collect(); + +// assert_eq!( +// found_ids, expected_ids, +// "Published receipt IDs don't match expected IDs" +// ); + +// s3_client.cleanup_after_testing().await; +// } + +// #[tokio::test(flavor = "multi_thread")] +// async fn publishes_inputs() { +// let (blocks_broadcaster, _) = broadcast::channel::(1); +// let s3_client = Arc::new(S3Client::new_for_testing().await); +// let publisher = new_publisher(blocks_broadcaster.clone(), &s3_client).await; + +// publish_block(&publisher, &blocks_broadcaster).await; + +// assert!(publisher +// .get_fuel_streams() +// .inputs() +// .get_last_published(InputsByIdSubject::WILDCARD) +// .await +// .is_ok_and(|result| result.is_some())); +// s3_client.cleanup_after_testing().await; +// } + +// async fn new_publisher( +// broadcaster: Sender, +// s3_client: &Arc, +// ) -> Publisher { +// let fuel_core = TestFuelCore::default(broadcaster).arc(); +// Publisher::new_for_testing(&nats_client().await, s3_client, fuel_core) +// .await +// .unwrap() +// } + +// async fn publish_block( +// publisher: &Publisher, +// blocks_broadcaster: &Sender, +// ) { +// let shutdown_controller = start_publisher(publisher).await; +// send_block(blocks_broadcaster); +// stop_publisher(shutdown_controller).await; +// } + +// async fn start_publisher(publisher: &Publisher) -> ShutdownController { +// let (shutdown_controller, shutdown_token) = get_controller_and_token(); +// tokio::spawn({ +// let publisher = publisher.clone(); +// async move { +// publisher.run(shutdown_token, true).await.unwrap(); +// } +// }); +// wait_for_publisher_to_start().await; +// shutdown_controller +// } +// async fn stop_publisher(shutdown_controller: ShutdownController) { +// wait_for_publisher_to_process_block().await; + +// assert!(shutdown_controller.initiate_shutdown().is_ok()); +// } + +// async fn wait_for_publisher_to_start() { +// tokio::time::sleep(std::time::Duration::from_secs(1)).await; +// } +// async fn wait_for_publisher_to_process_block() { +// tokio::time::sleep(std::time::Duration::from_secs(1)).await; +// } + +// fn send_block(broadcaster: &Sender) { +// let block = create_test_block(); +// assert!(broadcaster.send(block).is_ok()); +// } +// fn create_test_block() -> ImporterResult { +// let mut block_entity = FuelCoreBlock::default(); +// let tx = FuelCoreTransaction::default_test_tx(); + +// *block_entity.transactions_mut() = vec![tx]; + +// ImporterResult { +// shared_result: Arc::new(FuelCoreImportResult { +// sealed_block: SealedBlock { +// entity: block_entity, +// ..Default::default() +// }, +// ..Default::default() +// }), +// changes: Arc::new(HashMap::new()), +// } +// } + +// async fn nats_client() -> NatsClient { +// let opts = NatsClientOpts::admin_opts().with_rdn_namespace(); +// NatsClient::connect(&opts) +// .await +// .expect("NATS connection failed") +// } From 448c5ecf0e32a9f1ed26454e0b5c4d38e9c05142 Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Mon, 23 Dec 2024 02:23:47 -0300 Subject: [PATCH 13/15] feat(repo): integrate rust sdk with websockets --- Cargo.lock | 49 +- Cargo.toml | 5 +- crates/fuel-networks/Cargo.toml | 16 - crates/fuel-streams-core/Cargo.toml | 1 - crates/fuel-streams-core/src/lib.rs | 2 - .../src/stream/stream_impl.rs | 12 +- crates/fuel-streams-nats/src/types.rs | 2 +- crates/fuel-streams/Cargo.toml | 9 + crates/fuel-streams/src/client/client_impl.rs | 234 ++++-- crates/fuel-streams/src/client/connection.rs | 222 ++++++ crates/fuel-streams/src/client/error.rs | 34 +- crates/fuel-streams/src/client/mod.rs | 4 +- crates/fuel-streams/src/client/types.rs | 17 +- crates/fuel-streams/src/error.rs | 10 +- crates/fuel-streams/src/lib.rs | 13 +- .../src/networks/mod.rs} | 36 +- crates/fuel-streams/src/stream/error.rs | 17 - crates/fuel-streams/src/stream/mod.rs | 11 - crates/fuel-streams/src/stream/stream_impl.rs | 304 -------- crates/sv-webserver/Cargo.toml | 1 - crates/sv-webserver/src/client/mod.rs | 350 --------- crates/sv-webserver/src/lib.rs | 1 - crates/sv-webserver/src/server/ws/models.rs | 4 +- crates/sv-webserver/src/server/ws/socket.rs | 53 +- examples/Cargo.toml | 8 - examples/blocks.rs | 29 +- examples/inputs.rs | 29 +- examples/logs.rs | 29 +- examples/multiple-streams.rs | 396 ---------- examples/outputs.rs | 29 +- examples/receipts.rs | 102 +-- examples/transactions.rs | 31 +- examples/utxos.rs | 29 +- examples/websockets.rs | 55 -- scripts/run_webserver.sh | 14 +- tests/Cargo.toml | 4 - tests/src/lib.rs | 11 +- tests/src/main.rs | 142 ---- tests/tests/client.rs | 684 +++++++++--------- tests/tests/stream.rs | 484 ++++++------- 40 files changed, 1183 insertions(+), 2300 deletions(-) delete mode 100644 crates/fuel-networks/Cargo.toml create mode 100644 crates/fuel-streams/src/client/connection.rs rename crates/{fuel-networks/src/lib.rs => fuel-streams/src/networks/mod.rs} (69%) delete mode 100644 crates/fuel-streams/src/stream/error.rs delete mode 100644 crates/fuel-streams/src/stream/mod.rs delete mode 100644 crates/fuel-streams/src/stream/stream_impl.rs delete mode 100644 crates/sv-webserver/src/client/mod.rs delete mode 100644 examples/multiple-streams.rs delete mode 100755 examples/websockets.rs delete mode 100644 tests/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index a8c25d4f..3a7caf6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4085,15 +4085,6 @@ dependencies = [ "sha2 0.10.8", ] -[[package]] -name = "fuel-networks" -version = "0.0.13" -dependencies = [ - "clap 4.5.23", - "serde", - "url", -] - [[package]] name = "fuel-sequencer-proto" version = "0.1.0" @@ -4125,8 +4116,16 @@ dependencies = [ "displaydoc", "fuel-streams-core", "futures", + "reqwest 0.12.9", + "serde", + "serde_json", + "sv-webserver", "thiserror 2.0.8", "tokio", + "tokio-stream", + "tokio-tungstenite 0.26.1", + "url", + "validator", ] [[package]] @@ -4146,7 +4145,6 @@ dependencies = [ "fuel-core-storage", "fuel-core-types 0.40.2", "fuel-data-parser", - "fuel-networks", "fuel-streams-macros", "fuel-streams-nats", "fuel-streams-storage", @@ -9180,7 +9178,6 @@ dependencies = [ "displaydoc", "dotenvy", "elasticsearch", - "fuel-streams", "fuel-streams-core", "fuel-streams-nats", "fuel-streams-storage", @@ -9796,6 +9793,18 @@ dependencies = [ "tungstenite 0.24.0", ] +[[package]] +name = "tokio-tungstenite" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4bf6fecd69fcdede0ec680aaf474cdab988f9de6bc73d3758f0160e3b7025a" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.26.1", +] + [[package]] name = "tokio-util" version = "0.7.13" @@ -10168,6 +10177,24 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413083a99c579593656008130e29255e54dcaae495be556cc26888f211648c24" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.2.0", + "httparse", + "log", + "rand", + "sha1", + "thiserror 2.0.8", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" diff --git a/Cargo.toml b/Cargo.toml index a2ca7839..f7ad8b6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,17 +70,16 @@ tracing-actix-web = "0.7" thiserror = "2.0" fuel-streams = { path = "crates/fuel-streams" } -fuel-networks = { path = "crates/fuel-networks" } fuel-data-parser = { version = "0.0.13", path = "crates/fuel-data-parser" } fuel-streams-core = { version = "0.0.13", path = "crates/fuel-streams-core" } -sv-webserver = { version = "0.0.13", path = "crates/sv-webserver" } fuel-streams-macros = { version = "0.0.13", path = "crates/fuel-streams-macros" } fuel-streams-nats = { version = "0.0.13", path = "crates/fuel-streams-nats" } fuel-streams-storage = { version = "0.0.13", path = "crates/fuel-streams-storage" } -subject-derive = { version = "0.0.13", path = "crates/fuel-streams-macros/subject-derive" } fuel-streams-executors = { version = "0.0.13", path = "crates/fuel-streams-executors" } +subject-derive = { version = "0.0.13", path = "crates/fuel-streams-macros/subject-derive" } sv-publisher = { version = "0.0.13", path = "crates/sv-publisher" } sv-consumer = { version = "0.0.13", path = "crates/sv-consumer" } +sv-webserver = { version = "0.0.13", path = "crates/sv-webserver" } # Workspace projects [workspace.metadata.cargo-machete] diff --git a/crates/fuel-networks/Cargo.toml b/crates/fuel-networks/Cargo.toml deleted file mode 100644 index cc8000ad..00000000 --- a/crates/fuel-networks/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "fuel-networks" -description = "Networks for the Fuel blockchain and utilities for tranforming them for different apps" -authors = { workspace = true } -keywords = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -repository = { workspace = true } -version = { workspace = true } -rust-version = { workspace = true } - -[dependencies] -clap = { workspace = true } -serde = { workspace = true } -url = "2.5" diff --git a/crates/fuel-streams-core/Cargo.toml b/crates/fuel-streams-core/Cargo.toml index 9e6178fc..2087376f 100644 --- a/crates/fuel-streams-core/Cargo.toml +++ b/crates/fuel-streams-core/Cargo.toml @@ -33,7 +33,6 @@ fuel-core-services = { workspace = true, default-features = false, features = [" fuel-core-storage = { workspace = true } fuel-core-types = { workspace = true, default-features = false, features = ["std", "serde"] } fuel-data-parser = { workspace = true } -fuel-networks = { workspace = true } fuel-streams-macros = { workspace = true } fuel-streams-nats = { workspace = true } fuel-streams-storage = { workspace = true } diff --git a/crates/fuel-streams-core/src/lib.rs b/crates/fuel-streams-core/src/lib.rs index ba8b30f8..219576fc 100644 --- a/crates/fuel-streams-core/src/lib.rs +++ b/crates/fuel-streams-core/src/lib.rs @@ -17,7 +17,6 @@ pub mod s3 { } pub mod stream; - pub mod subjects; pub mod fuel_core_like; @@ -28,7 +27,6 @@ pub mod types; pub use stream::*; pub mod prelude { - pub use fuel_networks::*; #[allow(unused_imports)] pub use fuel_streams_macros::subject::*; diff --git a/crates/fuel-streams-core/src/stream/stream_impl.rs b/crates/fuel-streams-core/src/stream/stream_impl.rs index fc8d974b..3d40d4f7 100644 --- a/crates/fuel-streams-core/src/stream/stream_impl.rs +++ b/crates/fuel-streams-core/src/stream/stream_impl.rs @@ -219,7 +219,7 @@ impl Stream { ) -> Result>, StreamError> { let mut config = PullConsumerConfig { filter_subjects: vec![S::WILDCARD_LIST[0].to_string()], - deliver_policy: DeliverPolicy::All, + deliver_policy: NatsDeliverPolicy::All, ack_policy: AckPolicy::None, ..Default::default() }; @@ -263,7 +263,7 @@ impl Stream { ) -> Result, StreamError> { let mut config = PullConsumerConfig { filter_subjects: vec![S::WILDCARD_LIST[0].to_string()], - deliver_policy: DeliverPolicy::All, + deliver_policy: NatsDeliverPolicy::All, ack_policy: AckPolicy::None, ..Default::default() }; @@ -309,7 +309,7 @@ impl Stream { ) -> Result>, StreamError> { let config = PullConsumerConfig { filter_subjects: self.all_filter_subjects(), - deliver_policy: DeliverPolicy::All, + deliver_policy: NatsDeliverPolicy::All, ack_policy: AckPolicy::None, ..Default::default() }; @@ -442,15 +442,15 @@ impl Stream { /// /// ``` /// use fuel_streams_core::stream::SubscriptionConfig; -/// use async_nats::jetstream::consumer::DeliverPolicy; +/// use async_nats::jetstream::consumer::NatsDeliverPolicy; /// /// let config = SubscriptionConfig { /// filter_subjects: vec!["example.*".to_string()], -/// deliver_policy: DeliverPolicy::All, +/// deliver_policy: NatsDeliverPolicy::All, /// }; /// ``` #[derive(Debug, Clone, Default)] pub struct SubscriptionConfig { pub filter_subjects: Vec, - pub deliver_policy: DeliverPolicy, + pub deliver_policy: NatsDeliverPolicy, } diff --git a/crates/fuel-streams-nats/src/types.rs b/crates/fuel-streams-nats/src/types.rs index eb121ffe..70ac9d02 100644 --- a/crates/fuel-streams-nats/src/types.rs +++ b/crates/fuel-streams-nats/src/types.rs @@ -10,7 +10,7 @@ pub use async_nats::{ AckPolicy, Config as ConsumerConfig, Consumer as NatsConsumer, - DeliverPolicy, + DeliverPolicy as NatsDeliverPolicy, }, kv::Config as KvStoreConfig, stream::Config as NatsStreamConfig, diff --git a/crates/fuel-streams/Cargo.toml b/crates/fuel-streams/Cargo.toml index 7a907181..dfa535c4 100644 --- a/crates/fuel-streams/Cargo.toml +++ b/crates/fuel-streams/Cargo.toml @@ -14,7 +14,16 @@ version = "0.0.13" displaydoc = { workspace = true } fuel-streams-core = { workspace = true } futures = { workspace = true } +reqwest = "0.12.9" +serde = { workspace = true } +serde_json = { workspace = true } +sv-webserver = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tokio-tungstenite = "0.26.1" +url = "2.5.4" +validator = "0.19.0" [dev-dependencies] tokio = { workspace = true } diff --git a/crates/fuel-streams/src/client/client_impl.rs b/crates/fuel-streams/src/client/client_impl.rs index b2d9deb1..2ecd8152 100644 --- a/crates/fuel-streams/src/client/client_impl.rs +++ b/crates/fuel-streams/src/client/client_impl.rs @@ -1,107 +1,189 @@ -use std::sync::Arc; +use reqwest::{ + header::{ + ACCEPT, + AUTHORIZATION, + CONNECTION, + CONTENT_TYPE, + HOST, + SEC_WEBSOCKET_KEY, + SEC_WEBSOCKET_VERSION, + UPGRADE, + }, + Client as HttpClient, +}; +use tokio_tungstenite::tungstenite::{ + client::IntoClientRequest, + handshake::client::generate_key, +}; -use fuel_streams_core::prelude::*; +use super::{ + error::ClientError, + Connection, + ConnectionOpts, + LoginRequest, + LoginResponse, +}; +use crate::FuelNetwork; -use super::ClientError; - -/// A client for connecting to a NATS server. +/// A client for connecting to the Fuel websocket server. +/// +/// # Examples +/// +/// ```no_run +/// use fuel_streams::{Client, FuelNetwork}; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box> { +/// // Basic usage with default credentials +/// let mut client = Client::new(FuelNetwork::Local).await?; +/// let connection = client.connect().await?; /// -/// This struct represents a connected NATS client. +/// // Or with custom connection options +/// let client = Client::with_opts(ConnectionOpts { +/// network: FuelNetwork::Local, +/// username: "custom_user".to_string(), +/// password: "custom_pass".to_string(), +/// }).await?; +/// Ok(()) +/// } +/// ``` #[derive(Debug, Clone)] pub struct Client { - /// The underlying NATS client connection. - pub nats_conn: Arc, - pub s3_conn: Arc, + pub opts: ConnectionOpts, + pub jwt_token: Option, } impl Client { - /// Connects to a NATS server using the provided URL. - /// - /// # Parameters - /// - /// * `network`: An enum variant representing the fuel network we are connecting to. - /// - /// # Returns - /// - /// Returns a `Result` containing the connected client on success, or an error on failure. + /// Creates a new WebSocket client with default connection options for the specified network. + pub async fn new(network: FuelNetwork) -> Result { + Self::with_opts(ConnectionOpts { + network, + ..Default::default() + }) + .await + } + + /// Creates a new WebSocket client with custom connection options. + pub async fn with_opts(opts: ConnectionOpts) -> Result { + let jwt_token = + Self::fetch_jwt(opts.network, &opts.username, &opts.password) + .await?; + Ok(Self { + opts, + jwt_token: Some(jwt_token), + }) + } + + /// Establishes a WebSocket connection using the client's configuration. /// /// # Examples /// /// ```no_run - /// use fuel_streams::client::{Client, FuelNetwork}; + /// use fuel_streams::{Client, FuelNetwork}; /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// let client = Client::connect(FuelNetwork::Local).await?; - /// # Ok(()) - /// # } + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let mut client = Client::new(FuelNetwork::Local).await?; + /// let connection = client.connect().await?; + /// Ok(()) + /// } /// ``` - pub async fn connect(network: FuelNetwork) -> Result { - let nats_opts = - NatsClientOpts::public_opts().with_url(network.to_nats_url()); - let nats_client = NatsClient::connect(&nats_opts) - .await - .map_err(ClientError::NatsConnectionFailed)?; - - let s3_client_opts = match network { - FuelNetwork::Local => { - S3ClientOpts::new(S3Env::Local, S3Role::Admin) - } - FuelNetwork::Testnet => { - S3ClientOpts::new(S3Env::Testnet, S3Role::Public) - } - FuelNetwork::Mainnet => { - S3ClientOpts::new(S3Env::Mainnet, S3Role::Public) - } - }; + pub async fn connect(&mut self) -> Result { + let ws_url = self.opts.network.to_ws_url().join("/api/v1/ws")?; + let host = ws_url + .host_str() + .ok_or_else(|| ClientError::HostParseFailed)?; - let s3_client = S3Client::new(&s3_client_opts) - .await - .map_err(ClientError::S3ConnectionFailed)?; + let jwt_token = + self.jwt_token.clone().ok_or(ClientError::MissingJwtToken)?; - Ok(Self { - nats_conn: Arc::new(nats_client), - s3_conn: Arc::new(s3_client), - }) + let bearer_token = format!("Bearer {}", jwt_token); + let mut request = ws_url.as_str().into_client_request()?; + let headers_map = request.headers_mut(); + headers_map.insert(AUTHORIZATION, bearer_token.parse()?); + headers_map.insert(HOST, host.parse()?); + headers_map.insert(UPGRADE, "websocket".parse()?); + headers_map.insert(CONNECTION, "Upgrade".parse().unwrap()); + headers_map.insert(SEC_WEBSOCKET_KEY, generate_key().parse()?); + headers_map.insert(SEC_WEBSOCKET_VERSION, "13".parse()?); + Connection::new(request).await } - /// Connects to a NATS server using the provided options. + /// Fetches a JWT token from the server for authentication. /// - /// # Parameters + /// # Examples /// - /// * `opts`: A reference to `NatsClientOpts` containing the connection options. + /// ```no_run + /// use fuel_streams::{Client, FuelNetwork}; /// - /// # Returns + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let jwt = Client::fetch_jwt( + /// FuelNetwork::Local, + /// "admin", + /// "admin" + /// ).await?; /// - /// Returns a `ConnectionResult` containing the connected client on success, or an error on failure. + /// assert!(!jwt.is_empty()); + /// Ok(()) + /// } + /// ``` + async fn fetch_jwt( + network: FuelNetwork, + username: &str, + password: &str, + ) -> Result { + let client = HttpClient::new(); + let json_body = serde_json::to_string(&LoginRequest { + username: username.to_string(), + password: password.to_string(), + })?; + + let api_url = network.to_web_url().join("/api/v1/jwt")?; + let response = client + .get(api_url) + .header(ACCEPT, "application/json") + .header(CONTENT_TYPE, "application/json") + .body(json_body) + .send() + .await?; + + if response.status().is_success() { + let json_body = response.json::().await?; + Ok(json_body.jwt_token) + } else { + Err(ClientError::ApiResponse( + response.error_for_status_ref().unwrap_err(), + )) + } + } + + /// Refreshes the JWT token and establishes a new WebSocket connection. /// /// # Examples /// /// ```no_run - /// use fuel_streams::client::{Client, FuelNetwork}; - /// use fuel_streams_core::nats::NatsClientOpts; - /// use fuel_streams_core::s3::{S3ClientOpts, S3Env, S3Role}; + /// use fuel_streams::{Client, FuelNetwork}; /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// let nats_opts = NatsClientOpts::public_opts().with_url("nats://localhost:4222"); - /// let s3_opts = S3ClientOpts::new(S3Env::Local, S3Role::Admin); + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let mut client = Client::new(FuelNetwork::Local).await?; /// - /// let client = Client::with_opts(&nats_opts, &s3_opts).await?; - /// # Ok(()) - /// # } + /// // Refresh token and reconnect + /// let new_connection = client.refresh_jwt_and_connect().await?; + /// Ok(()) + /// } /// ``` - pub async fn with_opts( - nats_opts: &NatsClientOpts, - s3_opts: &S3ClientOpts, - ) -> Result { - let nats_client = NatsClient::connect(nats_opts) - .await - .map_err(ClientError::NatsConnectionFailed)?; - let s3_client = S3Client::new(s3_opts) - .await - .map_err(ClientError::S3ConnectionFailed)?; - Ok(Self { - nats_conn: Arc::new(nats_client), - s3_conn: Arc::new(s3_client), - }) + pub async fn refresh_jwt_and_connect( + &mut self, + ) -> Result { + let jwt_token = Self::fetch_jwt( + self.opts.network, + &self.opts.username, + &self.opts.password, + ) + .await?; + self.jwt_token = Some(jwt_token); + self.connect().await } } diff --git a/crates/fuel-streams/src/client/connection.rs b/crates/fuel-streams/src/client/connection.rs new file mode 100644 index 00000000..e7413d3d --- /dev/null +++ b/crates/fuel-streams/src/client/connection.rs @@ -0,0 +1,222 @@ +use fuel_streams_core::{subjects::IntoSubject, Streamable}; +use futures::{ + stream::{SplitSink, SplitStream}, + SinkExt, + Stream, + StreamExt, +}; +use tokio::sync::RwLock; +use tokio_tungstenite::{ + connect_async, + tungstenite::{http::Request, protocol::Message}, + MaybeTlsStream, +}; + +use super::{ + error::ClientError, + types::{ClientMessage, DeliverPolicy, ServerMessage, SubscriptionPayload}, +}; +use crate::FuelNetwork; + +/// Connection options for establishing a WebSocket connection. +/// +/// # Examples +/// +/// ``` +/// use fuel_streams::{ConnectionOpts, FuelNetwork}; +/// +/// // Create connection options with custom values +/// let opts = ConnectionOpts { +/// network: FuelNetwork::Local, +/// username: "admin".to_string(), +/// password: "admin".to_string(), +/// }; +/// +/// // Or use the default options +/// let default_opts = ConnectionOpts::default(); +/// assert_eq!(default_opts.username, "admin"); +/// assert_eq!(default_opts.password, "admin"); +/// assert!(matches!(default_opts.network, FuelNetwork::Local)); +/// ``` +#[derive(Debug, Clone)] +pub struct ConnectionOpts { + pub network: FuelNetwork, + pub username: String, + pub password: String, +} + +impl Default for ConnectionOpts { + fn default() -> Self { + Self { + network: FuelNetwork::Local, + username: "admin".to_string(), + password: "admin".to_string(), + } + } +} + +type ReadStream = SplitStream< + tokio_tungstenite::WebSocketStream>, +>; +type WriteSink = RwLock< + SplitSink< + tokio_tungstenite::WebSocketStream< + MaybeTlsStream, + >, + Message, + >, +>; + +#[derive(Debug)] +pub struct Connection { + pub read_stream: ReadStream, + pub write_sink: WriteSink, +} + +impl Connection { + pub async fn new(req: Request<()>) -> Result { + let (socket, _response) = connect_async(req).await?; + let (write, read) = socket.split(); + + Ok(Self { + read_stream: read, + write_sink: RwLock::new(write), + }) + } + /// Sends a client message through the WebSocket connection. + /// + /// # Examples + /// + /// ```no_run + /// use fuel_streams::{Client, ConnectionOpts, FuelNetwork}; + /// use sv_webserver::server::ws::models::ClientMessage; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let mut client = Client::new(FuelNetwork::Local).await?; + /// let connection = client.connect().await?; + /// + /// let message = ClientMessage::Ping; + /// connection.send_client_message(message).await?; + /// Ok(()) + /// } + /// ``` + async fn send_client_message( + &self, + message: ClientMessage, + ) -> Result<(), ClientError> { + let mut write_guard = self.write_sink.write().await; + let serialized = serde_json::to_vec(&message)?; + write_guard.send(Message::Binary(serialized.into())).await?; + Ok(()) + } + + /// Subscribes to a subject and returns a stream of messages. + /// + /// # Examples + /// + /// ```no_run + /// use fuel_streams::{Client, ConnectionOpts, FuelNetwork}; + /// use sv_webserver::server::ws::models::{DeliverPolicy, ServerMessage}; + /// use std::sync::Arc; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let mut client = Client::new(FuelNetwork::Local).await?; + /// let mut connection = client.connect().await?; + /// + /// let subject = Arc::new("example.subject"); + /// let mut stream = connection.subscribe::( + /// subject, + /// DeliverPolicy::All + /// ).await?; + /// + /// // Process messages from the stream + /// while let Some(message) = stream.next().await { + /// println!("Received: {:?}", message); + /// } + /// Ok(()) + /// } + /// ``` + pub async fn subscribe( + &mut self, + subject: impl IntoSubject, + deliver_policy: DeliverPolicy, + ) -> Result + '_ + Send + Unpin, ClientError> { + let message = ClientMessage::Subscribe(SubscriptionPayload { + wildcard: subject.parse(), + deliver_policy, + }); + self.send_client_message(message).await?; + + let stream = self.read_stream.by_ref().filter_map(|msg| async move { + match msg { + Ok(Message::Binary(bin)) => { + match serde_json::from_slice::(&bin) { + Ok(ServerMessage::Response(value)) => { + match serde_json::from_value::(value) { + Ok(parsed) => Some(parsed), + Err(e) => { + eprintln!("Failed to parse value: {:?}", e); + None + } + } + } + Ok(ServerMessage::Error(e)) => { + eprintln!("Server error: {}", e); + None + } + Ok(_) => None, + Err(e) => { + eprintln!("Unparsable server message: {:?}", e); + None + } + } + } + Ok(Message::Close(_)) => None, + Ok(msg) => { + println!("Received message: {:?}", msg); + None + } + Err(e) => { + eprintln!("WebSocket error: {:?}", e); + None + } + } + }); + + Ok(Box::pin(stream)) + } + + /// Unsubscribes from a subject. + /// + /// # Examples + /// + /// ```no_run + /// use fuel_streams::{Client, ConnectionOpts, FuelNetwork}; + /// use sv_webserver::server::ws::models::DeliverPolicy; + /// use std::sync::Arc; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let mut client = Client::new(FuelNetwork::Local).await?; + /// let connection = client.connect().await?; + /// + /// let subject = Arc::new("example.subject"); + /// connection.unsubscribe(subject, DeliverPolicy::All).await?; + /// Ok(()) + /// } + /// ``` + pub async fn unsubscribe( + &self, + subject: S, + deliver_policy: DeliverPolicy, + ) -> Result<(), ClientError> { + let message = ClientMessage::Unsubscribe(SubscriptionPayload { + wildcard: subject.parse(), + deliver_policy, + }); + self.send_client_message(message).await?; + Ok(()) + } +} diff --git a/crates/fuel-streams/src/client/error.rs b/crates/fuel-streams/src/client/error.rs index 216f855e..a4f78938 100644 --- a/crates/fuel-streams/src/client/error.rs +++ b/crates/fuel-streams/src/client/error.rs @@ -1,11 +1,35 @@ use displaydoc::Display as DisplayDoc; -use fuel_streams_core::{nats::NatsError, s3::S3ClientError}; use thiserror::Error; #[derive(Debug, Error, DisplayDoc)] pub enum ClientError { - /// Failed to establish connection with the NATS server: {0} - NatsConnectionFailed(#[from] NatsError), - /// Failed to establish connection with S3: {0} - S3ConnectionFailed(#[from] S3ClientError), + /// Failed to convert JSON to string: {0} + JsonToString(#[from] serde_json::Error), + + /// Failed to parse URL: {0} + UrlParse(#[from] url::ParseError), + + /// Failed to API response: {0} + ApiResponse(#[from] reqwest::Error), + + /// Failed to connect to WebSocket: {0} + WebSocketConnect(#[from] tokio_tungstenite::tungstenite::Error), + + /// Invalid header value: {0} + InvalidHeaderValue(#[from] reqwest::header::InvalidHeaderValue), + + /// Failed to parse host from URL + HostParseFailed, + + /// Missing JWT token + MissingJwtToken, + + /// Missing write sink + MissingWriteSink, + + /// Missing read stream + MissingReadStream, + + /// Missing WebSocket connection + MissingWebSocketConnection, } diff --git a/crates/fuel-streams/src/client/mod.rs b/crates/fuel-streams/src/client/mod.rs index 09883f4c..ba21bcb3 100644 --- a/crates/fuel-streams/src/client/mod.rs +++ b/crates/fuel-streams/src/client/mod.rs @@ -1,7 +1,9 @@ -mod client_impl; +pub mod client_impl; +pub mod connection; pub mod error; pub mod types; pub use client_impl::*; +pub use connection::*; pub use error::*; pub use types::*; diff --git a/crates/fuel-streams/src/client/types.rs b/crates/fuel-streams/src/client/types.rs index f2e5c976..938aa620 100644 --- a/crates/fuel-streams/src/client/types.rs +++ b/crates/fuel-streams/src/client/types.rs @@ -1,8 +1,9 @@ -pub use fuel_streams_core::prelude::FuelNetwork; - -#[derive(Debug, Clone, Eq, PartialEq, Default)] -pub enum ClientStatus { - #[default] - Pending, - Connected, -} +pub use sv_webserver::server::{ + http::models::{LoginRequest, LoginResponse}, + ws::models::{ + ClientMessage, + DeliverPolicy, + ServerMessage, + SubscriptionPayload, + }, +}; diff --git a/crates/fuel-streams/src/error.rs b/crates/fuel-streams/src/error.rs index 96e0b86b..46b5cf8f 100644 --- a/crates/fuel-streams/src/error.rs +++ b/crates/fuel-streams/src/error.rs @@ -3,12 +3,6 @@ use thiserror::Error as ThisError; #[derive(Debug, ThisError, DisplayDoc)] pub enum Error { - /// An error occurred in the client - ClientError(#[from] crate::client::ClientError), - - /// An error occurred in the stream - StreamError(#[from] crate::stream::StreamError), - - /// Consuming messages error - MessagesError(#[from] fuel_streams_core::types::MessagesError), + /// WebSocket client error: {0} + Client(#[from] crate::client::error::ClientError), } diff --git a/crates/fuel-streams/src/lib.rs b/crates/fuel-streams/src/lib.rs index 9bd61ae9..f4e965f8 100644 --- a/crates/fuel-streams/src/lib.rs +++ b/crates/fuel-streams/src/lib.rs @@ -2,21 +2,18 @@ pub mod client; pub mod error; -pub mod stream; +pub mod networks; +pub use client::*; pub use error::*; -pub use stream::*; +pub use networks::*; pub mod subjects { pub use fuel_streams_core::subjects::*; } pub mod types { - pub use fuel_streams_core::{ - nats::{types::*, NatsClientOpts}, - prelude::FuelNetwork, - types::*, - }; + pub use fuel_streams_core::types::*; pub use crate::client::types::*; } @@ -41,5 +38,5 @@ export_module!(utxos, subjects, types); #[cfg(any(test, feature = "test-helpers"))] pub mod prelude { - pub use crate::{client::*, error::*, stream::*, types::*}; + pub use crate::{client::*, error::*, networks::*, subjects::*, types::*}; } diff --git a/crates/fuel-networks/src/lib.rs b/crates/fuel-streams/src/networks/mod.rs similarity index 69% rename from crates/fuel-networks/src/lib.rs rename to crates/fuel-streams/src/networks/mod.rs index 78c3c963..98085076 100644 --- a/crates/fuel-networks/src/lib.rs +++ b/crates/fuel-streams/src/networks/mod.rs @@ -11,9 +11,7 @@ pub enum FuelNetworkUserRole { Default, } -#[derive( - Debug, Copy, Clone, Default, clap::ValueEnum, Deserialize, Serialize, -)] +#[derive(Debug, Copy, Clone, Default, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] pub enum FuelNetwork { #[default] @@ -66,7 +64,7 @@ impl FuelNetwork { pub fn to_web_url(&self) -> Url { match self { FuelNetwork::Local => { - Url::parse("http://0.0.0.0:9003").expect("working url") + Url::parse("http://localhost:9003").expect("working url") } FuelNetwork::Testnet => { Url::parse("http://stream-testnet.fuel.network:9003") @@ -92,34 +90,4 @@ impl FuelNetwork { .expect("working url"), } } - - pub fn to_s3_url(&self) -> String { - match self { - FuelNetwork::Local => "http://localhost:4566".to_string(), - FuelNetwork::Testnet | FuelNetwork::Mainnet => { - let bucket = self.to_s3_bucket(); - let region = self.to_s3_region(); - // TODO: Update for client streaming - format!("https://{bucket}.s3-website-{region}.amazonaws.com") - } - } - } - - pub fn to_s3_region(&self) -> String { - // TODO: Update correctly for client streaming - match self { - FuelNetwork::Local - | FuelNetwork::Testnet - | FuelNetwork::Mainnet => "us-east-1".to_string(), - } - } - - pub fn to_s3_bucket(&self) -> String { - match self { - FuelNetwork::Local => "fuel-streams-local", - FuelNetwork::Testnet => "fuel-streams-testnet", - FuelNetwork::Mainnet => "fuel-streams", - } - .to_string() - } } diff --git a/crates/fuel-streams/src/stream/error.rs b/crates/fuel-streams/src/stream/error.rs deleted file mode 100644 index 00e5857d..00000000 --- a/crates/fuel-streams/src/stream/error.rs +++ /dev/null @@ -1,17 +0,0 @@ -use displaydoc::Display as DisplayDoc; -use thiserror::Error; - -#[derive(Debug, Error, DisplayDoc)] -pub enum StreamError { - /// Failed to subscribe to the stream - Subscribe { - #[source] - source: fuel_streams_core::StreamError, - }, - - /// Failed to subscribe to the stream with custom options - SubscribeWithOpts { - #[source] - source: fuel_streams_core::StreamError, - }, -} diff --git a/crates/fuel-streams/src/stream/mod.rs b/crates/fuel-streams/src/stream/mod.rs deleted file mode 100644 index 755cda45..00000000 --- a/crates/fuel-streams/src/stream/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod error; -mod stream_impl; - -pub use error::*; -pub use fuel_streams_core::stream::{ - StreamData, - StreamEncoder, - Streamable, - SubscriptionConfig, -}; -pub use stream_impl::*; diff --git a/crates/fuel-streams/src/stream/stream_impl.rs b/crates/fuel-streams/src/stream/stream_impl.rs deleted file mode 100644 index e899b68e..00000000 --- a/crates/fuel-streams/src/stream/stream_impl.rs +++ /dev/null @@ -1,304 +0,0 @@ -use std::pin::Pin; - -use fuel_streams_core::{ - prelude::{IntoSubject, SubjectBuildable}, - types::DeliverPolicy, - Streamable, - SubscriptionConfig, -}; - -use crate::{client::Client, stream::StreamError}; - -/// A filter for stream subjects. -/// -/// This struct is used to build and represent filters for stream subjects. -#[derive(Debug, Clone)] -pub struct Filter { - /// The subject to filter on. - pub subject: S, -} - -impl Filter { - /// Builds a new subject filter. - /// - /// # Returns - /// - /// Returns a new instance of the subject type `S`. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::stream::Filter; - /// use fuel_streams::blocks::BlocksSubject; - /// - /// let filter = Filter::::build(); - /// ``` - pub fn build() -> S { - S::new() - } -} - -/// Configuration options for a stream. -#[derive(Debug, Clone, Default)] -pub struct StreamConfig { - /// The delivery policy for the stream. - pub deliver_policy: DeliverPolicy, -} - -/// Represents a stream of data. -/// -/// This struct wraps a `fuel_streams_core::Stream` and provides methods for -/// subscribing to and filtering the stream. -#[derive(Debug, Clone)] -pub struct Stream { - stream: fuel_streams_core::Stream, - filter_subjects: Vec, -} - -impl Stream { - /// Creates a new `Stream` instance. - /// - /// # Parameters - /// - /// * `client`: A reference to a `Client` instance used to establish the connection. - /// - /// # Returns - /// - /// Returns a new `Stream` instance. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::types::FuelNetwork; - /// use fuel_streams::client::Client; - /// use fuel_streams::stream::Stream; - /// use fuel_streams::blocks::Block; - /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// let client = Client::connect(FuelNetwork::Local).await?; - /// let stream = Stream::::new(&client).await; - /// # Ok(()) - /// # } - /// ``` - pub async fn new(client: &Client) -> Self { - let stream = fuel_streams_core::Stream::::get_or_init( - &client.nats_conn, - &client.s3_conn, - ) - .await; - Self { - stream, - filter_subjects: Vec::new(), - } - } - - /// Adds a filter to the stream. - /// - /// # Parameters - /// - /// * `filter`: An object that can be converted into a subject filter. - /// - /// # Returns - /// - /// Returns a reference to the `Stream` instance for method chaining. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::types::FuelNetwork; - /// use fuel_streams::client::Client; - /// use fuel_streams::stream::{Stream, Filter}; - /// use fuel_streams::blocks::{Block, BlocksSubject}; - /// use fuel_streams::types::Address; - /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// # let client = Client::connect(FuelNetwork::Local).await?; - /// # let mut stream = Stream::::new(&client).await; - /// let filter = Filter::::build() - /// .with_producer(Some(Address::zeroed())) - /// .with_height(Some(5.into())); - /// stream.with_filter(filter); - /// # Ok(()) - /// # } - /// ``` - pub fn with_filter(&mut self, filter: impl IntoSubject) -> &Self { - self.filter_subjects.push(filter.parse()); - self - } - - /// Subscribes to the stream item. - /// - /// # Returns - /// - /// Returns a `Result` containing a `futures::Stream` of byte vectors on success, - /// or a `StreamError` on failure. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::types::FuelNetwork; - /// use fuel_streams::client::Client; - /// use fuel_streams::stream::Stream; - /// use fuel_streams::blocks::Block; - /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// # let client = Client::connect(FuelNetwork::Local).await?; - /// # let stream = Stream::::new(&client).await; - /// let subscription = stream.subscribe().await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn subscribe<'a>( - &'a self, - ) -> Result + Send + 'a>>, StreamError> - { - // TODO: Why implicitly select a stream for the user? - // TODO: Should this be a combination of streams - self.stream - // TODO: Improve DX by ensuring the stream returns the streamable entity directly - .subscribe(None) - .await - .map_err(|source| StreamError::Subscribe { source }) - } - - /// Subscribes to the stream bytes. - /// - /// # Returns - /// - /// Returns a `Result` containing a `futures::Stream` of byte vectors on success, - /// or a `StreamError` on failure. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::types::FuelNetwork; - /// use fuel_streams::client::Client; - /// use fuel_streams::stream::Stream; - /// use fuel_streams::blocks::Block; - /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// # let client = Client::connect(FuelNetwork::Local).await?; - /// # let stream = Stream::::new(&client).await; - /// let subscription = stream.subscribe().await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn subscribe_raw<'a>( - &'a self, - ) -> Result< - Pin> + Send + 'a>>, - StreamError, - > { - // TODO: Why implicitly select a stream for the user? - // TODO: Should this be a combination of streams - self.stream - // TODO: Improve DX by ensuring the stream returns the streamable entity directly - .subscribe_raw(None) - .await - .map_err(|source| StreamError::Subscribe { source }) - } - - /// Subscribes to the stream item with custom configuration options. - /// - /// # Parameters - /// - /// * `opts`: A `StreamConfig` instance containing custom configuration options. - /// - /// # Returns - /// - /// Returns a `Result` containing a `PullConsumerStream` on success, - /// or a `StreamError` on failure. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::types::FuelNetwork; - /// use fuel_streams::client::Client; - /// use fuel_streams::stream::{Stream, StreamConfig}; - /// use fuel_streams::blocks::Block; - /// use fuel_streams::types::DeliverPolicy; - /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// # let client = Client::connect(FuelNetwork::Local).await?; - /// # let stream = Stream::::new(&client).await; - /// let config = StreamConfig { - /// deliver_policy: DeliverPolicy::All, - /// }; - /// let subscription = stream.subscribe_with_config(config).await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn subscribe_with_config<'a>( - &'a self, - opts: StreamConfig, - ) -> Result + Send + 'a>>, StreamError> - { - self.stream - // TODO: Improve DX by ensuring the stream returns the streamable entity directly - .subscribe(Some(SubscriptionConfig { - deliver_policy: opts.deliver_policy, - filter_subjects: self.filter_subjects.to_owned(), - })) - .await - .map_err(|source| StreamError::SubscribeWithOpts { source }) - } - - /// Subscribes to the stream bytes with custom configuration options. - /// - /// # Parameters - /// - /// * `opts`: A `StreamConfig` instance containing custom configuration options. - /// - /// # Returns - /// - /// Returns a `Result` containing a `PullConsumerStream` on success, - /// or a `StreamError` on failure. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::types::FuelNetwork; - /// use fuel_streams::client::Client; - /// use fuel_streams::stream::{Stream, StreamConfig}; - /// use fuel_streams::blocks::Block; - /// use fuel_streams::types::DeliverPolicy; - /// - /// # async fn example() -> Result<(), fuel_streams::Error> { - /// # let client = Client::connect(FuelNetwork::Local).await?; - /// # let stream = Stream::::new(&client).await; - /// let config = StreamConfig { - /// deliver_policy: DeliverPolicy::All, - /// }; - /// let subscription = stream.subscribe_with_config(config).await?; - /// # Ok(()) - /// # } - /// ``` - pub async fn subscribe_raw_with_config<'a>( - &'a self, - opts: StreamConfig, - ) -> Result< - Pin> + Send + 'a>>, - StreamError, - > { - self.stream - // TODO: Improve DX by ensuring the stream returns the streamable entity directly - .subscribe_raw(Some(SubscriptionConfig { - deliver_policy: opts.deliver_policy, - filter_subjects: self.filter_subjects.to_owned(), - })) - .await - .map_err(|source| StreamError::SubscribeWithOpts { source }) - } - - /// Returns a reference to the underlying `fuel_streams_core::Stream`. - /// - /// This method is only available when compiled with the `test` or `test-helpers` feature. - /// - /// # Returns - /// - /// Returns a reference to the underlying `fuel_streams_core::Stream`. - #[cfg(any(test, feature = "test-helpers"))] - pub fn stream(&self) -> &fuel_streams_core::Stream { - &self.stream - } -} diff --git a/crates/sv-webserver/Cargo.toml b/crates/sv-webserver/Cargo.toml index e1f77ddd..766c13aa 100644 --- a/crates/sv-webserver/Cargo.toml +++ b/crates/sv-webserver/Cargo.toml @@ -26,7 +26,6 @@ derive_more = { version = "1.0", features = ["full"] } displaydoc = { workspace = true } dotenvy = { workspace = true } elasticsearch = "8.15.0-alpha.1" -fuel-streams = { workspace = true, features = ["test-helpers"] } fuel-streams-core = { workspace = true, features = ["test-helpers"] } fuel-streams-nats = { workspace = true, features = ["test-helpers"] } fuel-streams-storage = { workspace = true, features = ["test-helpers"] } diff --git a/crates/sv-webserver/src/client/mod.rs b/crates/sv-webserver/src/client/mod.rs deleted file mode 100644 index bbf7aa5c..00000000 --- a/crates/sv-webserver/src/client/mod.rs +++ /dev/null @@ -1,350 +0,0 @@ -use fuel_streams::{ - logs::Log, - subjects::IntoSubject, - types::{Block, FuelNetwork, Input, Output, Receipt, Transaction}, - utxos::Utxo, - Streamable, -}; -use futures_util::{ - stream::{SplitSink, SplitStream}, - SinkExt, - StreamExt, -}; -use reqwest::{ - header::{ - ACCEPT, - AUTHORIZATION, - CONNECTION, - CONTENT_TYPE, - HOST, - SEC_WEBSOCKET_KEY, - SEC_WEBSOCKET_VERSION, - UPGRADE, - }, - Client as HttpClient, -}; -use tokio::sync::{mpsc, RwLock}; -use tokio_tungstenite::{ - connect_async, - tungstenite::{ - client::IntoClientRequest, - handshake::client::generate_key, - protocol::Message, - }, - MaybeTlsStream, -}; -use url::Url; - -use crate::server::{ - http::models::{LoginRequest, LoginResponse}, - ws::{ - errors::WsSubscriptionError, - models::{ - ClientMessage, - DeliverPolicy, - ServerMessage, - SubscriptionPayload, - }, - socket::verify_and_extract_subject_name, - }, -}; - -#[derive(Debug)] -pub struct WebSocketClient { - read_stream: Option< - SplitStream< - tokio_tungstenite::WebSocketStream< - MaybeTlsStream, - >, - >, - >, - write_sink: Option< - RwLock< - SplitSink< - tokio_tungstenite::WebSocketStream< - MaybeTlsStream, - >, - Message, - >, - >, - >, - jwt_token: Option, - ws_url: Url, - network: FuelNetwork, - username: String, - password: String, -} - -impl WebSocketClient { - pub async fn new( - network: FuelNetwork, - username: &str, - password: &str, - ) -> anyhow::Result { - let jwt_token = Self::fetch_jwt(network, username, password).await?; - - let ws_url = network.to_ws_url().join("/api/v1/ws")?; - - Ok(Self { - read_stream: None, - write_sink: None, - jwt_token: Some(jwt_token), - ws_url, - network, - username: username.to_string(), - password: password.to_string(), - }) - } - - async fn fetch_jwt( - network: FuelNetwork, - username: &str, - password: &str, - ) -> anyhow::Result { - let client = HttpClient::new(); - let json_body = serde_json::to_string(&LoginRequest { - username: username.to_string(), - password: password.to_string(), - })?; - - let api_url = network.to_web_url().join("/api/v1/jwt")?; - - let response = client - .get(api_url) - .header(ACCEPT, "application/json") - .header(CONTENT_TYPE, "application/json") - .body(json_body) - .send() - .await?; - - if response.status().is_success() { - let json_body = response.json::().await?; - Ok(json_body.jwt_token) - } else { - Err(anyhow::anyhow!( - "Failed to fetch JWT: {}", - response.status() - )) - } - } - - pub async fn refresh_jwt(&mut self) -> anyhow::Result<()> { - let jwt_token = - Self::fetch_jwt(self.network, &self.username, &self.password) - .await?; - self.jwt_token = Some(jwt_token); - Ok(()) - } - - pub async fn connect(&mut self) -> anyhow::Result<()> { - let host = self - .ws_url - .host_str() - .ok_or(anyhow::anyhow!("Unparsable ws host url"))?; - - let jwt_token = self - .jwt_token - .clone() - .ok_or(anyhow::anyhow!("JWT token is missing"))?; - - let mut request = self.ws_url.as_str().into_client_request()?; - let headers_map = request.headers_mut(); - headers_map - .insert(AUTHORIZATION, format!("Bearer {}", jwt_token).parse()?); - headers_map.insert(HOST, host.parse()?); - headers_map.insert(UPGRADE, "websocket".parse()?); - headers_map.insert(CONNECTION, "Upgrade".parse().unwrap()); - headers_map.insert(SEC_WEBSOCKET_KEY, generate_key().parse()?); - headers_map.insert(SEC_WEBSOCKET_VERSION, "13".parse()?); - - let (socket, _response) = connect_async(request).await?; - let (write, read) = socket.split(); - - self.read_stream = Some(read); - self.write_sink = Some(RwLock::new(write)); - - Ok(()) - } - - async fn send_client_message( - &mut self, - message: ClientMessage, - ) -> anyhow::Result<()> { - let write_sink = self - .write_sink - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Socket not connected"))?; - let mut write_guard = write_sink.write().await; - let serialized = serde_json::to_vec(&message)?; - write_guard.send(Message::Binary(serialized)).await?; - Ok(()) - } - - pub async fn subscribe( - &mut self, - wildcard: impl IntoSubject, - deliver_policy: DeliverPolicy, - ) -> anyhow::Result<()> { - let message = ClientMessage::Subscribe(SubscriptionPayload { - wildcard: wildcard.parse(), - deliver_policy, - }); - self.send_client_message(message).await?; - Ok(()) - } - - pub async fn unsubscribe( - &mut self, - wildcard: impl IntoSubject, - deliver_policy: DeliverPolicy, - ) -> anyhow::Result<()> { - let message = ClientMessage::Unsubscribe(SubscriptionPayload { - wildcard: wildcard.parse(), - deliver_policy, - }); - self.send_client_message(message).await?; - Ok(()) - } - - pub async fn listen( - &mut self, - ) -> anyhow::Result> { - let read_stream = self - .read_stream - .take() - .ok_or_else(|| anyhow::anyhow!("Socket not connected"))?; - let (tx, rx) = mpsc::unbounded_channel::(); - // TODO: the reason for using this type of channel is due to the fact that Streamable cannot be currently - // converted into a dynamic object trait, hence this approach of switching between types - tokio::spawn(async move { - let mut subscription_topic = String::new(); - let mut read_stream = read_stream; - while let Some(Ok(msg)) = read_stream.next().await { - match msg { - Message::Text(text) => { - println!("Received text: {:?} bytes", text.len()); - } - Message::Binary(bin) => { - let server_message = match serde_json::from_slice::< - ServerMessage, - >( - &bin - ) { - Ok(server_message) => server_message, - Err(e) => { - eprintln!("Unparsable server message: {:?}", e); - continue; - } - }; - - match &server_message { - ServerMessage::Subscribed(sub) => { - println!( - "Subscribed to wildcard: {:?}", - sub.wildcard - ); - subscription_topic = sub.wildcard.clone(); - } - ServerMessage::Unsubscribed(sub) => { - println!( - "Unsubscribed from wildcard: {:?}", - sub.wildcard - ); - } - ServerMessage::Update(update) => { - let _ = decode_print( - &subscription_topic, - update.clone(), - ) - .ok(); - // send server message over a channel to receivers - if tx.send(server_message).is_err() { - break; - } - } - ServerMessage::Error(err) => { - println!("Received error from ws: {:?}", err); - break; - } - } - } - Message::Ping(ping) => { - println!("Received ping: {:?} bytes", ping.len()); - } - Message::Pong(pong) => { - println!("Received pong: {:?} bytes", pong.len()); - } - Message::Close(close) => { - let close_code = close - .as_ref() - .map(|c| c.code.to_string()) - .unwrap_or_default(); - let close_reason = close - .as_ref() - .map(|c| c.reason.to_string()) - .unwrap_or_default(); - println!( - "Received close with code: {:?} and reason: {:?}", - close_code, close_reason - ); - break; - } - _ => { - eprintln!("Received unknown message type"); - break; - } - } - } - }); - - Ok(rx) - } -} - -pub fn decode_print( - subject_wildcard: &str, - s3_payload: Vec, -) -> Result<(), WsSubscriptionError> { - let subject = verify_and_extract_subject_name(subject_wildcard)?; - match subject.as_str() { - Transaction::NAME => { - let entity = serde_json::from_slice::(&s3_payload) - .map_err(WsSubscriptionError::UnparsablePayload)?; - println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); - } - Block::NAME => { - let entity = serde_json::from_slice::(&s3_payload) - .map_err(WsSubscriptionError::UnparsablePayload)?; - println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); - } - Input::NAME => { - let entity = serde_json::from_slice::(&s3_payload) - .map_err(WsSubscriptionError::UnparsablePayload)?; - println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); - } - Output::NAME => { - let entity = serde_json::from_slice::(&s3_payload) - .map_err(WsSubscriptionError::UnparsablePayload)?; - println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); - } - Receipt::NAME => { - let entity = serde_json::from_slice::(&s3_payload) - .map_err(WsSubscriptionError::UnparsablePayload)?; - println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); - } - Utxo::NAME => { - let entity = serde_json::from_slice::(&s3_payload) - .map_err(WsSubscriptionError::UnparsablePayload)?; - println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); - } - Log::NAME => { - let entity = serde_json::from_slice::(&s3_payload) - .map_err(WsSubscriptionError::UnparsablePayload)?; - println!("Update [{:?} bytes]-> {:?}", s3_payload.len(), entity); - } - _ => { - eprintln!("Unknown entity {:?}", subject.to_string()); - } - } - Ok(()) -} diff --git a/crates/sv-webserver/src/lib.rs b/crates/sv-webserver/src/lib.rs index e9c80f91..ed3faedc 100644 --- a/crates/sv-webserver/src/lib.rs +++ b/crates/sv-webserver/src/lib.rs @@ -1,5 +1,4 @@ pub mod cli; -pub mod client; pub mod config; pub mod server; pub mod telemetry; diff --git a/crates/sv-webserver/src/server/ws/models.rs b/crates/sv-webserver/src/server/ws/models.rs index 0dfcec38..3a296acf 100644 --- a/crates/sv-webserver/src/server/ws/models.rs +++ b/crates/sv-webserver/src/server/ws/models.rs @@ -1,4 +1,4 @@ -use fuel_streams_nats::DeliverPolicy as NatsDeliverPolicy; +use fuel_streams_nats::NatsDeliverPolicy; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] @@ -72,7 +72,7 @@ pub enum ClientMessage { pub enum ServerMessage { Subscribed(SubscriptionPayload), Unsubscribed(SubscriptionPayload), - Update(Vec), + Response(serde_json::Value), Error(String), } diff --git a/crates/sv-webserver/src/server/ws/socket.rs b/crates/sv-webserver/src/server/ws/socket.rs index 03512ed7..b07ad6b4 100644 --- a/crates/sv-webserver/src/server/ws/socket.rs +++ b/crates/sv-webserver/src/server/ws/socket.rs @@ -7,15 +7,8 @@ use actix_web::{ Responder, }; use actix_ws::{Message, Session}; -use fuel_streams::{ - logs::Log, - types::{Block, Input, Output, Receipt, Transaction}, - utxos::Utxo, - StreamEncoder, - Streamable, -}; use fuel_streams_core::prelude::*; -use fuel_streams_nats::DeliverPolicy; +use fuel_streams_nats::NatsDeliverPolicy; use futures::StreamExt; use uuid::Uuid; @@ -203,7 +196,7 @@ async fn handle_binary_message( // subscribe to the stream let config = SubscriptionConfig { - deliver_policy: DeliverPolicy::All, + deliver_policy: NatsDeliverPolicy::All, filter_subjects: vec![subject_wildcard.clone()], }; let mut sub = @@ -336,42 +329,50 @@ async fn decode( s3_payload: Vec, ) -> Result, WsSubscriptionError> { let subject = verify_and_extract_subject_name(subject_wildcard)?; - match subject.as_str() { + let entity = match subject.as_str() { Transaction::NAME => { let entity = Transaction::decode_or_panic(s3_payload); - serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload) + serde_json::to_value(entity) + .map_err(WsSubscriptionError::UnparsablePayload)? } Block::NAME => { let entity = Block::decode_or_panic(s3_payload); - serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload) + serde_json::to_value(entity) + .map_err(WsSubscriptionError::UnparsablePayload)? } Input::NAME => { let entity = Input::decode_or_panic(s3_payload); - serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload) + serde_json::to_value(entity) + .map_err(WsSubscriptionError::UnparsablePayload)? } Output::NAME => { let entity = Output::decode_or_panic(s3_payload); - serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload) + serde_json::to_value(entity) + .map_err(WsSubscriptionError::UnparsablePayload)? } Receipt::NAME => { let entity = Receipt::decode_or_panic(s3_payload); - serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload) + serde_json::to_value(entity) + .map_err(WsSubscriptionError::UnparsablePayload)? } Utxo::NAME => { let entity = Utxo::decode_or_panic(s3_payload); - serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload) + serde_json::to_value(entity) + .map_err(WsSubscriptionError::UnparsablePayload)? } Log::NAME => { let entity = Log::decode_or_panic(s3_payload); - serde_json::to_vec(&entity) - .map_err(WsSubscriptionError::UnparsablePayload) + serde_json::to_value(entity) + .map_err(WsSubscriptionError::UnparsablePayload)? } - _ => Err(WsSubscriptionError::UnknownSubjectName(subject.to_string())), - } + _ => { + return Err(WsSubscriptionError::UnknownSubjectName( + subject.to_string(), + )) + } + }; + + // Wrap the entity in ServerMessage::Response and serialize once + serde_json::to_vec(&ServerMessage::Response(entity)) + .map_err(WsSubscriptionError::UnserializableMessagePayload) } diff --git a/examples/Cargo.toml b/examples/Cargo.toml index c9a435e3..08fbbb7c 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -39,11 +39,3 @@ path = "utxos.rs" [[example]] name = "logs" path = "logs.rs" - -[[example]] -name = "multiple-streams" -path = "multiple-streams.rs" - -[[example]] -name = "websockets" -path = "websockets.rs" diff --git a/examples/blocks.rs b/examples/blocks.rs index 48e882e4..83a2a22f 100644 --- a/examples/blocks.rs +++ b/examples/blocks.rs @@ -20,31 +20,20 @@ use futures::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize a client connection to the Fuel streaming service - let client = Client::connect(FuelNetwork::Testnet).await?; + let mut client = Client::new(FuelNetwork::Local).await?; + let mut connection = client.connect().await?; - // Create a new stream for blocks - let stream = fuel_streams::Stream::::new(&client).await; - - // Configure the stream to start from the last published block - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; + println!("Listening for blocks..."); + let subject = BlocksSubject::new(); // Subscribe to the block stream with the specified configuration - let mut sub = stream.subscribe_raw_with_config(config).await?; - - println!("Listening for blocks..."); + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) + .await?; // Process incoming blocks - while let Some(bytes) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes).unwrap(); - let tx_subject = decoded_msg.subject; - let tx_published_at = decoded_msg.timestamp; - - println!( - "Received block:\n Subject: {}\n Published at: {}\n Block: {:?}\n", - tx_subject, tx_published_at, decoded_msg.payload - ); + while let Some(block) = stream.next().await { + println!("Received block: {:?}", block); } Ok(()) diff --git a/examples/inputs.rs b/examples/inputs.rs index da448d9f..5e874cdf 100644 --- a/examples/inputs.rs +++ b/examples/inputs.rs @@ -20,31 +20,20 @@ use futures::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize a client connection to the Fuel streaming service - let client = Client::connect(FuelNetwork::Testnet).await?; + let mut client = Client::new(FuelNetwork::Testnet).await?; + let mut connection = client.connect().await?; - // Create a new stream for inputs - let stream = fuel_streams::Stream::::new(&client).await; - - // Configure the stream to start from the last published input - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; + println!("Listening for inputs..."); + let subject = InputsCoinSubject::new(); // Subscribe to the input stream with the specified configuration - let mut sub = stream.subscribe_raw_with_config(config).await?; - - println!("Listening for inputs..."); + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) + .await?; // Process incoming inputs - while let Some(bytes) = sub.next().await { - let decoded_msg = Input::decode_raw(bytes).unwrap(); - let tx_subject = decoded_msg.subject; - let tx_published_at = decoded_msg.timestamp; - - println!( - "Received input:\n Subject: {}\n Published at: {}\n Input: {:?}\n", - tx_subject, tx_published_at, decoded_msg.payload - ); + while let Some(input) = stream.next().await { + println!("Received input: {:?}", input); } Ok(()) diff --git a/examples/logs.rs b/examples/logs.rs index f9e1100b..d703be4e 100644 --- a/examples/logs.rs +++ b/examples/logs.rs @@ -20,31 +20,20 @@ use futures::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize a client connection to the Fuel streaming service - let client = Client::connect(FuelNetwork::Testnet).await?; + let mut client = Client::new(FuelNetwork::Testnet).await?; + let mut connection = client.connect().await?; - // Create a new stream for logs - let stream = fuel_streams::Stream::::new(&client).await; - - // Configure the stream to start from the last published log - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; + println!("Listening for logs..."); + let subject = LogsSubject::new(); // Subscribe to the log stream with the specified configuration - let mut sub = stream.subscribe_raw_with_config(config).await?; - - println!("Listening for logs..."); + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) + .await?; // Process incoming logs - while let Some(bytes) = sub.next().await { - let decoded_msg = Log::decode_raw(bytes).unwrap(); - let log_subject = decoded_msg.subject; - let log_published_at = decoded_msg.timestamp; - - println!( - "Received log:\n Subject: {}\n Published at: {}\n Log: {:?}\n", - log_subject, log_published_at, decoded_msg.payload - ); + while let Some(log) = stream.next().await { + println!("Received log: {:?}", log); } Ok(()) diff --git a/examples/multiple-streams.rs b/examples/multiple-streams.rs deleted file mode 100644 index 7f5508f3..00000000 --- a/examples/multiple-streams.rs +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2024 Fuel Labs -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use fuel_core_types::fuel_tx::ContractId; -use fuel_streams::{ - client::Client, - subjects::*, - types::*, - Filter, - StreamConfig, - StreamEncoder, -}; -use futures::{future::try_join_all, StreamExt}; - -// This example demonstrates how to use the fuel-streams library to subscribe to multiple streams. -#[tokio::main] -async fn main() -> Result<(), anyhow::Error> { - // initialize a client - let client = Client::connect(FuelNetwork::Testnet).await?; - - let mut handles = vec![]; - - // stream blocks - let stream_client = client.clone(); - handles.push(tokio::spawn(async move { - stream_blocks(&stream_client, None).await.unwrap(); - })); - - // stream blocks with filter - let stream_client = client.clone(); - handles.push(tokio::spawn(async move { - let filter = Filter::::build() - .with_producer(Some(Address::zeroed())) - .with_height(Some(5.into())); - stream_blocks(&stream_client, Some(filter)).await.unwrap(); - })); - - // stream transactions - let txs_client = client.clone(); - handles.push(tokio::spawn(async move { - stream_transactions(&txs_client, None).await.unwrap(); - })); - - // stream transactions with filter - let txs_client = client.clone(); - handles.push(tokio::spawn(async move { - let filter = Filter::::build() - .with_block_height(Some(5.into())) - .with_kind(Some(TransactionKind::Mint)); - stream_transactions(&txs_client, Some(filter)) - .await - .unwrap(); - })); - - // stream contract receipts - handles.push(tokio::spawn({ - let contract_client = client.clone(); - // Replace with an actual contract ID - let contract_id = ContractId::from([0u8; 32]); - async move { - stream_contract(&contract_client, contract_id) - .await - .unwrap(); - } - })); - - // stream transactions by contract ID - handles.push(tokio::spawn({ - let txs_client = client.clone(); - // Replace with an actual contract ID - let contract_id = ContractId::from([0u8; 32]); - async move { - stream_transactions_by_contract(&txs_client, contract_id) - .await - .unwrap(); - } - })); - - // stream inputs by contract ID - handles.push(tokio::spawn({ - let inputs_client = client.clone(); - // Replace with an actual contract ID - let contract_id = ContractId::from([0u8; 32]); - async move { - stream_inputs_by_contract(&inputs_client, contract_id) - .await - .unwrap(); - } - })); - - // stream receipts by contract ID - handles.push(tokio::spawn({ - let receipts_client = client.clone(); - // Replace with an actual contract ID - let contract_id = ContractId::from([0u8; 32]); - async move { - stream_receipts_by_contract(&receipts_client, contract_id) - .await - .unwrap(); - } - })); - - // await all handles - try_join_all(handles).await?; - - Ok(()) -} - -async fn stream_blocks( - client: &Client, - filter: Option, -) -> anyhow::Result<()> { - let mut block_stream = fuel_streams::Stream::::new(client).await; - - let mut sub = match filter { - Some(filter) => { - block_stream.with_filter(filter).subscribe_raw().await? - } - None => block_stream.subscribe_raw().await?, - }; - while let Some(bytes) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes).unwrap(); - let block_height = decoded_msg.payload.height; - let block_subject = decoded_msg.subject; - let block_published_at = decoded_msg.timestamp; - println!( - "Received block: height={}, subject={}, published_at={}", - block_height, block_subject, block_published_at - ) - } - - Ok(()) -} - -async fn stream_transactions( - client: &Client, - filter: Option, -) -> anyhow::Result<()> { - let mut txs_stream = fuel_streams::Stream::::new(client).await; - - // here we apply a config to the streaming to start getting only from the last published transaction onwards - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; - - let mut sub = match filter { - Some(filter) => { - txs_stream - .with_filter(filter) - .subscribe_raw_with_config(config) - .await? - } - None => txs_stream.subscribe_raw_with_config(config).await?, - }; - - while let Some(bytes) = sub.next().await { - let decoded_msg = Transaction::decode_raw(bytes).unwrap(); - let tx = decoded_msg.payload; - let tx_subject = decoded_msg.subject; - let tx_published_at = decoded_msg.timestamp; - println!( - "Received transaction: data={:?}, subject={}, published_at={}", - tx, tx_subject, tx_published_at - ) - } - Ok(()) -} - -/// Streams transactions associated with a specific contract ID. -/// -/// This function creates a filtered stream of transactions related to the given contract ID -/// and processes each received transaction by printing its details. -/// -/// # Arguments -/// -/// * `client` - A reference to the NATS client used for streaming. -/// * `contract_id` - The `ContractId` to filter transactions by. -/// -/// # Returns -/// -/// Returns `Ok(())` if the stream processes successfully, or an error if there are any issues. -async fn stream_transactions_by_contract( - client: &Client, - contract_id: ContractId, -) -> anyhow::Result<()> { - let mut txs_stream = fuel_streams::Stream::::new(client).await; - - // Build a filter for transactions by contract ID - let filter = Filter::::build() - .with_id_kind(Some(IdentifierKind::ContractID)) - .with_id_value(Some((*contract_id).into())); - - // Filtered stream - let mut sub = txs_stream.with_filter(filter).subscribe_raw().await?; - - while let Some(bytes) = sub.next().await { - let decoded_msg = Transaction::decode_raw(bytes).unwrap(); - let tx = decoded_msg.payload; - let tx_subject = decoded_msg.subject; - let tx_published_at = decoded_msg.timestamp; - println!( - "Received transaction for contract: data={:?}, subject={}, published_at={}", - tx, tx_subject, tx_published_at - ); - } - Ok(()) -} - -/// Subscribes to receipts related to a specific contract, effectively listening to contract events. -/// -/// This function creates a stream that subscribes to various types of receipts (except `ScriptResult` -/// and `MessageOut`) that are associated with the specified contract ID. It's a way to monitor -/// contract-related events such as calls, returns, logs, transfers, mints, and burns. -/// -/// The function filters the receipts to ensure they match the given contract ID before processing them. -/// This approach allows for efficient monitoring of contract activities without the need to process -/// irrelevant receipts. -/// -/// # Arguments -/// -/// * `client` - A reference to the NATS client used for streaming. -/// * `contract_id` - The ID of the contract to monitor. -/// -/// # Returns -/// -/// Returns `Ok(())` if the streaming completes successfully, or an error if there are any issues. -async fn stream_contract( - client: &Client, - contract_id: ContractId, -) -> anyhow::Result<()> { - let mut receipt_stream = fuel_streams::Stream::::new(client).await; - - // Set up filters for all receipt types that can be associated with a contract - receipt_stream.with_filter( - ReceiptsBurnSubject::new().with_contract_id(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsCallSubject::new().with_from(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsReturnSubject::new().with_id(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsReturnDataSubject::new().with_id(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsPanicSubject::new().with_id(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsRevertSubject::new().with_id(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsLogSubject::new().with_id(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsLogDataSubject::new().with_id(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsTransferSubject::new().with_from(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsTransferOutSubject::new().with_from(Some(contract_id.into())), - ); - receipt_stream.with_filter( - ReceiptsMintSubject::new().with_contract_id(Some(contract_id.into())), - ); - - let mut sub = receipt_stream.subscribe_raw().await?; - while let Some(bytes) = sub.next().await { - let decoded_msg = Receipt::decode_raw(bytes).unwrap(); - let receipt = decoded_msg.payload; - - // Check if this is a contract-related receipt and matches our target - let should_process = match &receipt { - Receipt::Call(r) => { - r.id == contract_id.into() || r.to == contract_id.into() - } - Receipt::Return(r) => r.id == contract_id.into(), - Receipt::ReturnData(r) => r.id == contract_id.into(), - Receipt::Panic(r) => r.id == contract_id.into(), - Receipt::Revert(r) => r.id == contract_id.into(), - Receipt::Log(r) => r.id == contract_id.into(), - Receipt::LogData(r) => r.id == contract_id.into(), - Receipt::Transfer(r) => { - r.id == contract_id.into() || r.to == contract_id.into() - } - Receipt::TransferOut(r) => r.id == contract_id.into(), - Receipt::Mint(r) => r.contract_id == contract_id.into(), - Receipt::Burn(r) => r.contract_id == contract_id.into(), - Receipt::ScriptResult(_) | Receipt::MessageOut(_) => false, - }; - - if should_process { - let receipt_subject = decoded_msg.subject; - let receipt_published_at = decoded_msg.timestamp; - println!( - "Received contract receipt: data={:?}, subject={}, published_at={}", - receipt, receipt_subject, receipt_published_at - ); - } - } - - Ok(()) -} - -/// Streams inputs related to a specific contract ID. -/// -/// This function creates a filtered stream of inputs associated with the given contract ID -/// and processes each received input by printing its details. -/// -/// # Arguments -/// -/// * `client` - A reference to the NATS client used for streaming. -/// * `contract_id` - The `ContractId` to filter inputs by. -/// -/// # Returns -/// -/// Returns `Ok(())` if the stream processes successfully, or an error if there are any issues. -async fn stream_inputs_by_contract( - client: &Client, - contract_id: ContractId, -) -> anyhow::Result<()> { - let mut inputs_stream = fuel_streams::Stream::::new(client).await; - - inputs_stream.with_filter( - InputsByIdSubject::new() - .with_id_kind(Some(IdentifierKind::ContractID)) - .with_id_value(Some((*contract_id).into())), - ); - - let mut sub = inputs_stream.subscribe_raw().await?; - - while let Some(bytes) = sub.next().await { - let decoded_msg = Input::decode_raw(bytes).unwrap(); - let input = decoded_msg.payload; - let input_subject = decoded_msg.subject; - let input_published_at = decoded_msg.timestamp; - println!( - "Received input for contract: data={:?}, subject={}, published_at={}", - input, input_subject, input_published_at - ); - } - - Ok(()) -} - -/// Streams receipts associated with a specific contract ID. -/// -/// This function creates a filtered stream of receipts related to the given contract ID -/// and processes each received receipt by printing its details. -/// -/// # Arguments -/// -/// * `client` - A reference to the NATS client used for streaming. -/// * `contract_id` - The `ContractId` to filter receipts by. -/// -/// # Returns -/// -/// Returns `Ok(())` if the stream processes successfully, or an error if there are any issues. -async fn stream_receipts_by_contract( - client: &Client, - contract_id: ContractId, -) -> anyhow::Result<()> { - let mut receipt_stream = fuel_streams::Stream::::new(client).await; - - receipt_stream.with_filter( - ReceiptsByIdSubject::new() - .with_id_kind(Some(IdentifierKind::ContractID)) - .with_id_value(Some((*contract_id).into())), - ); - - let mut sub = receipt_stream.subscribe_raw().await?; - - while let Some(bytes) = sub.next().await { - let decoded_msg = Receipt::decode_raw(bytes).unwrap(); - let receipt = decoded_msg.payload; - let receipt_subject = decoded_msg.subject; - let receipt_published_at = decoded_msg.timestamp; - println!( - "Received receipt for contract: data={:?}, subject={}, published_at={}", - receipt, receipt_subject, receipt_published_at - ); - } - - Ok(()) -} diff --git a/examples/outputs.rs b/examples/outputs.rs index f9836030..6e069d52 100644 --- a/examples/outputs.rs +++ b/examples/outputs.rs @@ -20,31 +20,20 @@ use futures::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize a client connection to the Fuel streaming service - let client = Client::connect(FuelNetwork::Testnet).await?; + let mut client = Client::new(FuelNetwork::Testnet).await?; + let mut connection = client.connect().await?; - // Create a new stream for outputs - let stream = fuel_streams::Stream::::new(&client).await; - - // Configure the stream to start from the last published output - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; + println!("Listening for outputs..."); + let subject = OutputsCoinSubject::new(); // Subscribe to the output stream with the specified configuration - let mut sub = stream.subscribe_raw_with_config(config).await?; - - println!("Listening for outputs..."); + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) + .await?; // Process incoming outputs - while let Some(bytes) = sub.next().await { - let decoded_msg = Output::decode_raw(bytes).unwrap(); - let tx_subject = decoded_msg.subject; - let tx_published_at = decoded_msg.timestamp; - - println!( - "Received output:\n Subject: {}\n Published at: {}\n Output: {:?}\n", - tx_subject, tx_published_at, decoded_msg.payload - ); + while let Some(output) = stream.next().await { + println!("Received output: {:?}", output); } Ok(()) diff --git a/examples/receipts.rs b/examples/receipts.rs index 09430779..ae0f38dc 100644 --- a/examples/receipts.rs +++ b/examples/receipts.rs @@ -11,106 +11,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::Result; -use fuel_streams::{prelude::*, receipts::*}; +use fuel_streams::prelude::*; use futures::StreamExt; /// The contract ID to stream the receipts for. For this example, we're using the contract ID of the https://thundernft.market/ const CONTRACT_ID: &str = "0x243ef4c2301f44eecbeaf1c39fee9379664b59a2e5b75317e8c7e7f26a25ed4d"; -/// Subscribes to receipts related to a specific contract, effectively listening to contract events. -/// -/// This function creates a stream that subscribes to various types of receipts -/// -/// The function filters the receipts to ensure they match the given contract ID before processing them. -/// This approach allows for efficient monitoring of contract activities without the need to process -/// irrelevant receipts. -/// -/// # Arguments -/// -/// * `client` - A reference to the NATS client used for streaming. -/// * `contract_id` - The ID of the contract to monitor. -/// -/// # Returns -/// -/// Returns `Ok(())` if the streaming completes successfully, or an error if there are any issues. - -// This example demonstrates how to use the fuel-streams library to stream -// receipts from a Fuel network. It connects to a streaming service, -// subscribes to a receipt for a given contract, and prints incoming receipts. #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> anyhow::Result<()> { // Initialize a client connection to the Fuel streaming service - let client = Client::connect(FuelNetwork::Testnet).await?; - - let contract_id: ContractId = CONTRACT_ID.into(); - - // Create a new stream for receipts - let mut receipt_stream = - fuel_streams::Stream::::new(&client).await; + let mut client = Client::new(FuelNetwork::Testnet).await?; + let mut connection = client.connect().await?; - // Use multiple filters to subscribe to different types of receipts (all receipts except - // `ScriptResult` and `MessageOut`) that are associated with the specified contract ID. - // It's a way to monitor all contract-related events such as calls, returns, logs, transfers, - // mints, and burns. - receipt_stream.with_filter( - ReceiptsBurnSubject::default() - .with_contract_id(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsCallSubject::default().with_from(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsReturnSubject::default().with_id(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsReturnDataSubject::default().with_id(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsPanicSubject::default().with_id(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsRevertSubject::default().with_id(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsLogSubject::default().with_id(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsLogDataSubject::default().with_id(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsTransferSubject::default().with_from(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsTransferOutSubject::default() - .with_from(Some(contract_id.clone())), - ); - receipt_stream.with_filter( - ReceiptsMintSubject::default() - .with_contract_id(Some(contract_id.clone())), - ); - - // Configure the stream to start from the first published receipt - let config = StreamConfig { - deliver_policy: DeliverPolicy::All, - }; + println!("Listening for receipts..."); - // Subscribe to the receipt stream - let mut sub = receipt_stream.subscribe_raw_with_config(config).await?; + // Create a subject for all receipt types related to the contract + let subject = ReceiptsByIdSubject::new() + .with_id_kind(Some(IdentifierKind::ContractID)) + .with_id_value(Some(CONTRACT_ID.into())); - println!("Listening for receipts..."); + // Subscribe to the receipt stream with the specified configuration + let mut stream = connection + .subscribe::(subject, DeliverPolicy::All) + .await?; // Process incoming receipts - while let Some(bytes) = sub.next().await { - let decoded_msg = Receipt::decode_raw(bytes).unwrap(); - let receipt = decoded_msg.payload; - let receipt_subject = decoded_msg.subject; - let receipt_published_at = decoded_msg.timestamp; - println!( - "Received receipt:\n Subject: {}\n Published at: {}\n Data: {:?}\n", - receipt_subject, receipt_published_at, receipt - ); + while let Some(receipt) = stream.next().await { + println!("Received receipt: {:?}", receipt); } Ok(()) diff --git a/examples/transactions.rs b/examples/transactions.rs index 44025b13..0b8b8585 100644 --- a/examples/transactions.rs +++ b/examples/transactions.rs @@ -20,32 +20,23 @@ use futures::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize a client connection to the Fuel streaming service - let client = Client::connect(FuelNetwork::Testnet).await?; + let mut client = Client::new(FuelNetwork::Testnet).await?; + let mut connection = client.connect().await?; - // Create a new stream for transactions - let stream = fuel_streams::Stream::::new(&client).await; + println!("Listening for transactions..."); - // Configure the stream to start from the last published transaction - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; + // Create a subject for all transactions + let subject = + TransactionsSubject::new().with_kind(Some(TransactionKind::Script)); // Example: filter for script transactions // Subscribe to the transaction stream with the specified configuration - let mut sub = stream.subscribe_raw_with_config(config).await?; - - println!("Listening for transactions..."); + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) + .await?; // Process incoming transactions - while let Some(bytes) = sub.next().await { - let decoded_msg = Transaction::decode_raw(bytes).unwrap(); - let tx = decoded_msg.payload; - let tx_subject = decoded_msg.subject; - let tx_published_at = decoded_msg.timestamp; - - println!( - "Received transaction:\n Subject: {}\n Published at: {}\n Data: {:?}\n", - tx_subject, tx_published_at, tx - ); + while let Some(transaction) = stream.next().await { + println!("Received transaction: {:?}", transaction); } Ok(()) diff --git a/examples/utxos.rs b/examples/utxos.rs index ea46b64e..052436e5 100644 --- a/examples/utxos.rs +++ b/examples/utxos.rs @@ -20,31 +20,22 @@ use futures::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize a client connection to the Fuel streaming service - let client = Client::connect(FuelNetwork::Testnet).await?; + let mut client = Client::new(FuelNetwork::Testnet).await?; + let mut connection = client.connect().await?; - // Create a new stream for UTXOs - let stream = fuel_streams::Stream::::new(&client).await; + println!("Listening for UTXOs..."); - // Configure the stream to start from the last published UTXO - let config = StreamConfig { - deliver_policy: DeliverPolicy::Last, - }; + // Create a subject for all UTXOs, optionally filter by type + let subject = UtxosSubject::new().with_utxo_type(Some(UtxoType::Message)); // Example: filter for message UTXOs // Subscribe to the UTXO stream with the specified configuration - let mut sub = stream.subscribe_raw_with_config(config).await?; - - println!("Listening for UTXOs..."); + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) + .await?; // Process incoming UTXOs - while let Some(bytes) = sub.next().await { - let decoded_msg = Utxo::decode_raw(bytes).unwrap(); - let utxo_subject = decoded_msg.subject; - let utxo_published_at = decoded_msg.timestamp; - - println!( - "Received UTXO:\n Subject: {}\n Published at: {}\n UTXO: {:?}\n", - utxo_subject, utxo_published_at, decoded_msg.payload - ); + while let Some(utxo) = stream.next().await { + println!("Received UTXO: {:?}", utxo); } Ok(()) diff --git a/examples/websockets.rs b/examples/websockets.rs deleted file mode 100755 index c8de232a..00000000 --- a/examples/websockets.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2024 Fuel Labs -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::time::Duration; - -use fuel_streams::{ - blocks::BlocksSubject, - subjects::SubjectBuildable, - types::FuelNetwork, -}; -use sv_webserver::{ - client::WebSocketClient, - server::ws::models::DeliverPolicy, -}; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let mut client = - WebSocketClient::new(FuelNetwork::Local, "admin", "admin").await?; - - client.connect().await?; - - let subject = BlocksSubject::new(); - let deliver_policy = DeliverPolicy::New; - // .with_producer(Some(Address::zeroed())) - // .with_height(Some(183603.into())); - - println!("Subscribing to subject {:?} ...", subject); - client.subscribe(subject.clone(), deliver_policy).await?; - - let mut receiver = client.listen().await?; - - tokio::spawn(async move { - while let Some(_message) = receiver.recv().await { - // println!("Received: {:?}", message); - } - }); - - tokio::time::sleep(Duration::from_secs(15)).await; - - println!("Unsubscribing to subject {:?} ...", subject); - client.unsubscribe(subject, deliver_policy).await?; - - Ok(()) -} diff --git a/scripts/run_webserver.sh b/scripts/run_webserver.sh index 91845a8a..5b91c749 100755 --- a/scripts/run_webserver.sh +++ b/scripts/run_webserver.sh @@ -3,6 +3,12 @@ # Exit immediately if a command exits with a non-zero status set -e +# Load environment variables with defaults +PORT=${PORT:-9003} +NATS_URL=${NATS_URL:-nats://localhost:4222} +MODE=${MODE:-dev} +EXTRA_ARGS=${EXTRA_ARGS:-""} + # ------------------------------ # Function to Display Usage # ------------------------------ @@ -49,16 +55,10 @@ while [[ "$#" -gt 0 ]]; do esac done -# Load environment variables with defaults -PORT=${PORT:-9003} -NATS_URL=${NATS_URL:-nats://localhost:4222} -MODE=${MODE:-dev} -EXTRA_ARGS=${EXTRA_ARGS:-""} - # ------------------------------ # Load Environment # ------------------------------ -source ./scripts/set_env.sh +source ./scripts/set_env.sh NATS_URL=${NATS_URL} # Print the configuration being used echo -e "\n==========================================" diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 747eb520..90079962 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -17,10 +17,6 @@ harness = true name = "integration_tests" path = "tests/lib.rs" -[[bin]] -name = "special_integration_tests" -path = "src/main.rs" - [dependencies] fuel-core = { workspace = true, features = ["test-helpers"] } fuel-streams = { workspace = true, features = ["test-helpers"] } diff --git a/tests/src/lib.rs b/tests/src/lib.rs index bbce5580..394fc05c 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -1,6 +1,6 @@ use std::{sync::Arc, time::Duration}; -use fuel_streams::client::Client; +use fuel_streams::{client::Client, Connection, FuelNetwork}; use fuel_streams_core::{ nats::NatsClient, prelude::*, @@ -35,7 +35,7 @@ impl Streams { } } -pub async fn server_setup() -> BoxedResult<(NatsClient, Streams, Client)> { +pub async fn server_setup() -> BoxedResult<(NatsClient, Streams, Connection)> { let nats_client_opts = NatsClientOpts::admin_opts().with_rdn_namespace(); let nats_client = NatsClient::connect(&nats_client_opts).await?; @@ -45,11 +45,10 @@ pub async fn server_setup() -> BoxedResult<(NatsClient, Streams, Client)> { let streams = Streams::new(&nats_client, &s3_client).await; - let client = Client::with_opts(&nats_client_opts, &s3_client_opts) - .await - .unwrap(); + let mut client = Client::new(FuelNetwork::Local).await?; + let connection = client.connect().await?; - Ok((nats_client, streams, client)) + Ok((nats_client, streams, connection)) } pub fn publish_items( diff --git a/tests/src/main.rs b/tests/src/main.rs deleted file mode 100644 index e48fa797..00000000 --- a/tests/src/main.rs +++ /dev/null @@ -1,142 +0,0 @@ -use std::{ - env, - fs, - ops::Range, - path::{Path, PathBuf}, - process::Command, - time::Duration, -}; - -use fuel_streams::prelude::*; -use fuel_streams_core::prelude::*; -use futures::StreamExt; -use rand::Rng; -use streams_tests::{publish_blocks, server_setup}; - -const INTERVAL: Range = 10..15; - -fn find_workspace_root() -> Option { - let mut current_dir = env::current_dir().ok()?; - - loop { - if current_dir.join("Cargo.toml").exists() { - // Check if this is a workspace root - let cargo_toml = current_dir.join("Cargo.toml"); - let content = fs::read_to_string(&cargo_toml).ok()?; - if content.contains("[workspace]") { - return Some(current_dir); - } - } - - if !current_dir.pop() { - break; - } - } - - None -} - -fn start_nats(makefile_path: &Path) { - let status = Command::new("make") - .arg("-f") - .arg(makefile_path.to_str().unwrap()) - .arg("cluster_up") - .status() - .expect("Failed to start NATS"); - - if status.success() { - println!("NATS started successfully."); - } else { - println!("Failed to start NATS."); - } -} - -fn stop_nats(makefile_path: &Path) { - let status = Command::new("make") - .arg("-f") - .arg(makefile_path.to_str().unwrap()) - .arg("cluster_up") - .status() - .expect("Failed to stop NATS"); - - if status.success() { - println!("NATS stopped successfully."); - } else { - println!("Failed to stop NATS."); - } -} - -#[tokio::main] -async fn main() -> BoxedResult<()> { - let workspace_root = - find_workspace_root().expect("Could not find the workspace root"); - let makefile_path = workspace_root.join("Makefile"); - env::set_current_dir(&workspace_root) - .expect("Failed to change directory to workspace root"); - - // ensure nats is connected and running - let client_opts = NatsClientOpts::admin_opts() - .with_rdn_namespace() - .with_timeout(1); - - let s3_client_opts = S3ClientOpts::admin_opts(); - - let is_connected = Client::with_opts(&client_opts, &s3_client_opts) - .await - .ok() - .map(|c| c.nats_conn.is_connected()) - .unwrap_or_default(); - if !is_connected { - println!("Starting nats ..."); - start_nats(&makefile_path); - } - - // create a subscription - let (_, _, client) = server_setup().await.unwrap(); - let stream = fuel_streams::Stream::::new(&client).await; - let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - - // publish all items in a separate thread - let (items, publish_join_handle) = - publish_blocks(stream.stream(), Some(Address::zeroed()), None).unwrap(); - - // await publishing to finish - publish_join_handle.await.unwrap(); - println!("All items pushed to nats !"); - - let mut rng = rand::thread_rng(); - let mut action_interval = - tokio::time::interval(Duration::from_secs(rng.gen_range(INTERVAL))); - - loop { - tokio::select! { - Some((index, bytes)) = sub.next() => { - println!("Valid subscription"); - let decoded_msg = Block::decode_raw(bytes).unwrap(); - let (subject, block) = items[index].to_owned(); - let height = decoded_msg.payload.height; - assert_eq!(decoded_msg.subject, subject.parse()); - assert_eq!(decoded_msg.payload, block); - assert_eq!(height, index as u32); - if index == 9 { - break; - } - } - _ = action_interval.tick() => { - let client_opts = NatsClientOpts::admin_opts() - .with_rdn_namespace() - .with_timeout(1); - - let is_nats_connected = Client::with_opts(&client_opts, &s3_client_opts).await.ok().map(|c| c.nats_conn.is_connected()).unwrap_or_default(); - if is_nats_connected { - stop_nats(&makefile_path); - } else { - start_nats(&makefile_path); - } - action_interval = tokio::time::interval(Duration::from_secs(rng.gen_range(INTERVAL))); - } - } - } - - Ok(()) -} diff --git a/tests/tests/client.rs b/tests/tests/client.rs index 29ffcb5d..04e9c880 100644 --- a/tests/tests/client.rs +++ b/tests/tests/client.rs @@ -1,342 +1,342 @@ -use std::{collections::HashSet, sync::Arc, time::Duration}; - -use fuel_streams::prelude::*; -use fuel_streams_core::prelude::{types, *}; -use futures::{ - future::{try_join_all, BoxFuture}, - FutureExt, - StreamExt, - TryStreamExt, -}; -use rand::{distributions::Alphanumeric, Rng}; -use streams_tests::{publish_blocks, server_setup}; -use tokio::time::timeout; - -fn gen_random_string(size: usize) -> String { - rand::thread_rng() - .sample_iter(&Alphanumeric) - .take(size) - .map(char::from) - .collect() -} - -#[tokio::test] -async fn conn_streams_has_required_streams() -> BoxedResult<()> { - let (client, streams, _) = server_setup().await.unwrap(); - let mut context_streams = client.jetstream.stream_names(); - - let mut names = HashSet::new(); - while let Some(name) = context_streams.try_next().await? { - names.insert(name); - } - streams.blocks.assert_has_stream(&names).await; - streams.transactions.assert_has_stream(&names).await; - - for name in names.iter() { - let empty = streams.blocks.is_empty(name).await; - assert!(empty, "stream must be empty after creation"); - } - Ok(()) -} - -#[tokio::test] -async fn fuel_streams_client_connection() -> BoxedResult<()> { - let nats_opts = NatsClientOpts::admin_opts(); - let client = NatsClient::connect(&nats_opts).await?; - assert!(client.is_connected()); - let s3_opts = Arc::new(S3ClientOpts::admin_opts()); - let client = Client::with_opts(&nats_opts, &s3_opts).await?; - assert!(client.nats_conn.is_connected()); - Ok(()) -} - -#[tokio::test] -async fn multiple_client_connections() -> BoxedResult<()> { - let nats_opts = NatsClientOpts::admin_opts(); - let s3_opts = Arc::new(S3ClientOpts::admin_opts()); - let tasks: Vec<_> = (0..100) - .map(|_| { - let nats_opts = nats_opts.clone(); - let s3_opts = s3_opts.clone(); - async move { - let client = - Client::with_opts(&nats_opts, &s3_opts).await.unwrap(); - assert!(client.nats_conn.is_connected()); - Ok::<(), NatsError>(()) - } - }) - .collect(); - - assert!(try_join_all(tasks).await.is_ok()); - Ok(()) -} - -#[tokio::test] -async fn public_user_cannot_create_streams() -> BoxedResult<()> { - let network = FuelNetwork::Local; - let opts = NatsClientOpts::public_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - let client = NatsClient::connect(&opts).await?; - let (random_stream_title, random_subject) = - (gen_random_string(6), gen_random_string(6)); - - assert!(client - .jetstream - .create_stream(types::NatsStreamConfig { - name: random_stream_title, - subjects: vec![random_subject], - ..Default::default() - }) - .await - .is_err()); - - Ok(()) -} - -#[tokio::test] -async fn public_user_cannot_create_stores() -> BoxedResult<()> { - let network = FuelNetwork::Local; - let opts = NatsClientOpts::public_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - - let random_bucket_title = gen_random_string(6); - - let client = NatsClient::connect(&opts).await?; - assert!(client - .jetstream - .create_key_value(types::KvStoreConfig { - bucket: random_bucket_title, - ..Default::default() - }) - .await - .is_err()); - - Ok(()) -} - -#[tokio::test] -async fn public_user_cannot_delete_stores() -> BoxedResult<()> { - let network = FuelNetwork::Local; - let opts = NatsClientOpts::admin_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - - let random_bucket_title = gen_random_string(6); - - let client = NatsClient::connect(&opts).await?; - client - .jetstream - .create_key_value(types::KvStoreConfig { - bucket: random_bucket_title.clone(), - ..Default::default() - }) - .await?; - - let opts = NatsClientOpts::public_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - let client = NatsClient::connect(&opts).await?; - - assert!(client - .jetstream - .delete_key_value(&random_bucket_title) - .await - .is_err()); - - Ok(()) -} - -#[tokio::test] -async fn public_user_cannot_delete_stream() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts() - .with_rdn_namespace() - .with_timeout(1); - let client = NatsClient::connect(&opts).await?; - - let (random_stream_title, random_subject) = - (gen_random_string(6), gen_random_string(6)); - - client - .jetstream - .create_stream(types::NatsStreamConfig { - name: random_stream_title.clone(), - subjects: vec![random_subject], - ..Default::default() - }) - .await?; - - let network = FuelNetwork::Local; - let public_opts = - NatsClientOpts::public_opts().with_url(network.to_nats_url()); - let public_client = NatsClient::connect(&public_opts).await?; - - assert!( - public_client - .jetstream - .delete_stream(&random_stream_title) - .await - .is_err(), - "Stream must be deleted at this point" - ); - - Ok(()) -} - -#[tokio::test] -async fn public_user_can_access_streams_after_created() { - let network = FuelNetwork::Local; - let admin_opts = NatsClientOpts::admin_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - - let public_opts = NatsClientOpts::public_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - - assert!(NatsClient::connect(&admin_opts).await.is_ok()); - assert!(NatsClient::connect(&public_opts).await.is_ok()); -} - -#[tokio::test] -async fn public_and_admin_user_can_access_streams_after_created( -) -> BoxedResult<()> { - let network = FuelNetwork::Local; - let admin_opts = NatsClientOpts::admin_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - let s3_opts = Arc::new(S3ClientOpts::admin_opts()); - let admin_tasks: Vec>> = (0..100) - .map(|_| { - let opts: NatsClientOpts = admin_opts.clone(); - let s3_opts = s3_opts.clone(); - async move { - let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); - assert!(client.nats_conn.is_connected()); - Ok::<(), NatsError>(()) - } - .boxed() - }) - .collect(); - - let public_opts = NatsClientOpts::public_opts() - .with_url(network.to_nats_url()) - .with_rdn_namespace() - .with_timeout(1); - let s3_public_opts = - Arc::new(S3ClientOpts::new(S3Env::Local, S3Role::Public)); - let public_tasks: Vec>> = (0..100) - .map(|_| { - let opts: NatsClientOpts = public_opts.clone(); - let s3_opts = s3_public_opts.clone(); - async move { - let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); - assert!(client.nats_conn.is_connected()); - Ok::<(), NatsError>(()) - } - .boxed() - }) - .collect(); - - // Combine both vectors into one - let mut all_tasks = - Vec::with_capacity(admin_tasks.len() + public_tasks.len()); - all_tasks.extend(admin_tasks); - all_tasks.extend(public_tasks); - - assert!(try_join_all(all_tasks).await.is_ok()); - Ok(()) -} - -#[tokio::test] -async fn admin_user_can_delete_stream() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts() - .with_rdn_namespace() - .with_timeout(1); - let client = NatsClient::connect(&opts).await?; - - let (random_stream_title, random_subject) = - (gen_random_string(6), gen_random_string(6)); - - client - .jetstream - .create_stream(types::NatsStreamConfig { - name: random_stream_title.clone(), - subjects: vec![random_subject], - ..Default::default() - }) - .await?; - - let status = client.jetstream.delete_stream(&random_stream_title).await?; - assert!(status.success, "Stream must be deleted at this point"); - - Ok(()) -} - -#[tokio::test] -async fn admin_user_can_delete_stores() -> BoxedResult<()> { - let opts = NatsClientOpts::admin_opts() - .with_rdn_namespace() - .with_timeout(1); - - let random_bucket_title = gen_random_string(6); - - let client = NatsClient::connect(&opts).await?; - client - .jetstream - .create_key_value(types::KvStoreConfig { - bucket: random_bucket_title.clone(), - ..Default::default() - }) - .await?; - - assert!(client - .jetstream - .delete_key_value(&random_bucket_title) - .await - .is_ok()); - - Ok(()) -} - -#[tokio::test] -async fn ensure_deduplication_when_publishing() -> BoxedResult<()> { - let (_, _, client) = server_setup().await.unwrap(); - let stream = fuel_streams::Stream::::new(&client).await; - let producer = Some(Address::zeroed()); - let const_block_height = 1001; - let items = - publish_blocks(stream.stream(), producer, Some(const_block_height)) - .unwrap() - .0; - - let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - let timeout_duration = Duration::from_secs(1); - - // ensure just one message was published - 'l: loop { - match timeout(timeout_duration, sub.next()).await { - Ok(Some((idx, entry))) => { - let decoded_msg = Block::decode_raw(entry).unwrap(); - let (subject, _block) = items[idx].to_owned(); - let height = decoded_msg.payload.height; - assert_eq!(decoded_msg.subject, subject.parse()); - assert_eq!(height, const_block_height); - assert!(idx < 1); - } - _ => { - break 'l; - } - } - } - - Ok(()) -} +// use std::{collections::HashSet, sync::Arc, time::Duration}; + +// use fuel_streams::prelude::*; +// use fuel_streams_core::prelude::{types, *}; +// use futures::{ +// future::{try_join_all, BoxFuture}, +// FutureExt, +// StreamExt, +// TryStreamExt, +// }; +// use rand::{distributions::Alphanumeric, Rng}; +// use streams_tests::{publish_blocks, server_setup}; +// use tokio::time::timeout; + +// fn gen_random_string(size: usize) -> String { +// rand::thread_rng() +// .sample_iter(&Alphanumeric) +// .take(size) +// .map(char::from) +// .collect() +// } + +// #[tokio::test] +// async fn conn_streams_has_required_streams() -> BoxedResult<()> { +// let (client, streams, _) = server_setup().await.unwrap(); +// let mut context_streams = client.jetstream.stream_names(); + +// let mut names = HashSet::new(); +// while let Some(name) = context_streams.try_next().await? { +// names.insert(name); +// } +// streams.blocks.assert_has_stream(&names).await; +// streams.transactions.assert_has_stream(&names).await; + +// for name in names.iter() { +// let empty = streams.blocks.is_empty(name).await; +// assert!(empty, "stream must be empty after creation"); +// } +// Ok(()) +// } + +// #[tokio::test] +// async fn fuel_streams_client_connection() -> BoxedResult<()> { +// let nats_opts = NatsClientOpts::admin_opts(); +// let client = NatsClient::connect(&nats_opts).await?; +// assert!(client.is_connected()); +// let s3_opts = Arc::new(S3ClientOpts::admin_opts()); +// let client = Client::with_opts(&nats_opts, &s3_opts).await?; +// assert!(client.nats_conn.is_connected()); +// Ok(()) +// } + +// #[tokio::test] +// async fn multiple_client_connections() -> BoxedResult<()> { +// let nats_opts = NatsClientOpts::admin_opts(); +// let s3_opts = Arc::new(S3ClientOpts::admin_opts()); +// let tasks: Vec<_> = (0..100) +// .map(|_| { +// let nats_opts = nats_opts.clone(); +// let s3_opts = s3_opts.clone(); +// async move { +// let client = +// Client::with_opts(&nats_opts, &s3_opts).await.unwrap(); +// assert!(client.nats_conn.is_connected()); +// Ok::<(), NatsError>(()) +// } +// }) +// .collect(); + +// assert!(try_join_all(tasks).await.is_ok()); +// Ok(()) +// } + +// #[tokio::test] +// async fn public_user_cannot_create_streams() -> BoxedResult<()> { +// let network = FuelNetwork::Local; +// let opts = NatsClientOpts::public_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); +// let client = NatsClient::connect(&opts).await?; +// let (random_stream_title, random_subject) = +// (gen_random_string(6), gen_random_string(6)); + +// assert!(client +// .jetstream +// .create_stream(types::NatsStreamConfig { +// name: random_stream_title, +// subjects: vec![random_subject], +// ..Default::default() +// }) +// .await +// .is_err()); + +// Ok(()) +// } + +// #[tokio::test] +// async fn public_user_cannot_create_stores() -> BoxedResult<()> { +// let network = FuelNetwork::Local; +// let opts = NatsClientOpts::public_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); + +// let random_bucket_title = gen_random_string(6); + +// let client = NatsClient::connect(&opts).await?; +// assert!(client +// .jetstream +// .create_key_value(types::KvStoreConfig { +// bucket: random_bucket_title, +// ..Default::default() +// }) +// .await +// .is_err()); + +// Ok(()) +// } + +// #[tokio::test] +// async fn public_user_cannot_delete_stores() -> BoxedResult<()> { +// let network = FuelNetwork::Local; +// let opts = NatsClientOpts::admin_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); + +// let random_bucket_title = gen_random_string(6); + +// let client = NatsClient::connect(&opts).await?; +// client +// .jetstream +// .create_key_value(types::KvStoreConfig { +// bucket: random_bucket_title.clone(), +// ..Default::default() +// }) +// .await?; + +// let opts = NatsClientOpts::public_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); +// let client = NatsClient::connect(&opts).await?; + +// assert!(client +// .jetstream +// .delete_key_value(&random_bucket_title) +// .await +// .is_err()); + +// Ok(()) +// } + +// #[tokio::test] +// async fn public_user_cannot_delete_stream() -> BoxedResult<()> { +// let opts = NatsClientOpts::admin_opts() +// .with_rdn_namespace() +// .with_timeout(1); +// let client = NatsClient::connect(&opts).await?; + +// let (random_stream_title, random_subject) = +// (gen_random_string(6), gen_random_string(6)); + +// client +// .jetstream +// .create_stream(types::NatsStreamConfig { +// name: random_stream_title.clone(), +// subjects: vec![random_subject], +// ..Default::default() +// }) +// .await?; + +// let network = FuelNetwork::Local; +// let public_opts = +// NatsClientOpts::public_opts().with_url(network.to_nats_url()); +// let public_client = NatsClient::connect(&public_opts).await?; + +// assert!( +// public_client +// .jetstream +// .delete_stream(&random_stream_title) +// .await +// .is_err(), +// "Stream must be deleted at this point" +// ); + +// Ok(()) +// } + +// #[tokio::test] +// async fn public_user_can_access_streams_after_created() { +// let network = FuelNetwork::Local; +// let admin_opts = NatsClientOpts::admin_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); + +// let public_opts = NatsClientOpts::public_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); + +// assert!(NatsClient::connect(&admin_opts).await.is_ok()); +// assert!(NatsClient::connect(&public_opts).await.is_ok()); +// } + +// #[tokio::test] +// async fn public_and_admin_user_can_access_streams_after_created( +// ) -> BoxedResult<()> { +// let network = FuelNetwork::Local; +// let admin_opts = NatsClientOpts::admin_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); +// let s3_opts = Arc::new(S3ClientOpts::admin_opts()); +// let admin_tasks: Vec>> = (0..100) +// .map(|_| { +// let opts: NatsClientOpts = admin_opts.clone(); +// let s3_opts = s3_opts.clone(); +// async move { +// let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); +// assert!(client.nats_conn.is_connected()); +// Ok::<(), NatsError>(()) +// } +// .boxed() +// }) +// .collect(); + +// let public_opts = NatsClientOpts::public_opts() +// .with_url(network.to_nats_url()) +// .with_rdn_namespace() +// .with_timeout(1); +// let s3_public_opts = +// Arc::new(S3ClientOpts::new(S3Env::Local, S3Role::Public)); +// let public_tasks: Vec>> = (0..100) +// .map(|_| { +// let opts: NatsClientOpts = public_opts.clone(); +// let s3_opts = s3_public_opts.clone(); +// async move { +// let client = Client::with_opts(&opts, &s3_opts).await.unwrap(); +// assert!(client.nats_conn.is_connected()); +// Ok::<(), NatsError>(()) +// } +// .boxed() +// }) +// .collect(); + +// // Combine both vectors into one +// let mut all_tasks = +// Vec::with_capacity(admin_tasks.len() + public_tasks.len()); +// all_tasks.extend(admin_tasks); +// all_tasks.extend(public_tasks); + +// assert!(try_join_all(all_tasks).await.is_ok()); +// Ok(()) +// } + +// #[tokio::test] +// async fn admin_user_can_delete_stream() -> BoxedResult<()> { +// let opts = NatsClientOpts::admin_opts() +// .with_rdn_namespace() +// .with_timeout(1); +// let client = NatsClient::connect(&opts).await?; + +// let (random_stream_title, random_subject) = +// (gen_random_string(6), gen_random_string(6)); + +// client +// .jetstream +// .create_stream(types::NatsStreamConfig { +// name: random_stream_title.clone(), +// subjects: vec![random_subject], +// ..Default::default() +// }) +// .await?; + +// let status = client.jetstream.delete_stream(&random_stream_title).await?; +// assert!(status.success, "Stream must be deleted at this point"); + +// Ok(()) +// } + +// #[tokio::test] +// async fn admin_user_can_delete_stores() -> BoxedResult<()> { +// let opts = NatsClientOpts::admin_opts() +// .with_rdn_namespace() +// .with_timeout(1); + +// let random_bucket_title = gen_random_string(6); + +// let client = NatsClient::connect(&opts).await?; +// client +// .jetstream +// .create_key_value(types::KvStoreConfig { +// bucket: random_bucket_title.clone(), +// ..Default::default() +// }) +// .await?; + +// assert!(client +// .jetstream +// .delete_key_value(&random_bucket_title) +// .await +// .is_ok()); + +// Ok(()) +// } + +// #[tokio::test] +// async fn ensure_deduplication_when_publishing() -> BoxedResult<()> { +// let (_, _, client) = server_setup().await.unwrap(); +// let stream = fuel_streams::Stream::::new(&client).await; +// let producer = Some(Address::zeroed()); +// let const_block_height = 1001; +// let items = +// publish_blocks(stream.stream(), producer, Some(const_block_height)) +// .unwrap() +// .0; + +// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); +// let timeout_duration = Duration::from_secs(1); + +// // ensure just one message was published +// 'l: loop { +// match timeout(timeout_duration, sub.next()).await { +// Ok(Some((idx, entry))) => { +// let decoded_msg = Block::decode_raw(entry).unwrap(); +// let (subject, _block) = items[idx].to_owned(); +// let height = decoded_msg.payload.height; +// assert_eq!(decoded_msg.subject, subject.parse()); +// assert_eq!(height, const_block_height); +// assert!(idx < 1); +// } +// _ => { +// break 'l; +// } +// } +// } + +// Ok(()) +// } diff --git a/tests/tests/stream.rs b/tests/tests/stream.rs index f8f11ac5..2df55a23 100644 --- a/tests/tests/stream.rs +++ b/tests/tests/stream.rs @@ -1,242 +1,242 @@ -use fuel_streams::prelude::*; -use fuel_streams_core::prelude::*; -use futures::{future::try_join_all, StreamExt}; -use pretty_assertions::assert_eq; -use streams_tests::{publish_blocks, publish_transactions, server_setup}; - -#[tokio::test] -async fn blocks_streams_subscribe() { - let (_, _, client) = server_setup().await.unwrap(); - let stream = fuel_streams::Stream::::new(&client).await; - let producer = Some(Address::zeroed()); - let items = publish_blocks(stream.stream(), producer, None).unwrap().0; - - let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - - while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes).unwrap(); - let (subject, block) = items[i].to_owned(); - let height = decoded_msg.payload.height; - - assert_eq!(decoded_msg.subject, subject.parse()); - assert_eq!(decoded_msg.payload, block); - assert_eq!(height, i as u32); - if i == 9 { - break; - } - } -} - -#[tokio::test] -async fn blocks_streams_subscribe_with_filter() { - let (_, _, client) = server_setup().await.unwrap(); - let mut stream = fuel_streams::Stream::::new(&client).await; - let producer = Some(Address::zeroed()); - - // publishing 10 blocks - publish_blocks(stream.stream(), producer, None).unwrap(); - - // filtering by producer 0x000 and height 5 - let filter = Filter::::build() - .with_producer(Some(Address::zeroed())) - .with_height(Some(5.into())); - - // creating subscription - let mut sub = stream - .with_filter(filter) - .subscribe_raw_with_config(StreamConfig::default()) - .await - .unwrap() - .take(10); - - // result should be just 1 single message with height 5 - while let Some(bytes) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes).unwrap(); - let height = decoded_msg.payload.height; - assert_eq!(height, 5); - if height == 5 { - break; - } - } -} - -#[tokio::test] -async fn transactions_streams_subscribe() { - let (_, _, client) = server_setup().await.unwrap(); - let stream = fuel_streams::Stream::::new(&client).await; - - let mock_block = MockBlock::build(1); - let items = publish_transactions(stream.stream(), &mock_block, None) - .unwrap() - .0; - - let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Transaction::decode_raw(bytes).unwrap(); - - let (_, transaction) = items[i].to_owned(); - assert_eq!(decoded_msg.payload, transaction); - if i == 9 { - break; - } - } -} - -#[tokio::test] -async fn transactions_streams_subscribe_with_filter() { - let (_, _, client) = server_setup().await.unwrap(); - let mut stream = fuel_streams::Stream::::new(&client).await; - - // publishing 10 transactions - let mock_block = MockBlock::build(5); - let items = publish_transactions(stream.stream(), &mock_block, None) - .unwrap() - .0; - - // filtering by transaction on block with height 5 - let filter = Filter::::build() - .with_block_height(Some(5.into())); - - // creating subscription - let mut sub = stream - .with_filter(filter) - .subscribe_raw_with_config(StreamConfig::default()) - .await - .unwrap() - .take(10) - .enumerate(); - - // result should be 10 transactions messages - while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Transaction::decode(bytes).unwrap(); - - let (_, transaction) = items[i].to_owned(); - assert_eq!(decoded_msg, transaction); - if i == 9 { - break; - } - } -} - -#[tokio::test] -async fn multiple_subscribers_same_subject() { - let (_, _, client) = server_setup().await.unwrap(); - let stream = fuel_streams::Stream::::new(&client).await; - let producer = Some(Address::zeroed()); - let items = publish_blocks(stream.stream(), producer.clone(), None) - .unwrap() - .0; - - let clients_count = 100; - let done_signal = 99; - let mut handles = Vec::new(); - for _ in 0..clients_count { - let stream = stream.clone(); - let items = items.clone(); - handles.push(tokio::spawn(async move { - let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes).unwrap(); - let (subject, block) = items[i].to_owned(); - let height = decoded_msg.payload.height; - - assert_eq!(decoded_msg.subject, subject.parse()); - assert_eq!(decoded_msg.payload, block); - assert_eq!(height, i as u32); - if i == 9 { - return done_signal; - } - } - done_signal + 1 - })); - } - - let mut client_results = try_join_all(handles).await.unwrap(); - assert!( - client_results.len() == clients_count, - "must have all clients subscribed to one subject" - ); - client_results.dedup(); - assert!( - client_results.len() == 1, - "all clients must have the same result" - ); - assert!( - client_results.first().cloned().unwrap() == done_signal, - "all clients must have the same received the complete signal" - ); -} - -#[tokio::test] -async fn multiple_subscribers_different_subjects() { - let (_, _, client) = server_setup().await.unwrap(); - let producer = Some(Address::zeroed()); - let block_stream = fuel_streams::Stream::::new(&client).await; - let block_items = - publish_blocks(block_stream.stream(), producer.clone(), None) - .unwrap() - .0; - - let txs_stream = fuel_streams::Stream::::new(&client).await; - let mock_block = MockBlock::build(1); - let txs_items = - publish_transactions(txs_stream.stream(), &mock_block, None) - .unwrap() - .0; - - let clients_count = 100; - let done_signal = 99; - let mut handles = Vec::new(); - for _ in 0..clients_count { - // blocks stream - let stream = block_stream.clone(); - let items = block_items.clone(); - handles.push(tokio::spawn(async move { - let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Block::decode_raw(bytes).unwrap(); - let (subject, block) = items[i].to_owned(); - let height = decoded_msg.payload.height; - - assert_eq!(decoded_msg.subject, subject.parse()); - assert_eq!(decoded_msg.payload, block); - assert_eq!(height, i as u32); - if i == 9 { - return done_signal; - } - } - done_signal + 1 - })); - - // txs stream - let stream = txs_stream.clone(); - let items = txs_items.clone(); - handles.push(tokio::spawn(async move { - let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); - while let Some((i, bytes)) = sub.next().await { - let decoded_msg = Transaction::decode_raw(bytes).unwrap(); - let (_, transaction) = items[i].to_owned(); - assert_eq!(decoded_msg.payload, transaction); - if i == 9 { - return done_signal; - } - } - done_signal + 1 - })); - } - - let mut client_results = try_join_all(handles).await.unwrap(); - assert!( - client_results.len() == 2 * clients_count, - "must have all clients subscribed to two subjects" - ); - client_results.dedup(); - assert!( - client_results.len() == 1, - "all clients must have the same result" - ); - assert!( - client_results.first().cloned().unwrap() == done_signal, - "all clients must have the same received the complete signal" - ); -} +// use fuel_streams::prelude::*; +// use fuel_streams_core::prelude::*; +// use futures::{future::try_join_all, StreamExt}; +// use pretty_assertions::assert_eq; +// use streams_tests::{publish_blocks, publish_transactions, server_setup}; + +// #[tokio::test] +// async fn blocks_streams_subscribe() { +// let (_, _, client) = server_setup().await.unwrap(); +// let stream = fuel_streams::Stream::::new(&client).await; +// let producer = Some(Address::zeroed()); +// let items = publish_blocks(stream.stream(), producer, None).unwrap().0; + +// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); + +// while let Some((i, bytes)) = sub.next().await { +// let decoded_msg = Block::decode_raw(bytes).unwrap(); +// let (subject, block) = items[i].to_owned(); +// let height = decoded_msg.payload.height; + +// assert_eq!(decoded_msg.subject, subject.parse()); +// assert_eq!(decoded_msg.payload, block); +// assert_eq!(height, i as u32); +// if i == 9 { +// break; +// } +// } +// } + +// #[tokio::test] +// async fn blocks_streams_subscribe_with_filter() { +// let (_, _, client) = server_setup().await.unwrap(); +// let mut stream = fuel_streams::Stream::::new(&client).await; +// let producer = Some(Address::zeroed()); + +// // publishing 10 blocks +// publish_blocks(stream.stream(), producer, None).unwrap(); + +// // filtering by producer 0x000 and height 5 +// let filter = Filter::::build() +// .with_producer(Some(Address::zeroed())) +// .with_height(Some(5.into())); + +// // creating subscription +// let mut sub = stream +// .with_filter(filter) +// .subscribe_raw_with_config(StreamConfig::default()) +// .await +// .unwrap() +// .take(10); + +// // result should be just 1 single message with height 5 +// while let Some(bytes) = sub.next().await { +// let decoded_msg = Block::decode_raw(bytes).unwrap(); +// let height = decoded_msg.payload.height; +// assert_eq!(height, 5); +// if height == 5 { +// break; +// } +// } +// } + +// #[tokio::test] +// async fn transactions_streams_subscribe() { +// let (_, _, client) = server_setup().await.unwrap(); +// let stream = fuel_streams::Stream::::new(&client).await; + +// let mock_block = MockBlock::build(1); +// let items = publish_transactions(stream.stream(), &mock_block, None) +// .unwrap() +// .0; + +// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); +// while let Some((i, bytes)) = sub.next().await { +// let decoded_msg = Transaction::decode_raw(bytes).unwrap(); + +// let (_, transaction) = items[i].to_owned(); +// assert_eq!(decoded_msg.payload, transaction); +// if i == 9 { +// break; +// } +// } +// } + +// #[tokio::test] +// async fn transactions_streams_subscribe_with_filter() { +// let (_, _, client) = server_setup().await.unwrap(); +// let mut stream = fuel_streams::Stream::::new(&client).await; + +// // publishing 10 transactions +// let mock_block = MockBlock::build(5); +// let items = publish_transactions(stream.stream(), &mock_block, None) +// .unwrap() +// .0; + +// // filtering by transaction on block with height 5 +// let filter = Filter::::build() +// .with_block_height(Some(5.into())); + +// // creating subscription +// let mut sub = stream +// .with_filter(filter) +// .subscribe_raw_with_config(StreamConfig::default()) +// .await +// .unwrap() +// .take(10) +// .enumerate(); + +// // result should be 10 transactions messages +// while let Some((i, bytes)) = sub.next().await { +// let decoded_msg = Transaction::decode(bytes).unwrap(); + +// let (_, transaction) = items[i].to_owned(); +// assert_eq!(decoded_msg, transaction); +// if i == 9 { +// break; +// } +// } +// } + +// #[tokio::test] +// async fn multiple_subscribers_same_subject() { +// let (_, _, client) = server_setup().await.unwrap(); +// let stream = fuel_streams::Stream::::new(&client).await; +// let producer = Some(Address::zeroed()); +// let items = publish_blocks(stream.stream(), producer.clone(), None) +// .unwrap() +// .0; + +// let clients_count = 100; +// let done_signal = 99; +// let mut handles = Vec::new(); +// for _ in 0..clients_count { +// let stream = stream.clone(); +// let items = items.clone(); +// handles.push(tokio::spawn(async move { +// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); +// while let Some((i, bytes)) = sub.next().await { +// let decoded_msg = Block::decode_raw(bytes).unwrap(); +// let (subject, block) = items[i].to_owned(); +// let height = decoded_msg.payload.height; + +// assert_eq!(decoded_msg.subject, subject.parse()); +// assert_eq!(decoded_msg.payload, block); +// assert_eq!(height, i as u32); +// if i == 9 { +// return done_signal; +// } +// } +// done_signal + 1 +// })); +// } + +// let mut client_results = try_join_all(handles).await.unwrap(); +// assert!( +// client_results.len() == clients_count, +// "must have all clients subscribed to one subject" +// ); +// client_results.dedup(); +// assert!( +// client_results.len() == 1, +// "all clients must have the same result" +// ); +// assert!( +// client_results.first().cloned().unwrap() == done_signal, +// "all clients must have the same received the complete signal" +// ); +// } + +// #[tokio::test] +// async fn multiple_subscribers_different_subjects() { +// let (_, _, client) = server_setup().await.unwrap(); +// let producer = Some(Address::zeroed()); +// let block_stream = fuel_streams::Stream::::new(&client).await; +// let block_items = +// publish_blocks(block_stream.stream(), producer.clone(), None) +// .unwrap() +// .0; + +// let txs_stream = fuel_streams::Stream::::new(&client).await; +// let mock_block = MockBlock::build(1); +// let txs_items = +// publish_transactions(txs_stream.stream(), &mock_block, None) +// .unwrap() +// .0; + +// let clients_count = 100; +// let done_signal = 99; +// let mut handles = Vec::new(); +// for _ in 0..clients_count { +// // blocks stream +// let stream = block_stream.clone(); +// let items = block_items.clone(); +// handles.push(tokio::spawn(async move { +// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); +// while let Some((i, bytes)) = sub.next().await { +// let decoded_msg = Block::decode_raw(bytes).unwrap(); +// let (subject, block) = items[i].to_owned(); +// let height = decoded_msg.payload.height; + +// assert_eq!(decoded_msg.subject, subject.parse()); +// assert_eq!(decoded_msg.payload, block); +// assert_eq!(height, i as u32); +// if i == 9 { +// return done_signal; +// } +// } +// done_signal + 1 +// })); + +// // txs stream +// let stream = txs_stream.clone(); +// let items = txs_items.clone(); +// handles.push(tokio::spawn(async move { +// let mut sub = stream.subscribe_raw().await.unwrap().enumerate(); +// while let Some((i, bytes)) = sub.next().await { +// let decoded_msg = Transaction::decode_raw(bytes).unwrap(); +// let (_, transaction) = items[i].to_owned(); +// assert_eq!(decoded_msg.payload, transaction); +// if i == 9 { +// return done_signal; +// } +// } +// done_signal + 1 +// })); +// } + +// let mut client_results = try_join_all(handles).await.unwrap(); +// assert!( +// client_results.len() == 2 * clients_count, +// "must have all clients subscribed to two subjects" +// ); +// client_results.dedup(); +// assert!( +// client_results.len() == 1, +// "all clients must have the same result" +// ); +// assert!( +// client_results.first().cloned().unwrap() == done_signal, +// "all clients must have the same received the complete signal" +// ); +// } From ed2b1945d9542a7bf9b9b3b56727bc14bba56b98 Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Mon, 23 Dec 2024 02:38:26 -0300 Subject: [PATCH 14/15] build(repo): fix tests --- .../src/stream/stream_impl.rs | 2 +- crates/fuel-streams/README.md | 157 ++++++------------ crates/fuel-streams/src/client/client_impl.rs | 73 -------- crates/fuel-streams/src/client/connection.rs | 85 +--------- 4 files changed, 56 insertions(+), 261 deletions(-) diff --git a/crates/fuel-streams-core/src/stream/stream_impl.rs b/crates/fuel-streams-core/src/stream/stream_impl.rs index 3d40d4f7..bc5690cb 100644 --- a/crates/fuel-streams-core/src/stream/stream_impl.rs +++ b/crates/fuel-streams-core/src/stream/stream_impl.rs @@ -442,7 +442,7 @@ impl Stream { /// /// ``` /// use fuel_streams_core::stream::SubscriptionConfig; -/// use async_nats::jetstream::consumer::NatsDeliverPolicy; +/// use async_nats::jetstream::consumer::DeliverPolicy as NatsDeliverPolicy; /// /// let config = SubscriptionConfig { /// filter_subjects: vec!["example.*".to_string()], diff --git a/crates/fuel-streams/README.md b/crates/fuel-streams/README.md index 410e1e37..9e6dbaa4 100644 --- a/crates/fuel-streams/README.md +++ b/crates/fuel-streams/README.md @@ -60,155 +60,106 @@ cargo add fuel-streams futures tokio Here are some examples to get you started with Fuel Streams: -### Subscribing to all new blocks +### Basic Connection and Subscription ```rust,no_run -use fuel_streams::types::FuelNetwork; -use fuel_streams::client::Client; -use fuel_streams::stream::{Stream, StreamEncoder}; -use fuel_streams::blocks::Block; +use fuel_streams::prelude::*; use futures::StreamExt; #[tokio::main] -async fn main() -> Result<(), fuel_streams::Error> { - let client = Client::connect(FuelNetwork::Local).await?; - let stream = fuel_streams::Stream::::new(&client).await; +async fn main() -> Result<(), Box> { + // Create a client and establish connection + let mut client = Client::new(FuelNetwork::Local).await?; + let mut connection = client.connect().await?; - let mut subscription = stream.subscribe().await?; - while let Some(block) = subscription.next().await { - println!("Received block: {:?}", block); - } - - Ok(()) -} -``` - -### Subscribing to all transactions (Filtering by block height) + println!("Listening for blocks..."); -```rust,no_run -use fuel_streams::types::FuelNetwork; -use fuel_streams::client::Client; -use fuel_streams::stream::{Filter, Stream, StreamEncoder, StreamConfig}; -use fuel_streams::transactions::{Transaction, TransactionKind, TransactionsSubject}; -use futures::StreamExt; - -#[tokio::main] -async fn main() -> Result<(), fuel_streams::Error> { - let client = Client::connect(FuelNetwork::Local).await?; - let mut stream = fuel_streams::Stream::::new(&client).await; + // Create a subject for all blocks + let subject = BlocksSubject::new(); - // Filter transactions from block height 5 - let filter = Filter::::build() - .with_block_height(Some(5.into())); - - let mut subscription = stream - .with_filter(filter) - .subscribe_with_config(StreamConfig::default()) + // Subscribe to blocks with last delivery policy + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) .await?; - while let Some(transaction) = subscription.next().await { - println!("Received transaction: {:?}", transaction); + while let Some(block) = stream.next().await { + println!("Received block: {:?}", block); } Ok(()) } ``` -## Advanced - -### `DeliverPolicy` - -The `DeliverPolicy` provides fine-grained control over message delivery in your stream. This powerful feature allows you to customize how and when messages are received. Below is an illustrative example demonstrating how to subscribe to all blocks from the first block until the last block in the stream: +### Custom Connection Options ```rust,no_run -use fuel_streams::types::FuelNetwork; -use fuel_streams::client::Client; -use fuel_streams::stream::{Stream, StreamConfig, StreamEncoder, Filter}; -use fuel_streams::blocks::{Block, BlocksSubject}; -use fuel_streams::types::DeliverPolicy; -use futures::StreamExt; +use fuel_streams::prelude::*; #[tokio::main] -async fn main() -> Result<(), fuel_streams::Error> { - let client = Client::connect(FuelNetwork::Local).await?; - let mut stream = fuel_streams::Stream::::new(&client).await; - - let filter = Filter::::build(); - let mut subscription = stream - .with_filter(filter) - .subscribe_with_config(StreamConfig { - // Set the deliver policy to `All` to receive all blocks - // from the first block until the last block in the stream - deliver_policy: DeliverPolicy::All, - }) - .await?; - - while let Some(block) = subscription.next().await { - println!("Received block: {:?}", block); - } +async fn main() -> Result<(), Box> { + // Create client with custom connection options + let client = Client::with_opts(ConnectionOpts { + network: FuelNetwork::Local, + username: "custom_user".to_string(), + password: "custom_pass".to_string(), + }).await?; Ok(()) } ``` -Available `DeliverPolicy` options: +### Subject Types and Filtering -- `All`: Delivers all messages in the stream. -- `Last`: Delivers the last message for the selected subjects. -- `New`: Delivers only new messages that are received after the subscription is created. -- `ByStartSequence(u64)`: Delivers messages starting from a specific sequence number. -- `ByStartTime(DateTime)`: Delivers messages starting from a specific time. - -Choose the appropriate `DeliverPolicy` based on your application's requirements for historical data processing or real-time updates. - -### Filters - -Filters allow you to narrow down the data you receive from a stream based on specific criteria. This is particularly useful when you're only interested in a subset of the data. The `Stream` struct provides a `with_filter` method that allows you to apply filters to your subscription. - -Here's an example of how to use filters with a stream of transactions: +Each data type has its own subject builder for filtering. Here's an example using transaction filtering: ```rust,no_run -use fuel_streams::types::FuelNetwork; -use fuel_streams::client::Client; -use fuel_streams::stream::{Stream, StreamConfig, StreamEncoder, Filter}; -use fuel_streams::transactions::{Transaction, TransactionsSubject, TransactionKind}; -use fuel_streams::types::Address; +use fuel_streams::prelude::*; use futures::StreamExt; #[tokio::main] -async fn main() -> Result<(), fuel_streams::Error> { - let client = Client::connect(FuelNetwork::Local).await?; - let mut stream = fuel_streams::Stream::::new(&client).await; +async fn main() -> Result<(), Box> { + let mut client = Client::new(FuelNetwork::Testnet).await?; + let mut connection = client.connect().await?; - // Create a filter for transactions from a specific block height and kind - let filter = Filter::::build() - .with_block_height(Some(1000.into())) + println!("Listening for transactions..."); + + // Create a subject for script transactions + let subject = TransactionsSubject::new() .with_kind(Some(TransactionKind::Script)); - let mut subscription = stream - .with_filter(filter) - .subscribe_with_config(StreamConfig::default()) + // Subscribe to the filtered transaction stream + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) .await?; - while let Some(transaction) = subscription.next().await { - println!("Received filtered transaction: {:?}", transaction); + while let Some(transaction) = stream.next().await { + println!("Received transaction: {:?}", transaction); } Ok(()) } ``` -In this example, we're creating a filter that will only return transactions from a specific kind (`TransactionKind::Script`) and from a specific block height (1000). +Available subject builders include: + +- `BlocksSubject::new()` +- `TransactionsSubject::new()` +- `InputsSubject::new()` +- `OutputsSubject::new()` +- `LogsSubject::new()` +- `UtxosSubject::new()` -Available filter methods depend on the subject type. The project currently supports subjects for the following data types: +Each subject builder provides specific filtering methods relevant to its data type. For example, `TransactionsSubject` allows filtering by transaction kind using the `with_kind()` method. -- [Blocks](../fuel-streams-core/src/blocks/subjects.rs) -- [Transactions](../fuel-streams-core/src/transactions/subjects.rs) +### `DeliverPolicy` Options -Filters can be combined to create more specific queries. Each filter method narrows down the results further. +The `DeliverPolicy` enum provides control over message delivery in your subscriptions: -> [!NOTE] -> Remember that the effectiveness of filters depends on how the data is structured in the NATS streams. Filters are applied on the client side, so they can help reduce the amount of data your application needs to process, but they don't reduce the amount of data transferred over the network. +- `All`: Delivers all messages in the stream +- `Last`: Delivers only the last message for the selected subjects +- `New`: Delivers only new messages that arrive after subscription +- `ByStartSequence(u64)`: Delivers messages starting from a specific sequence number +- `ByStartTime(DateTime)`: Delivers messages starting from a specific time ## 🤝 Contributing diff --git a/crates/fuel-streams/src/client/client_impl.rs b/crates/fuel-streams/src/client/client_impl.rs index 2ecd8152..e7bfdf11 100644 --- a/crates/fuel-streams/src/client/client_impl.rs +++ b/crates/fuel-streams/src/client/client_impl.rs @@ -25,28 +25,6 @@ use super::{ }; use crate::FuelNetwork; -/// A client for connecting to the Fuel websocket server. -/// -/// # Examples -/// -/// ```no_run -/// use fuel_streams::{Client, FuelNetwork}; -/// -/// #[tokio::main] -/// async fn main() -> Result<(), Box> { -/// // Basic usage with default credentials -/// let mut client = Client::new(FuelNetwork::Local).await?; -/// let connection = client.connect().await?; -/// -/// // Or with custom connection options -/// let client = Client::with_opts(ConnectionOpts { -/// network: FuelNetwork::Local, -/// username: "custom_user".to_string(), -/// password: "custom_pass".to_string(), -/// }).await?; -/// Ok(()) -/// } -/// ``` #[derive(Debug, Clone)] pub struct Client { pub opts: ConnectionOpts, @@ -54,7 +32,6 @@ pub struct Client { } impl Client { - /// Creates a new WebSocket client with default connection options for the specified network. pub async fn new(network: FuelNetwork) -> Result { Self::with_opts(ConnectionOpts { network, @@ -63,7 +40,6 @@ impl Client { .await } - /// Creates a new WebSocket client with custom connection options. pub async fn with_opts(opts: ConnectionOpts) -> Result { let jwt_token = Self::fetch_jwt(opts.network, &opts.username, &opts.password) @@ -74,20 +50,6 @@ impl Client { }) } - /// Establishes a WebSocket connection using the client's configuration. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::{Client, FuelNetwork}; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let mut client = Client::new(FuelNetwork::Local).await?; - /// let connection = client.connect().await?; - /// Ok(()) - /// } - /// ``` pub async fn connect(&mut self) -> Result { let ws_url = self.opts.network.to_ws_url().join("/api/v1/ws")?; let host = ws_url @@ -109,25 +71,6 @@ impl Client { Connection::new(request).await } - /// Fetches a JWT token from the server for authentication. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::{Client, FuelNetwork}; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let jwt = Client::fetch_jwt( - /// FuelNetwork::Local, - /// "admin", - /// "admin" - /// ).await?; - /// - /// assert!(!jwt.is_empty()); - /// Ok(()) - /// } - /// ``` async fn fetch_jwt( network: FuelNetwork, username: &str, @@ -158,22 +101,6 @@ impl Client { } } - /// Refreshes the JWT token and establishes a new WebSocket connection. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::{Client, FuelNetwork}; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let mut client = Client::new(FuelNetwork::Local).await?; - /// - /// // Refresh token and reconnect - /// let new_connection = client.refresh_jwt_and_connect().await?; - /// Ok(()) - /// } - /// ``` pub async fn refresh_jwt_and_connect( &mut self, ) -> Result { diff --git a/crates/fuel-streams/src/client/connection.rs b/crates/fuel-streams/src/client/connection.rs index e7413d3d..808c519d 100644 --- a/crates/fuel-streams/src/client/connection.rs +++ b/crates/fuel-streams/src/client/connection.rs @@ -18,26 +18,6 @@ use super::{ }; use crate::FuelNetwork; -/// Connection options for establishing a WebSocket connection. -/// -/// # Examples -/// -/// ``` -/// use fuel_streams::{ConnectionOpts, FuelNetwork}; -/// -/// // Create connection options with custom values -/// let opts = ConnectionOpts { -/// network: FuelNetwork::Local, -/// username: "admin".to_string(), -/// password: "admin".to_string(), -/// }; -/// -/// // Or use the default options -/// let default_opts = ConnectionOpts::default(); -/// assert_eq!(default_opts.username, "admin"); -/// assert_eq!(default_opts.password, "admin"); -/// assert!(matches!(default_opts.network, FuelNetwork::Local)); -/// ``` #[derive(Debug, Clone)] pub struct ConnectionOpts { pub network: FuelNetwork, @@ -83,24 +63,7 @@ impl Connection { write_sink: RwLock::new(write), }) } - /// Sends a client message through the WebSocket connection. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::{Client, ConnectionOpts, FuelNetwork}; - /// use sv_webserver::server::ws::models::ClientMessage; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let mut client = Client::new(FuelNetwork::Local).await?; - /// let connection = client.connect().await?; - /// - /// let message = ClientMessage::Ping; - /// connection.send_client_message(message).await?; - /// Ok(()) - /// } - /// ``` + async fn send_client_message( &self, message: ClientMessage, @@ -111,33 +74,6 @@ impl Connection { Ok(()) } - /// Subscribes to a subject and returns a stream of messages. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::{Client, ConnectionOpts, FuelNetwork}; - /// use sv_webserver::server::ws::models::{DeliverPolicy, ServerMessage}; - /// use std::sync::Arc; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let mut client = Client::new(FuelNetwork::Local).await?; - /// let mut connection = client.connect().await?; - /// - /// let subject = Arc::new("example.subject"); - /// let mut stream = connection.subscribe::( - /// subject, - /// DeliverPolicy::All - /// ).await?; - /// - /// // Process messages from the stream - /// while let Some(message) = stream.next().await { - /// println!("Received: {:?}", message); - /// } - /// Ok(()) - /// } - /// ``` pub async fn subscribe( &mut self, subject: impl IntoSubject, @@ -188,25 +124,6 @@ impl Connection { Ok(Box::pin(stream)) } - /// Unsubscribes from a subject. - /// - /// # Examples - /// - /// ```no_run - /// use fuel_streams::{Client, ConnectionOpts, FuelNetwork}; - /// use sv_webserver::server::ws::models::DeliverPolicy; - /// use std::sync::Arc; - /// - /// #[tokio::main] - /// async fn main() -> Result<(), Box> { - /// let mut client = Client::new(FuelNetwork::Local).await?; - /// let connection = client.connect().await?; - /// - /// let subject = Arc::new("example.subject"); - /// connection.unsubscribe(subject, DeliverPolicy::All).await?; - /// Ok(()) - /// } - /// ``` pub async fn unsubscribe( &self, subject: S, From 5b12c1c44bace8e16a6406d1f0811d194b1247ab Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Mon, 23 Dec 2024 02:40:47 -0300 Subject: [PATCH 15/15] docs(repo): adjust main readme --- README.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 880e5f05..dcb22bda 100644 --- a/README.md +++ b/README.md @@ -51,21 +51,26 @@ With Fuel Data Systems, developers can build sophisticated applications that lev 2. Create a new Rust file (e.g., `src/main.rs`) with the following code to subscribe to new blocks: ```rust - use fuel_streams::client::Client; - use fuel_streams::types::FuelNetwork; - use fuel_streams::stream::{Stream, StreamEncoder}; - use fuel_streams::blocks::Block; + use fuel_streams::prelude::*; use futures::StreamExt; #[tokio::main] - async fn main() -> Result<(), fuel_streams::Error> { - let client = Client::connect(FuelNetwork::Mainnet).await?; - let stream = fuel_streams::Stream::::new(&client).await; - - let mut subscription = stream.subscribe().await?; - while let Some(entry) = subscription.next().await { - let entry = entry.unwrap().clone(); - let block = Block::decode(entry); + async fn main() -> Result<(), Box> { + // Create a client and establish connection + let mut client = Client::new(FuelNetwork::Local).await?; + let mut connection = client.connect().await?; + + println!("Listening for blocks..."); + + // Create a subject for all blocks + let subject = BlocksSubject::new(); + + // Subscribe to blocks with last delivery policy + let mut stream = connection + .subscribe::(subject, DeliverPolicy::Last) + .await?; + + while let Some(block) = stream.next().await { println!("Received block: {:?}", block); } @@ -78,7 +83,7 @@ With Fuel Data Systems, developers can build sophisticated applications that lev cargo run ``` -This example connects to the Fuel Network's NATS server and listens for new blocks. You can customize the data types or apply filters based on your specific requirements. +This example connects to the Fuel Network and listens for new blocks. You can customize the data types or apply filters based on your specific requirements. For advanced usage, including custom filters and delivery policies, refer to the [`fuel-streams` documentation](https://docs.rs/fuel-streams).