diff --git a/.drone.yml b/.drone.yml index de2422d53f..c2a46ec12e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,6 +3,7 @@ name: test-on-arm64 platform: arch: arm64 + os: linux steps: - name: test diff --git a/.github/checks/copyright.sh b/.github/checks/copyright.sh index 0dd451f7e7..f1d543201d 100755 --- a/.github/checks/copyright.sh +++ b/.github/checks/copyright.sh @@ -4,7 +4,7 @@ count=0 for file in $(find . -name '*.rs' | grep -v '/target') do - if ! grep 'Copyright \(....-\)\?202[12], The Tremor Team' "$file" > /dev/null + if ! grep 'Copyright \(....-\)\?202[123], The Tremor Team' "$file" > /dev/null then echo "##[error] Copyright missing in $file" count=$((count + 1)) diff --git a/.github/checks/safety.sh b/.github/checks/safety.sh index 0190f3df90..199285b81a 100755 --- a/.github/checks/safety.sh +++ b/.github/checks/safety.sh @@ -25,7 +25,7 @@ EOF -files=$(find . -name '*.rs' | grep -v -f .checkignore | grep -v 'test.rs$') +files=$(find . -name '*.rs' | grep -v -f .checkignore | grep -v 'test.rs$' | grep -v '/test/') while getopts hamuiprebldxcft opt; do case $opt in diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index e947c2e524..60dd70bf03 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -31,8 +31,6 @@ jobs: - uses: actions/checkout@v3 - name: Check for unused dependencies (tremor-runtime) run: ./.github/checks/deps.sh . - - name: Check for unused dependencies (tremor-api) - run: ./.github/checks/deps.sh tremor-api - name: Check for unused dependencies (tremor-influx) run: ./.github/checks/deps.sh tremor-influx - name: Check for unused dependencies (tremor-pipeline) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index b3f0a51795..20fd45d51d 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -30,7 +30,7 @@ jobs: env: TREMOR_PATH: "${{ github.workspace }}/tremor-script/lib:${{ github.workspace }}/tremor-cli/tests/lib" - RUSTFLAGS: -D warnings -C target-feature=${{ matrix.target_feature }} --cfg tokio_unstable + RUSTFLAGS: -D warnings -C target-feature=${{ matrix.target_feature }} runs-on: ${{ matrix.os }} steps: diff --git a/.vscode/launch.json b/.vscode/launch.json index 2421a0797e..71989671ff 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,11 @@ "request": "launch", "name": "Debug tremor-cli", "cargo": { - "args": ["build", "-p", "tremor-cli"], + "args": [ + "build", + "-p", + "tremor-cli" + ], "filter": { "name": "tremor", "kind": "bin" @@ -48,46 +52,41 @@ { "type": "lldb", "request": "launch", - "name": "Debug value::test::obj_eq unit test", + "name": "Debug raft::test::cluster_test unit test", "cargo": { "args": [ "test", "--no-run", - "--package", - "tremor-value", - "value::test::obj_eq" - ] + "--package=tremor-runtime", + ], + "filter": { + "name": "tremor-runtime", + "kind": "lib" + } }, - "args": [], + "args": [ + "raft::test::cluster_test" + ], "cwd": "${workspaceFolder}" }, - { "type": "lldb", "request": "launch", "name": "Debug script_error unit test", "cargo": { - "args": ["test", "--no-run", "pp_embed_unrecognized_token2"], + "args": [ + "test", + "--no-run", + "pp_embed_unrecognized_token2" + ], "filter": { "name": "script_error", "kind": "test" } }, - "args": ["${selectedText}"], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "Debug unit tests in library 'tremor-api'", - "cargo": { - "args": ["test", "--no-run", "--lib", "--package=tremor-api"], - "filter": { - "name": "tremor-api", - "kind": "lib" - } - }, - "args": [], + "args": [ + "${selectedText}" + ], "cwd": "${workspaceFolder}" }, { @@ -95,7 +94,12 @@ "request": "launch", "name": "Debug unit tests in library 'tremor-pipeline'", "cargo": { - "args": ["test", "--no-run", "--lib", "--package=tremor-pipeline"], + "args": [ + "test", + "--no-run", + "--lib", + "--package=tremor-pipeline" + ], "filter": { "name": "tremor-pipeline", "kind": "lib" @@ -109,13 +113,19 @@ "request": "launch", "name": "Debug integration test 'query'", "cargo": { - "args": ["test", "--no-run", "--test=query"], + "args": [ + "test", + "--no-run", + "--test=query" + ], "filter": { "name": "query", "kind": "test" } }, - "args": ["window_mixed_1"], + "args": [ + "window_mixed_1" + ], "cwd": "${workspaceFolder}" }, { @@ -123,7 +133,12 @@ "request": "launch", "name": "Debug unit tests in library 'tremor-influx'", "cargo": { - "args": ["test", "--no-run", "--lib", "--package=tremor-influx"], + "args": [ + "test", + "--no-run", + "--lib", + "--package=tremor-influx" + ], "filter": { "name": "tremor-influx", "kind": "lib" @@ -156,7 +171,12 @@ "request": "launch", "name": "Debug unit tests in library 'tremor_script'", "cargo": { - "args": ["test", "--no-run", "--lib", "--package=tremor-script"], + "args": [ + "test", + "--no-run", + "--lib", + "--package=tremor-script" + ], "filter": { "name": "tremor_script", "kind": "lib" @@ -170,7 +190,11 @@ "request": "launch", "name": "Debug executable 'tremor-script'", "cargo": { - "args": ["build", "--bin=tremor-script", "--package=tremor-script"], + "args": [ + "build", + "--bin=tremor-script", + "--package=tremor-script" + ], "filter": { "name": "tremor-script", "kind": "bin" @@ -203,7 +227,12 @@ "request": "launch", "name": "Debug unit tests in library 'tremor-runtime'", "cargo": { - "args": ["test", "--no-run", "--lib", "--package=tremor-runtime"], + "args": [ + "test", + "--no-run", + "--lib", + "--package=tremor-runtime" + ], "filter": { "name": "tremor-runtime", "kind": "lib" @@ -350,7 +379,11 @@ "request": "launch", "name": "Debug executable 'tremor-query'", "cargo": { - "args": ["build", "--bin=tremor-query", "--package=tremor-query"], + "args": [ + "build", + "--bin=tremor-query", + "--package=tremor-query" + ], "filter": { "name": "tremor-query", "kind": "bin" @@ -383,7 +416,11 @@ "request": "launch", "name": "Debug executable 'tremor'", "cargo": { - "args": ["build", "--bin=tremor", "--package=tremor-cli"], + "args": [ + "build", + "--bin=tremor", + "--package=tremor-cli" + ], "filter": { "name": "tremor", "kind": "bin" @@ -397,7 +434,11 @@ "request": "launch", "name": "Debug real-workflow-throughput-json", "cargo": { - "args": ["build", "--bin=tremor", "--package=tremor-cli"], + "args": [ + "build", + "--bin=tremor", + "--package=tremor-cli" + ], "filter": { "name": "tremor", "kind": "bin" @@ -416,7 +457,12 @@ "request": "launch", "name": "Release real-workflow-throughput-json", "cargo": { - "args": ["build", "--bin=tremor", "--package=tremor-cli", "--release"], + "args": [ + "build", + "--bin=tremor", + "--package=tremor-cli", + "--release" + ], "filter": { "name": "tremor", "kind": "bin" @@ -435,7 +481,12 @@ "request": "launch", "name": "Debug unit tests in executable 'tremor'", "cargo": { - "args": ["test", "--no-run", "--bin=tremor", "--package=tremor-cli"], + "args": [ + "test", + "--no-run", + "--bin=tremor", + "--package=tremor-cli" + ], "filter": { "name": "tremor", "kind": "bin" @@ -449,7 +500,11 @@ "request": "launch", "name": "Debug executable 'tremor-tool'", "cargo": { - "args": ["build", "--bin=tremor-tool", "--package=tremor-tool"], + "args": [ + "build", + "--bin=tremor-tool", + "--package=tremor-tool" + ], "filter": { "name": "tremor-tool", "kind": "bin" @@ -478,4 +533,4 @@ "cwd": "${workspaceFolder}" } ] -} +} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 2bc495f692..5f6f6d4d9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,7 +49,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cipher 0.4.4", "cpufeatures", ] @@ -94,7 +94,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "getrandom 0.2.10", "once_cell", "version_check", @@ -185,6 +185,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "anyerror" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eef94a982ad4ebc3036bb072f45fcbb63a6b1b91932a6f5d982a166d8d529916" +dependencies = [ + "serde", +] + [[package]] name = "anyhow" version = "1.0.75" @@ -225,6 +234,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + [[package]] name = "arrayvec" version = "0.5.2" @@ -329,7 +344,7 @@ checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ "async-lock", "autocfg", - "cfg-if", + "cfg-if 1.0.0", "concurrent-queue", "futures-lite", "log", @@ -360,7 +375,7 @@ dependencies = [ "async-lock", "async-signal", "blocking", - "cfg-if", + "cfg-if 1.0.0", "event-listener 3.0.0", "futures-lite", "rustix 0.38.20", @@ -378,6 +393,27 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "async-session" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345022a2eed092cd105cc1b26fd61c341e100bd5fcbbd792df4baf31c2cc631f" +dependencies = [ + "anyhow", + "async-std", + "async-trait", + "base64 0.12.3", + "bincode", + "blake3", + "chrono", + "hmac 0.8.1", + "kv-log-macro", + "rand 0.7.3", + "serde", + "serde_json", + "sha2 0.9.9", +] + [[package]] name = "async-signal" version = "0.2.4" @@ -387,7 +423,7 @@ dependencies = [ "async-io", "async-lock", "atomic-waker", - "cfg-if", + "cfg-if 1.0.0", "futures-core", "futures-io", "rustix 0.38.20", @@ -946,6 +982,56 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e246206a63c9830e118d12c894f56a82033da1a2361f5544deeee3df85c99d9" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa 1.0.9", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite 0.2.13", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded 0.7.1", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -954,7 +1040,7 @@ checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide", "object", @@ -1025,6 +1111,35 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.63.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36d860121800b2a9a94f9b5604b332d5cffb234ce17609ea479d723dbc9d3885" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 1.0.109", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -1052,6 +1167,21 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "blake3" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 0.1.10", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -1113,6 +1243,16 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "byte-unit" +version = "4.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c" +dependencies = [ + "serde", + "utf8-width", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -1194,6 +1334,21 @@ dependencies = [ "libc", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -1296,6 +1451,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.4.7" @@ -1483,6 +1649,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "cookie" version = "0.14.4" @@ -1555,7 +1727,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1611,7 +1783,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -1625,7 +1797,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", ] @@ -1635,7 +1807,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] @@ -1647,7 +1819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", "memoffset", "scopeguard", @@ -1659,7 +1831,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", ] @@ -1669,7 +1841,7 @@ version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1688,6 +1860,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "crypto-mac" version = "0.10.1" @@ -1775,7 +1957,7 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown 0.14.2", "lock_api", "once_cell", @@ -1830,6 +2012,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "syn 1.0.109", +] + [[package]] name = "destructure_traitobject" version = "0.2.0" @@ -1883,7 +2078,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -1948,6 +2143,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +[[package]] +name = "educe" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0188e3c3ba8df5753894d54461f0e39bc91741dc5b22e1c46999ec2c71f4e4" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "either" version = "1.9.0" @@ -1998,7 +2205,7 @@ version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2013,6 +2220,20 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "enum-ordinalize" +version = "3.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bb1df8b45ecb7ffa78dca1c17a438fb193eb083db0b1b494d2a61bcb5096a" +dependencies = [ + "num-bigint 0.4.4", + "num-traits", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "syn 1.0.109", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -2093,6 +2314,22 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "femme" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc04871e5ae3aa2952d552dae6b291b3099723bf779a8054281c1366a54613ef" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "file-mode" version = "0.1.2" @@ -2108,7 +2345,7 @@ version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.3.5", "windows-sys 0.48.0", @@ -2328,7 +2565,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -2339,7 +2576,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -2470,7 +2707,7 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crunchy", ] @@ -2570,13 +2807,23 @@ dependencies = [ "hmac 0.10.1", ] +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "hmac" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac", + "crypto-mac 0.10.1", "digest 0.9.0", ] @@ -2641,7 +2888,7 @@ dependencies = [ "async-std", "async-tls", "async-trait", - "cfg-if", + "cfg-if 1.0.0", "dashmap", "deadpool", "futures", @@ -2650,6 +2897,12 @@ dependencies = [ "rustls 0.18.1", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + [[package]] name = "http-types" version = "2.12.0" @@ -2864,7 +3117,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -3026,6 +3279,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lexical" version = "6.1.1" @@ -3043,7 +3302,7 @@ checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "ryu", "static_assertions", ] @@ -3142,12 +3401,37 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if 1.0.0", + "winapi", +] + [[package]] name = "libm" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "librocksdb-sys" +version = "0.8.0+7.7.3" +source = "git+https://github.com/tremor-rs/rust-rocksdb?rev=a02b00f029fb45d294715748cd2459491594e0e3#a02b00f029fb45d294715748cd2459491594e0e3" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + [[package]] name = "libz-sys" version = "1.1.12" @@ -3279,6 +3563,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "match_cfg" version = "0.1.0" @@ -3291,6 +3581,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "matrixmultiply" version = "0.3.8" @@ -3307,7 +3603,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "digest 0.10.7", ] @@ -3569,6 +3865,27 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openraft" +version = "0.8.4" +source = "git+https://github.com/datafuselabs/openraft.git?rev=6098f5cf61b074c8fccecf722278183c3ee5cd27#6098f5cf61b074c8fccecf722278183c3ee5cd27" +dependencies = [ + "anyerror", + "async-trait", + "byte-unit", + "clap", + "derive_more", + "futures", + "maplit", + "pin-utils", + "rand 0.8.5", + "serde", + "thiserror", + "tokio", + "tracing", + "tracing-futures", +] + [[package]] name = "openssl" version = "0.10.57" @@ -3576,7 +3893,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ "bitflags 2.4.1", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -3613,6 +3930,49 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite 0.2.13", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand 0.8.5", + "thiserror", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -3670,7 +4030,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall 0.2.16", @@ -3684,7 +4044,7 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.4.1", "smallvec", @@ -3729,6 +4089,12 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "pem" version = "0.8.3" @@ -3900,7 +4266,7 @@ checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "concurrent-queue", "libc", "log", @@ -4462,6 +4828,15 @@ dependencies = [ "serde", ] +[[package]] +name = "rocksdb" +version = "0.19.0" +source = "git+https://github.com/tremor-rs/rust-rocksdb?rev=a02b00f029fb45d294715748cd2459491594e0e3#a02b00f029fb45d294715748cd2459491594e0e3" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "route-recognizer" version = "0.2.0" @@ -4514,6 +4889,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -4827,6 +5208,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +dependencies = [ + "serde", +] + [[package]] name = "serde_qs" version = "0.8.5" @@ -4920,7 +5310,7 @@ dependencies = [ "base64 0.21.5", "bitflags 1.3.2", "bytes", - "cfg-if", + "cfg-if 1.0.0", "dashmap", "flate2", "futures", @@ -4970,7 +5360,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -4990,7 +5380,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -5008,7 +5398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -5020,17 +5410,32 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "shell-words" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "signal-hook" version = "0.3.17" @@ -5356,7 +5761,7 @@ checksum = "718b1ae6b50351982dedff021db0def601677f2120938b070eadb10ba4038dd7" dependencies = [ "async-std", "async-trait", - "cfg-if", + "cfg-if 1.0.0", "encoding_rs", "futures-util", "getrandom 0.2.10", @@ -5462,6 +5867,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "syslog_loose" version = "0.19.0" @@ -5504,6 +5915,41 @@ dependencies = [ "xattr", ] +[[package]] +name = "tarpc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f41bce44d290df0598ae4b9cd6ea7f58f651fd3aa4af1b26060c4fa32b08af7" +dependencies = [ + "anyhow", + "fnv", + "futures", + "humantime", + "opentelemetry", + "pin-project", + "rand 0.8.5", + "serde", + "static_assertions", + "tarpc-plugins", + "thiserror", + "tokio", + "tokio-serde", + "tokio-util 0.7.10", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "tarpc-plugins" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee42b4e559f17bce0385ebf511a7beb67d5cc33c12c96b7f4e9789919d9c10f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tch" version = "0.13.0" @@ -5533,7 +5979,7 @@ version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand 2.0.1", "redox_syscall 0.3.5", "rustix 0.38.20", @@ -5575,7 +6021,7 @@ version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54c25e2cb8f5fcd7318157634e8838aa6f7e4715c96637f969fabaccd1ef5462" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "proc-macro-error", "proc-macro2", "quote", @@ -5650,7 +6096,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", ] @@ -5661,9 +6107,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c459573f0dd2cc734b539047f57489ea875af8ee950860ded20cf93a79a1dee0" dependencies = [ "async-h1", + "async-session", "async-sse", "async-std", "async-trait", + "femme", "futures-util", "http-client", "http-types", @@ -5858,6 +6306,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-serde" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" +dependencies = [ + "bincode", + "bytes", + "educe", + "futures-core", + "futures-sink", + "pin-project", + "serde", + "serde_json", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -5905,6 +6369,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite 0.2.13", + "slab", "tokio", "tracing", ] @@ -6002,6 +6467,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "bitflags 1.3.2", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite 0.2.13", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.2" @@ -6044,6 +6528,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", ] [[package]] @@ -6057,24 +6542,27 @@ dependencies = [ ] [[package]] -name = "tremor-api" -version = "0.13.0-rc.16" +name = "tracing-opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" dependencies = [ - "env_logger", - "halfbrown", - "http-types", - "log", - "serde", - "serde_yaml 0.9.27", - "simd-json", - "surf", - "tide", - "tokio", - "tremor-common", - "tremor-pipeline", - "tremor-runtime", - "tremor-script", - "tremor-value", + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", ] [[package]] @@ -6100,6 +6588,7 @@ dependencies = [ "port_scanner", "pretty_assertions", "serde", + "serde_json", "serde_yaml 0.9.27", "shell-words", "signal-hook", @@ -6111,7 +6600,6 @@ dependencies = [ "temp-dir", "termcolor", "tokio", - "tremor-api", "tremor-codec", "tremor-common", "tremor-interceptor", @@ -6171,6 +6659,7 @@ dependencies = [ "serde", "simd-json", "simd-json-derive", + "tempfile", "test-case", "tokio", "url", @@ -6253,7 +6742,6 @@ dependencies = [ "error-chain", "halfbrown", "indexmap 2.0.2", - "lazy_static", "log", "lru", "petgraph", @@ -6291,9 +6779,11 @@ dependencies = [ "aws-sdk-s3", "aws-smithy-http", "aws-types", + "axum", "base64 0.21.5", "beef", "bimap", + "byteorder", "bytes", "chrono", "chrono-tz", @@ -6323,12 +6813,14 @@ dependencies = [ "hyper-rustls", "indexmap 2.0.2", "itoa 1.0.9", + "jumphash", "lazy_static", "lexical", "log", "matches", "mime", "num_cpus", + "openraft", "pin-project-lite 0.2.13", "port_scanner", "pretty_assertions", @@ -6341,14 +6833,18 @@ dependencies = [ "rdkafka-sys", "regex", "reqwest", + "rmp-serde", + "rocksdb", "rustls 0.21.8", "rustls-native-certs", "rustls-pemfile", "ryu", "serde", + "serde_json", "serde_yaml 0.9.27", "serenity", "serial_test", + "sha2 0.10.8", "signal-hook", "signal-hook-tokio", "simd-json", @@ -6356,9 +6852,12 @@ dependencies = [ "simdutf8", "sled", "socket2 0.5.5", + "tar", + "tarpc", "tempfile", "test-case", "testcontainers", + "tide", "tokio", "tokio-rustls 0.24.1", "tokio-stream", @@ -6457,7 +6956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" dependencies = [ "async-trait", - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "enum-as-inner", "futures-channel", @@ -6481,7 +6980,7 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "futures-util", "ipconfig", "lru-cache", @@ -6707,6 +7206,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-width" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" + [[package]] name = "utf8parse" version = "0.2.1" @@ -6723,6 +7228,12 @@ dependencies = [ "serde", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "value-bag" version = "1.4.2" @@ -6867,7 +7378,9 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -6892,7 +7405,7 @@ version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -7222,7 +7735,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "windows-sys 0.48.0", ] diff --git a/Cargo.toml b/Cargo.toml index 12fa9b4dff..f11b62d8a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ rust-version = "1.62" [workspace] members = [ - "tremor-api", "tremor-cli", "tremor-codec", "tremor-common", @@ -49,6 +48,7 @@ beef = { version = "0.5", features = ["impl_serde"] } bimap = { version = "0.6", features = ["serde"] } chrono = "0.4" chrono-tz = "0.8" +jumphash = "0.1" # Once a new version of clickhouse-rs is released, we can stop using a git # repository as a dependency. The declaration can then be replaced with the @@ -91,12 +91,14 @@ tremor-config = { path = "tremor-config" } tremor-codec = { path = "tremor-codec" } tremor-influx = { path = "tremor-influx" } tremor-pipeline = { path = "tremor-pipeline" } -tremor-script = { path = "tremor-script" } +tremor-script = { path = "tremor-script", features = ["arena-delete"] } tremor-value = { path = "tremor-value" } tremor-interceptor = { path = "tremor-interceptor" } url = "2.4" value-trait = "0.8" - +tide = "0.16" +byteorder = "1.5" +axum = "0.6" # blaster / blackhole# codecs hdrhistogram = "7" @@ -194,6 +196,17 @@ ryu = "1" lexical = "6" simdutf8 = "0.1" +# Clustering +tarpc = { version = "0.33", features = ["full"] } +openraft = { version = "0.8.4", features = [ + "serde", +], git = "https://github.com/datafuselabs/openraft.git", rev = "6098f5cf61b074c8fccecf722278183c3ee5cd27" } +serde_json = "1.0.57" +rocksdb = { version = "0.19.0", git = "https://github.com/tremor-rs/rust-rocksdb", rev = "a02b00f029fb45d294715748cd2459491594e0e3" } +tar = "0.4" +sha2 = "0.10" +rmp-serde = "1" + [dev-dependencies] port_scanner = "0.1" serial_test = { version = "2.0", features = ["logging"] } @@ -213,10 +226,10 @@ num_cpus = "1" bytes = "1" [features] -default = [] +default = ["integration"] # support for 128bit numbers in tremor-value -128bit = ["tremor-value/128bit"] +128bit = ["tremor-value/128bit", "simd-json/128bit", "value-trait/128bit"] bert = ["tremor-pipeline/bert"] integration = ["integration-docker", "integration-local"] @@ -250,7 +263,6 @@ socket-integration = [] net-integration = [] wal-integration = [] clickhouse-integration = [] -tarpaulin-exclude = [] # those are falky tests flaky-test = [] diff --git a/Cluster.md b/Cluster.md new file mode 100644 index 0000000000..d1522dda36 --- /dev/null +++ b/Cluster.md @@ -0,0 +1,42 @@ + +# Initialize +```bash +rm -r temp/test-db*; cargo run -p tremor-cli -- cluster bootstrap --db-dir temp/test-db1 --api 127.0.0.1:8001 --rpc 127.0.0.1:9001 +``` + +# Join +``` +target/debug/tremor cluster start --db-dir temp/test-db2 --api 127.0.0.1:8002 --rpc 127.0.0.1:9002 --join 127.0.0.1:8001 +target/debug/tremor cluster start --db-dir temp/test-db3 --api 127.0.0.1:8003 --rpc 127.0.0.1:9003 --join 127.0.0.1:8002 +``` + +# Restart +``` +target/debug/tremor cluster start --db-dir temp/test-db2 --api 127.0.0.1:8002 --rpc 127.0.0.1:9002 +target/debug/tremor cluster start --db-dir temp/test-db3 --api 127.0.0.1:8003 --rpc 127.0.0.1:9003 +``` + + +# package +```bash +./target/debug/tremor cluster package --out temp/tick.tar temp/tick.troy +./target/debug/tremor cluster package --out temp/cs.tar temp/clustered_storage/cs.troy +``` +# install +```bash +./target/debug/tremor cluster apps install temp/cs.tar && \ +./target/debug/tremor cluster apps start cs test +``` + +```bash +./target/debug/tremor cluster apps install temp/tick.tar +./target/debug/tremor cluster apps start tick test +``` + + +websocat ws://0.0.0.0:8080 +{"get": "the-key"} + +{"put": "the-key", "data": "snot"} +{"put": "the-key", "data": "badger"} + diff --git a/Dockerfile b/Dockerfile index e21b5dd546..c1871039b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update \ # Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog -ENV RUSTFLAGS="-C target-feature=+avx,+avx2,+sse4.2 --cfg tokio_unstable" +ENV RUSTFLAGS="-C target-feature=+avx,+avx2,+sse4.2" WORKDIR /app @@ -27,7 +27,6 @@ COPY .cargo /app/.cargo COPY tremor-pipeline /app/tremor-pipeline COPY tremor-script /app/tremor-script COPY tremor-script-nif /app/tremor-script-nif -COPY tremor-api /app/tremor-api COPY tremor-influx /app/tremor-influx COPY tremor-value /app/tremor-value # Binaries diff --git a/Dockerfile.cluster b/Dockerfile.cluster new file mode 100644 index 0000000000..2a83032233 --- /dev/null +++ b/Dockerfile.cluster @@ -0,0 +1,84 @@ +FROM rust:1.65-bullseye as builder + +# Avoid warnings by switching to noninteractive +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y libclang-dev cmake git \ + # + # Clean up + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +# Switch back to dialog for any ad-hoc use of apt-get +ENV DEBIAN_FRONTEND=dialog +ENV RUSTFLAGS="-C target-feature=+avx,+avx2,+sse4.2" + +WORKDIR /app + +COPY Cargo.* /app/ + +# We change lto to 'thin' for docker builds so it +# can be build on more moderate system +RUN mv Cargo.toml Cargo.toml.orig && sed 's/lto = true/lto = "thin"/' Cargo.toml.orig > Cargo.toml + +# Main library +COPY src /app/src +COPY build.rs /app/build.rs +COPY .cargo /app/.cargo +# supporting libraries +COPY tremor-pipeline /app/tremor-pipeline +COPY tremor-script /app/tremor-script +COPY tremor-script-nif /app/tremor-script-nif +COPY tremor-influx /app/tremor-influx +COPY tremor-value /app/tremor-value +COPY tremor-common /app/tremor-common +# Binaries +COPY tremor-cli /app/tremor-cli +# Git info to track version +COPY .git /app/.git + +RUN cat /proc/cpuinfo + +# RUN cargo build --release -p tremor-cli --verbose +# RUN strip target/release/tremor +RUN cargo build -p tremor-cli --verbose +RUN strip target/debug/tremor + +FROM debian:bullseye-slim + +RUN useradd -ms /bin/bash tremor + +RUN apt-get update \ + && apt-get install -y libatomic1 tini curl \ + # + # Clean up + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +#COPY --from=builder /app/target/release/tremor /tremor +COPY --from=builder /app/target/debug/tremor /tremor + +# stdlib +RUN mkdir -p /usr/share/tremor/lib +COPY tremor-script/lib /usr/share/tremor/lib + +# Entrypoint +COPY docker/cluster.sh /entrypoint.sh +# configuration file +RUN mkdir /etc/tremor +RUN mkdir /data +COPY docker/config /etc/tremor/config +# logger configuration +COPY docker/logger.yaml /etc/tremor/logger.yaml + +# setting TREMOR_PATH +# /usr/local/share/tremor - for host-specific local tremor-script modules and libraries, takes precedence +# /usr/share/tremor/lib - place for the tremor-script stdlib +ENV TREMOR_PATH="/usr/local/share/tremor:/usr/share/tremor/lib" + +ENTRYPOINT ["/entrypoint.sh"] + +HEALTHCHECK --interval=30s --timeout=1s --start-period=5s --retries=3 CMD curl -f http://localhost:9898/v1/status || exit 1 \ No newline at end of file diff --git a/Dockerfile.native b/Dockerfile.native index 421eb6676f..65d97f85e5 100644 --- a/Dockerfile.native +++ b/Dockerfile.native @@ -25,7 +25,6 @@ COPY src ./src COPY tremor-pipeline ./tremor-pipeline COPY tremor-script ./tremor-script COPY tremor-script-nif /app/tremor-script-nif -COPY tremor-api ./tremor-api COPY tremor-influx ./tremor-influx COPY tremor-value ./tremor-value # Binaries diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 4f2f2bd54e..b6c334b2ba 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -5,7 +5,6 @@ * Update version in all Cargo.toml files in the repository - ./Cargo.toml - - ./tremor-api/Cargo.toml - ./tremor-cli/Cargo.toml - ./tremor-common/Cargo.toml - ./tremor-influx/Cargo.toml (if we have changes compared to the previous release) diff --git a/docker/cluster.sh b/docker/cluster.sh new file mode 100755 index 0000000000..72512e6b69 --- /dev/null +++ b/docker/cluster.sh @@ -0,0 +1,62 @@ +#!/bin/sh +set -x + +# Possible environment variables: +# TREMOR_DIR: The base directory for tremor config (`/etc/tremor`) +# CFG_DIR: Directory to load config from (`${TREMOR_DIR}/config`) +# LOGGER_FILE: the logger configuration (`${TREMOR_DIR}/logger.yaml`) +# SLEEP: Number of seconds to sleep before starting tremor + +if [ -n "${SLEEP+x}" ] +then + sleep "$SLEEP" +fi + +if [ -z ${DATA_DIR+x} ] +then + echo "DATA_DIR is not set, defaulting to /data" + DATA_DIR="/data" +fi + +if [ ! -d "${DATA_DIR}" ] +then + mkdir -p "${DATA_DIR}" +fi + +if [ -z ${TREMOR_DIR+x} ] +then + TREMOR_DIR="/etc/tremor" +fi + +if [ -z ${LOGGER_FILE+x} ] +then + LOGGER_FILE="${TREMOR_DIR}/logger.yaml" +fi + +if [ -z ${RPC_IP+x} ] +then + echo "RPC_IP is not set, defaulting to ${POD_IP}:9000" + RPC_IP="${POD_IP}:9000" +fi + +if [ -z ${API_IP+x} ] +then + echo "API_IP is not set, defaulting to ${POD_IP}:8000" + API_IP="${POD_IP}:8000" +fi + +if [ ! -z ${WORKER+x} ] +then + ARGS="--remove-on-sigterm --passive" +fi + +# tremor-0.tremor.default.svc.cluster.local +# export MY_NODE_NAME='minikube' +# export MY_POD_NAME='tremor-0' +# export MY_POD_NAMESPACE='default' +if [ ${TREMOR_SEED} = "${MY_POD_NAME}.tremor.${MY_POD_NAMESPACE}.svc.cluster.local" ] +then + exec /usr/bin/tini /tremor -- cluster bootstrap --db-dir "${DATA_DIR}" --rpc "${RPC_IP}" --api "${API_IP}" ${ARGS} +else + exec /usr/bin/tini /tremor -- cluster start --db-dir "${DATA_DIR}" --rpc "${RPC_IP}" --api "${API_IP}" --join "${TREMOR_SEED}:8000" ${ARGS} +fi \ No newline at end of file diff --git a/examples/kv/kv.troy b/examples/kv/kv.troy new file mode 100644 index 0000000000..9eaab5df4a --- /dev/null +++ b/examples/kv/kv.troy @@ -0,0 +1,23 @@ +define flow main +flow + use tremor::pipelines as tp; + use tremor::connectors; + use lib::pipelines as lp; + + + define connector cluster from cluster_kv; + create connector console from connectors::console; + create connector cluster; + + + create pipeline main from lp::main; + create pipeline reply from tp::passthrough; + create pipeline debug from tp::passthrough; + + connect /connector/console to /pipeline/main; + connect /pipeline/main to /connector/cluster; + + connect /connector/cluster to /pipeline/reply; + + connect /pipeline/reply to /connector/console; +end; \ No newline at end of file diff --git a/examples/kv/lib/pipelines.tremor b/examples/kv/lib/pipelines.tremor new file mode 100644 index 0000000000..b42d55f3e4 --- /dev/null +++ b/examples/kv/lib/pipelines.tremor @@ -0,0 +1,9 @@ + define pipeline main + pipeline + use lib::scripts; + + create script with_meta from scripts::with_meta; + select event from in into with_meta; + select event from with_meta into out; + select event from with_meta/err into err; + end; \ No newline at end of file diff --git a/examples/kv/lib/scripts.tremor b/examples/kv/lib/scripts.tremor new file mode 100644 index 0000000000..835ba1269e --- /dev/null +++ b/examples/kv/lib/scripts.tremor @@ -0,0 +1,14 @@ +define script with_meta +script + use std::json; + let decoded = json::decode(event); + match decoded of + case %{present get} => + let $kv = {"get": decoded.get}; + emit null + case %{present put, present data} => + let $kv = {"put": decoded.put}; + emit decoded.data + case _ => emit event => "err" + end +end; \ No newline at end of file diff --git a/examples/tick/lib/pipelines.tremor b/examples/tick/lib/pipelines.tremor new file mode 100644 index 0000000000..b26b9cc3c1 --- /dev/null +++ b/examples/tick/lib/pipelines.tremor @@ -0,0 +1,12 @@ +## Defines the main pipeline +define pipeline main +from in, console +pipeline + use lib::scripts; + + create script port_test from scripts::port_test; + select event from console into port_test/console; + select event from in into port_test; + select event from port_test into out; + select event from port_test/err into out; +end; \ No newline at end of file diff --git a/examples/tick/lib/scripts.tremor b/examples/tick/lib/scripts.tremor new file mode 100644 index 0000000000..da450aa91e --- /dev/null +++ b/examples/tick/lib/scripts.tremor @@ -0,0 +1,12 @@ +## The main script, the event gets processed here +define script port_test +state "initial" +script from console + let last = state; + let state = "console"; + emit "(c) last port used: #{last}"; +script + let last = state; + let state = "in"; + emit "(i) last port used: #{last}"; +end; \ No newline at end of file diff --git a/examples/tick/tick.troy b/examples/tick/tick.troy new file mode 100644 index 0000000000..64281a84f3 --- /dev/null +++ b/examples/tick/tick.troy @@ -0,0 +1,20 @@ +define flow main +flow + use lib::pipelines; + use tremor::connectors; + + use std::time::nanos; + use tremor::connectors; + define connector every_second from metronome + with + config = {"interval": nanos::from_seconds(2)} + end; + + create connector console from connectors::console; + create connector every_second; + create pipeline main from pipelines::main; + + connect /connector/console to /pipeline/main/console; + connect /connector/every_second to /pipeline/main; + connect /pipeline/main to /connector/console; +end; \ No newline at end of file diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000000..07e55d6e4f --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,30 @@ +# NO! +Experimental, for testing, don't use in production or if you do fix all the things. + +Based on https://kubernetes.io/docs/tutorials/stateful-application/cassandra/ + +this are purely notes + +``` +minikube start --cpus 16 --memory 8192 --disk-size 50000mb +eval $(minikube -p minikube docker-env) +docker build . -t tremor-cluster -f Dockerfile.cluster + +kubectl apply -f k8s/service.yml + +kubectl apply -f k8s/statefulset.yml + +kubectl get statefulset tremor +kubectl get pods -l="app=tremor" + +kubectl logs pod/tremor-0 +kubectl describe pods -l="app=tremor" + +kubectl exec -it tremor-0 -- /tremor cluster status --api tremor-0:8000 + +kubectl apply -f k8s/deployment.yml +kubectl get deployment tremor +kubectl autoscale deployment tremor --cpu-percent=50 --min=2 --max=10 + +``` + diff --git a/k8s/deployment.yml b/k8s/deployment.yml new file mode 100644 index 0000000000..7cace10a30 --- /dev/null +++ b/k8s/deployment.yml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tremor +spec: + selector: + matchLabels: + run: tremor + replicas: 1 + template: + metadata: + labels: + run: tremor + spec: + containers: + - name: tremor + image: tremor-cluster + imagePullPolicy: Never + ports: + - containerPort: 8000 + - containerPort: 9000 + resources: + limits: + cpu: 500m + requests: + cpu: 200m + env: + - name: WORKER + value: "true" + - name: TREMOR_SEED + value: "tremor-0.tremor.default.svc.cluster.local" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace \ No newline at end of file diff --git a/k8s/service.yml b/k8s/service.yml new file mode 100644 index 0000000000..03dda910c3 --- /dev/null +++ b/k8s/service.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: tremor + name: tremor +spec: + clusterIP: None + ports: + - port: 8000 + selector: + app: tremor diff --git a/k8s/statefulset.yml b/k8s/statefulset.yml new file mode 100644 index 0000000000..734da08a45 --- /dev/null +++ b/k8s/statefulset.yml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tremor + labels: + app: tremor +spec: + serviceName: tremor + replicas: 3 + selector: + matchLabels: + app: tremor + template: + metadata: + labels: + app: tremor + spec: + terminationGracePeriodSeconds: 1800 + containers: + - name: tremor + image: tremor-cluster + imagePullPolicy: Never + ports: + - containerPort: 8000 + name: api + - containerPort: 9000 + name: intra-node + resources: + limits: + cpu: "100m" + memory: 1Gi + requests: + cpu: "100m" + memory: 1Gi + # securityContext: + # capabilities: + # add: + # - IPC_LOCK + # lifecycle: + # preStop: + # exec: + # command: + # - /bin/sh + # - -c + # - nodetool drain + env: + - name: TREMOR_SEED + value: "tremor-0.tremor.default.svc.cluster.local" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # readinessProbe: + # exec: + # command: + # - /bin/bash + # - -c + # - /ready-probe.sh + # initialDelaySeconds: 15 + # timeoutSeconds: 5 + # These volume mounts are persistent. They are like inline claims, + # but not exactly because the names need to match exactly one of + # the stateful pod volumes. + # volumeMounts: + # - name: tremor-ring + # mountPath: /data + # These are converted to volume claims by the controller + # and mounted at the paths mentioned above. + # do not use these in production until ssd GCEPersistentDisk or other ssd pd +# volumeClaimTemplates: +# - metadata: +# name: tremor-ring +# spec: +# accessModes: [ "ReadWriteOnce" ] +# storageClassName: fast +# resources: +# requests: +# storage: 1Gi +# --- +# kind: StorageClass +# apiVersion: storage.k8s.io/v1 +# metadata: +# name: fast +# provisioner: k8s.io/minikube-hostpath +# parameters: +# type: pd-ssd diff --git a/packaging/cross_build.sh b/packaging/cross_build.sh index f0e7e044f6..f3adc5a146 100755 --- a/packaging/cross_build.sh +++ b/packaging/cross_build.sh @@ -113,7 +113,7 @@ echo "Successfully built the binary: ${TARGET_BIN}" # linking check echo "Printing linking information for the binary..." file "$TARGET_BIN" -ldd "$TARGET_BIN" +ldd "$TARGET_BIN" || true # back to the origin dir, just in case popd > /dev/null diff --git a/packaging/functions.sh b/packaging/functions.sh index ad5736e20b..c9b5ef58ac 100644 --- a/packaging/functions.sh +++ b/packaging/functions.sh @@ -47,7 +47,7 @@ function package_archive { # print package details echo "PACKAGE SIZE:" - du --human-readable --summarize "$archive_file" | awk '{print $1}' + du -h -s "$archive_file" | awk '{print $1}' echo "PACKAGE INFO:" gzip --list "$archive_file" echo "PACKAGE CONTENTS:" diff --git a/packaging/run.sh b/packaging/run.sh index aa7061317c..0980de0333 100755 --- a/packaging/run.sh +++ b/packaging/run.sh @@ -99,7 +99,7 @@ echo "Found the target binary: ${TARGET_BIN}" # get the package version from cargo manifest (assumption is first instance of # the regex match pattern here is the package version, which is true for most packages) -VERSION=$(grep --max-count 1 '^version\s*=' Cargo.toml | cut --delimiter '=' -f2 | tr --delete ' ' | tr --delete '"' || true) +VERSION=$(grep --max-count 1 '^version\s*=' Cargo.toml | cut -d '=' -f2 | tr -d ' ' | tr -d '"' || true) # # accurate determination, but depends on remarshal which won't be available by default #VERSION=$(remarshal -i Cargo.toml -of json | jq -r '.package.version') diff --git a/release.sh b/release.sh index c1ff1d7db8..f9d8f5c9bc 100755 --- a/release.sh +++ b/release.sh @@ -4,7 +4,6 @@ set -e TOML_FILES="\ Cargo.toml \ -tremor-api/Cargo.toml \ tremor-cli/Cargo.toml \ tremor-common/Cargo.toml \ tremor-influx/Cargo.toml \ diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000000..c0413448df --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +indent_style = "Block" +reorder_imports = true diff --git a/src/channel.rs b/src/channel.rs index 77e71a2c3f..d6ee327407 100644 --- a/src/channel.rs +++ b/src/channel.rs @@ -16,5 +16,8 @@ pub(crate) type Sender = tokio::sync::mpsc::Sender; pub(crate) type Receiver = tokio::sync::mpsc::Receiver; pub(crate) type UnboundedSender = tokio::sync::mpsc::UnboundedSender; pub(crate) type UnboundedReceiver = tokio::sync::mpsc::UnboundedReceiver; +pub(crate) type OneShotSender = tokio::sync::oneshot::Sender; +// pub(crate) type OneShotReceiver = tokio::sync::oneshot::Receiver; pub(crate) use tokio::sync::mpsc::error::{SendError, TryRecvError}; pub(crate) use tokio::sync::mpsc::{channel as bounded, unbounded_channel as unbounded}; +pub(crate) use tokio::sync::oneshot::channel as oneshot; diff --git a/src/config.rs b/src/config.rs index 2a470ccd16..56c13bc07d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -281,7 +281,7 @@ mod tests { #[test] fn test_config_builtin_preproc_with_config() -> Result<()> { let c = Connector::from_config( - &alias::Connector::new("flow", "my_otel_client"), + &alias::Connector::new("my_otel_client"), ConnectorType::from("otel_client".to_string()), &literal!({ "preprocessors": [ {"name": "snot", "config": { "separator": "\n" }}], @@ -312,9 +312,10 @@ mod tests { "reconnect": {}, "metrics_interval_s": "wrong_type" }); - let id = alias::Connector::new(tremor_common::alias::Flow::new("flow"), "my_id"); + + let id = alias::Connector::new("my_id"); let res = Connector::from_config(&id, "fancy_schmancy".into(), &config); assert!(res.is_err()); - assert_eq!(String::from("Invalid Definition for connector \"flow::my_id\": Expected type I64 for key metrics_interval_s but got String"), res.err().map(|e| e.to_string()).unwrap_or_default()); + assert_eq!(String::from("Invalid Definition for connector \"my_id\": Expected type I64 for key metrics_interval_s but got String"), res.err().map(|e| e.to_string()).unwrap_or_default()); } } diff --git a/src/connectors.rs b/src/connectors.rs index 0903e2f635..8c9a78b69f 100644 --- a/src/connectors.rs +++ b/src/connectors.rs @@ -27,37 +27,38 @@ pub(crate) mod utils; mod google; #[cfg(test)] -mod tests; +pub(crate) mod tests; -use self::metrics::{SinkReporter, SourceReporter}; -use self::sink::{SinkAddr, SinkContext, SinkMsg}; -use self::source::{SourceAddr, SourceContext, SourceMsg}; use self::utils::quiescence::QuiescenceBeacon; +use self::{prelude::Attempt, utils::reconnect}; +use self::{ + sink::{SinkAddr, SinkContext, SinkMsg}, + utils::{metrics::SourceReporter, reconnect::ConnectionLostNotifier}, +}; +use self::{ + source::{SourceAddr, SourceContext, SourceMsg}, + utils::{metrics::SinkReporter, reconnect::ReconnectRuntime}, +}; pub(crate) use crate::config::Connector as ConnectorConfig; use crate::{ channel::{bounded, Sender}, errors::{connector_send_err, Error, Kind as ErrorKind, Result}, instance::State, - log_error, pipeline, qsize, - system::{KillSwitch, World}, + log_error, pipeline, qsize, raft, + system::{flow::AppContext, KillSwitch, Runtime}, }; -use beef::Cow; use futures::Future; use halfbrown::HashMap; +use simd_json::prelude::*; use std::{fmt::Display, time::Duration}; use tokio::task::{self, JoinHandle}; use tremor_common::{ alias, - ids::{ConnectorId, ConnectorIdGen, SourceId}, ports::{Port, ERR, IN, OUT}, + uids::{ConnectorUId, ConnectorUIdGen, SourceUId}, }; -use tremor_pipeline::METRICS_CHANNEL; use tremor_script::ast::DeployEndpoint; use tremor_value::Value; -use utils::reconnect::{Attempt, ConnectionLostNotifier, ReconnectRuntime}; -/// quiescence stuff -pub(crate) use utils::{metrics, reconnect}; -use value_trait::prelude::*; /// Accept timeout pub(crate) const ACCEPT_TIMEOUT: Duration = Duration::from_millis(100); @@ -259,6 +260,12 @@ pub(crate) trait Context: Display + Clone { /// get the connector type fn connector_type(&self) -> &ConnectorType; + /// gets the API sender + fn raft(&self) -> &raft::Cluster; + + /// the application context + fn app_ctx(&self) -> &AppContext; + /// only log an error and swallow the result #[inline] fn swallow_err(&self, expr: std::result::Result, msg: &M) @@ -307,7 +314,7 @@ pub(crate) trait Context: Display + Clone { 'ct: 'event, { let t: &str = self.connector_type().into(); - event_meta.get(&Cow::borrowed(t)) + event_meta.get(t) } } @@ -322,11 +329,13 @@ pub(crate) struct ConnectorContext { quiescence_beacon: QuiescenceBeacon, /// Notifier notifier: reconnect::ConnectionLostNotifier, + /// sender for raft requests + app_ctx: AppContext, } impl Display for ConnectorContext { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "[Connector::{}]", &self.alias) + write!(f, "{}[Connector::{}]", self.app_ctx, &self.alias) } } @@ -346,6 +355,14 @@ impl Context for ConnectorContext { fn notifier(&self) -> &reconnect::ConnectionLostNotifier { &self.notifier } + + fn raft(&self) -> &raft::Cluster { + &self.app_ctx.raft + } + + fn app_ctx(&self) -> &AppContext { + &self.app_ctx + } } /// Connector instance status report @@ -422,14 +439,23 @@ pub(crate) type Known = /// if the connector can not be built or the config is invalid pub(crate) async fn spawn( alias: &alias::Connector, - connector_id_gen: &mut ConnectorIdGen, + connector_id_gen: &mut ConnectorUIdGen, builder: &dyn ConnectorBuilder, config: ConnectorConfig, kill_switch: &KillSwitch, + app_ctx: AppContext, ) -> Result { // instantiate connector - let connector = builder.build(alias, &config, kill_switch).await?; - let r = connector_task(alias.clone(), connector, config, connector_id_gen.next_id()).await?; + let connector = builder.build(alias, &config).await?; + let r = connector_task( + alias.clone(), + connector, + config, + connector_id_gen.next_id(), + app_ctx, + kill_switch.clone(), + ) + .await?; Ok(r) } @@ -440,7 +466,9 @@ async fn connector_task( alias: alias::Connector, mut connector: Box, config: ConnectorConfig, - uid: ConnectorId, + uid: ConnectorUId, + app_ctx: AppContext, + killswitch: KillSwitch, ) -> Result { let qsize = qsize(); // channel for connector-level control plane communication @@ -451,8 +479,9 @@ async fn connector_task( let notifier = ConnectionLostNotifier::new(msg_tx.clone()); let source_metrics_reporter = SourceReporter::new( + app_ctx.clone(), alias.clone(), - METRICS_CHANNEL.tx(), + app_ctx.metrics.tx(), config.metrics_interval_s, ); @@ -464,22 +493,25 @@ async fn connector_task( .into()); } let source_builder = source::builder( - SourceId::from(uid), + SourceUId::from(uid), &config, codec_requirement, source_metrics_reporter, )?; - let source_ctx = SourceContext { - alias: alias.clone(), - uid: uid.into(), - connector_type: config.connector_type.clone(), - quiescence_beacon: quiescence_beacon.clone(), - notifier: notifier.clone(), - }; + let source_ctx = SourceContext::new( + uid.into(), + alias.clone(), + config.connector_type.clone(), + quiescence_beacon.clone(), + notifier.clone(), + app_ctx.clone(), + killswitch.clone(), + ); let sink_metrics_reporter = SinkReporter::new( + app_ctx.clone(), alias.clone(), - METRICS_CHANNEL.tx(), + app_ctx.metrics.tx(), config.metrics_interval_s, ); let sink_builder = sink::builder(&config, codec_requirement, &alias, sink_metrics_reporter)?; @@ -489,6 +521,8 @@ async fn connector_task( config.connector_type.clone(), quiescence_beacon.clone(), notifier.clone(), + app_ctx.clone(), + killswitch, ); // create source instance let source_addr = connector.create_source(source_ctx, source_builder).await?; @@ -503,8 +537,7 @@ async fn connector_task( sink: sink_addr, }; - let mut reconnect: ReconnectRuntime = - ReconnectRuntime::new(&connector_addr, notifier.clone(), &config.reconnect); + let mut reconnect = ReconnectRuntime::new(&connector_addr, notifier.clone(), &config.reconnect); let notifier = reconnect.notifier(); let ctx = ConnectorContext { @@ -512,6 +545,7 @@ async fn connector_task( connector_type: config.connector_type.clone(), quiescence_beacon: quiescence_beacon.clone(), notifier, + app_ctx, }; let send_addr = connector_addr.clone(); @@ -1139,13 +1173,12 @@ pub(crate) trait ConnectorBuilder: Sync + Send + std::fmt::Debug { &self, alias: &alias::Connector, config: &ConnectorConfig, - kill_switch: &KillSwitch, ) -> Result> { let cc = config .config .as_ref() .ok_or_else(|| ErrorKind::MissingConfiguration(alias.to_string()))?; - self.build_cfg(alias, config, cc, kill_switch).await + self.build_cfg(alias, config, cc).await } /// create a connector from the given `alias`, outer `ConnectorConfig` and the connector-specific `connector_config` @@ -1157,7 +1190,6 @@ pub(crate) trait ConnectorBuilder: Sync + Send + std::fmt::Debug { _alias: &alias::Connector, _config: &ConnectorConfig, _connector_config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { Err("build_cfg is unimplemented".into()) } @@ -1168,41 +1200,43 @@ pub(crate) trait ConnectorBuilder: Sync + Send + std::fmt::Debug { #[must_use] pub(crate) fn builtin_connector_types() -> Vec> { vec![ + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), + Box::::default(), Box::::default(), Box::::default(), Box::::default(), Box::::default(), Box::::default(), - Box::::default(), - Box::::default(), + #[cfg(unix)] + Box::::default(), + #[cfg(unix)] + Box::::default(), Box::::default(), - Box::::default(), - Box::::default(), Box::::default(), Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - #[cfg(unix)] - Box::::default(), - #[cfg(unix)] - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), - Box::::default(), + Box::::default(), ] } @@ -1222,7 +1256,7 @@ pub(crate) fn debug_connector_types() -> Vec /// # Errors /// * If a builtin connector couldn't be registered -pub(crate) async fn register_builtin_connector_types(world: &World, debug: bool) -> Result<()> { +pub(crate) async fn register_builtin_connector_types(world: &Runtime, debug: bool) -> Result<()> { for builder in builtin_connector_types() { world.register_builtin_connector_type(builder).await?; } @@ -1259,6 +1293,7 @@ where #[cfg(test)] pub(crate) mod unit_tests { + use super::*; #[derive(Clone)] @@ -1267,15 +1302,17 @@ pub(crate) mod unit_tests { alias: alias::Connector, notifier: reconnect::ConnectionLostNotifier, beacon: QuiescenceBeacon, + app_ctx: AppContext, } impl FakeContext { pub(crate) fn new(tx: Sender) -> Self { Self { t: ConnectorType::from("snot"), - alias: alias::Connector::new(alias::Flow::new("fake"), "fake"), + alias: alias::Connector::new("fake"), notifier: reconnect::ConnectionLostNotifier::new(tx), beacon: QuiescenceBeacon::default(), + app_ctx: AppContext::default(), } } } @@ -1290,18 +1327,21 @@ pub(crate) mod unit_tests { fn alias(&self) -> &alias::Connector { &self.alias } - fn quiescence_beacon(&self) -> &QuiescenceBeacon { &self.beacon } - fn notifier(&self) -> &reconnect::ConnectionLostNotifier { &self.notifier } - fn connector_type(&self) -> &ConnectorType { &self.t } + fn raft(&self) -> &raft::Cluster { + &self.app_ctx.raft + } + fn app_ctx(&self) -> &AppContext { + &self.app_ctx + } } #[tokio::test(flavor = "multi_thread")] diff --git a/src/connectors/google.rs b/src/connectors/google.rs index 3b61cd9955..36dd48826f 100644 --- a/src/connectors/google.rs +++ b/src/connectors/google.rs @@ -158,13 +158,15 @@ where #[cfg(test)] #[cfg(feature = "gcp-integration")] pub(crate) mod tests { + use super::*; + use crate::{ + connectors::tests::free_port, + errors::{Error, Result}, + }; use hyper::{ service::{make_service_fn, service_fn}, Body, }; - - use super::*; - use crate::errors::{Error, Result}; use std::{convert::Infallible, io::Write, net::ToSocketAddrs}; #[derive(Clone)] @@ -250,7 +252,7 @@ PX8efvDMhv16QqDFF0k80d0= async fn gouth_token() -> Result<()> { let mut file = tempfile::NamedTempFile::new()?; - let port = crate::connectors::tests::free_port::find_free_tcp_port().await?; + let port = free_port::find_free_tcp_port().await?; let sa = ServiceAccount { client_email: "snot@tremor.rs".to_string(), private_key_id: "badger".to_string(), diff --git a/src/connectors/impls.rs b/src/connectors/impls.rs index 33d6054ba5..1411128ec9 100644 --- a/src/connectors/impls.rs +++ b/src/connectors/impls.rs @@ -48,6 +48,8 @@ pub(crate) mod metronome; /// Never send any events and swallow all events it receives into the void. pub(crate) mod null; +/// Cluster wide shared kv store +pub(crate) mod cluster_kv; /// `WebSockets` pub(crate) mod ws; @@ -66,3 +68,6 @@ pub(crate) mod udp; pub(crate) mod unix_socket; /// Write Ahead Log pub(crate) mod wal; + +/// One shot connector +pub(crate) mod oneshot; diff --git a/src/connectors/impls/bench.rs b/src/connectors/impls/bench.rs index 4ef6a3fca7..7a1c53790e 100644 --- a/src/connectors/impls/bench.rs +++ b/src/connectors/impls/bench.rs @@ -209,11 +209,7 @@ use std::{ io::{stdout, BufRead as StdBufRead, BufReader, Read, Write}, time::Duration, }; -use tremor_common::{ - base64::{Engine, BASE64}, - file, - time::nanotime, -}; +use tremor_common::{base64, file, time::nanotime}; use xz2::read::XzDecoder; #[derive(Deserialize, Debug, Clone)] @@ -262,7 +258,7 @@ pub(crate) struct Builder {} fn decode>(base64: bool, data: T) -> Result> { if base64 { - Ok(BASE64.decode(data)?) + Ok(base64::decode(data)?) } else { let d: &[u8] = data.as_ref(); Ok(d.to_vec()) @@ -276,7 +272,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, config: &Value, - kill_switch: &KillSwitch, ) -> Result> { let config: Config = Config::new(config)?; let mut source_data_file = file::open(&config.path)?; @@ -325,7 +320,6 @@ impl ConnectorBuilder for Builder { acc: Acc { elements, count: 0 }, origin_uri, stop_after, - kill_switch: kill_switch.clone(), })) } @@ -358,7 +352,6 @@ pub(crate) struct Bench { acc: Acc, origin_uri: EventOriginUri, stop_after: StopAfter, - kill_switch: KillSwitch, } #[async_trait::async_trait] @@ -385,7 +378,7 @@ impl Connector for Bench { ctx: SinkContext, builder: SinkManagerBuilder, ) -> Result> { - let sink = Blackhole::new(&self.config, self.stop_after, self.kill_switch.clone()); + let sink = Blackhole::new(&self.config, self.stop_after, ctx.killswitch()); Ok(Some(builder.spawn(sink, ctx))) } diff --git a/src/connectors/impls/cb.rs b/src/connectors/impls/cb.rs index 3f82d9b2be..eb0f769865 100644 --- a/src/connectors/impls/cb.rs +++ b/src/connectors/impls/cb.rs @@ -155,13 +155,9 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, raw: &Value, - kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(raw)?; - Ok(Box::new(Cb { - config, - kill_switch: kill_switch.clone(), - })) + Ok(Box::new(Cb { config })) } } @@ -178,7 +174,6 @@ impl ConnectorBuilder for Builder { /// * In case the pipeline branches off, it copies the event and it reaches two offramps, we might receive more than 1 ack or fail for an event with the current runtime. pub(crate) struct Cb { config: Config, - kill_switch: KillSwitch, } #[async_trait::async_trait()] @@ -199,7 +194,7 @@ impl Connector for Cb { ) .into()); } - let source = CbSource::new(&self.config, ctx.alias(), self.kill_switch.clone()).await?; + let source = CbSource::new(&self.config, ctx.alias(), ctx.killswitch()).await?; Ok(Some(builder.spawn(source, ctx))) } diff --git a/src/connectors/impls/clickhouse.rs b/src/connectors/impls/clickhouse.rs index 77ed8619ac..9ae4818e7a 100644 --- a/src/connectors/impls/clickhouse.rs +++ b/src/connectors/impls/clickhouse.rs @@ -306,7 +306,6 @@ impl ConnectorBuilder for Builder { _alias: &alias::Connector, _config: &ConnectorConfig, connector_config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = ClickhouseConfig::new(connector_config)?; diff --git a/src/connectors/impls/cluster_kv.rs b/src/connectors/impls/cluster_kv.rs new file mode 100644 index 0000000000..61e2ae4f6f --- /dev/null +++ b/src/connectors/impls/cluster_kv.rs @@ -0,0 +1,436 @@ +// Copyright 2021, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// #![cfg_attr(coverage, no_coverage)] +use crate::{ + channel::{bounded, Receiver, Sender}, + errors::already_created_error, + raft, +}; +use crate::{connectors::prelude::*, system::flow::AppContext}; +use serde::Deserialize; +use std::sync::Arc; +use std::{boxed::Box, convert::TryFrom, sync::atomic::AtomicBool}; +use tremor_codec::{ + json::{Json, Sorted}, + Codec, +}; +use tremor_common::alias::Generic; + +#[derive(Debug)] +enum Command { + /// Format: + /// ```json + /// {"get": "the-key", strict: true} + /// ``` + /// + /// Response: the value behing "the-key" or `null` + Get { key: String, strict: bool }, + /// Format: + /// ```json + /// {"put": "the-key"} + /// ``` + /// Event Payload: data to put here + /// Response: the putted value if successful + Put { key: String }, + // /// Format: + // /// ```json + // /// {"swap": "the-key"} + // /// ``` + // /// Event Payload: data to put here + // /// + // /// Response: the old value or `null` is there was no previous value for this key + // Swap { key: Vec }, + + // /// Format: + // /// ```json + // /// {"delete": "the-key"} + // /// ``` + // /// + // /// Response: the old value + // Delete { key: Vec }, + // /// Format: + // /// ```json + // /// { + // /// "start": "key1", + // /// "end": "key2", + // /// } + // /// ``` + // /// + // /// Response: 1 event for each value in the scanned range + // Scan { + // start: Vec, + // end: Option>, + // }, + // /// Format: + // /// ```json + // /// { + // /// "cas": "key", + // /// "old": "", + // /// } + // /// ``` + // /// EventPayload: event payload + // /// + // /// Response: `null` if the operation succeeded, an event on `err` if it failed + // Cas { + // key: Vec, + // old: Option<&'v Value<'v>>, + // }, +} + +impl<'v> TryFrom<&'v Value<'v>> for Command { + type Error = crate::Error; + + fn try_from(v: &'v Value<'v>) -> Result { + let v = v.get("kv").ok_or("Missing `$kv` field for commands")?; + if let Some(key) = v.get_str("get").map(ToString::to_string) { + Ok(Command::Get { + key, + strict: v.get_bool("strict").unwrap_or(false), + }) + } else if let Some(key) = v.get_str("put").map(ToString::to_string) { + Ok(Command::Put { key }) + // } else if let Some(key) = v.get_bytes("swap").map(<[u8]>::to_vec) { + // Ok(Command::Swap { key }) + // } else if let Some(key) = v.get_bytes("cas").map(<[u8]>::to_vec) { + // Ok(Command::Cas { + // key, + // old: v.get("old"), + // }) + // } else if let Some(key) = v.get_bytes("delete").map(<[u8]>::to_vec) { + // Ok(Command::Delete { key }) + // } else if let Some(start) = v.get_bytes("scan").map(<[u8]>::to_vec) { + // Ok(Command::Scan { + // start, + // end: v.get_bytes("end").map(<[u8]>::to_vec), + // }) + } else { + Err(format!("Invalid KV command: {v}").into()) + } + } +} + +impl Command { + fn op_name(&self) -> &'static str { + match self { + Command::Get { .. } => "get", + Command::Put { .. } => "put", + // Command::Swap { .. } => "swap", + // Command::Delete { .. } => "delete", + // Command::Scan { .. } => "scan", + // Command::Cas { .. } => "cas", + } + } + + fn key(&self) -> &str { + match self { + Command::Get { key, .. } | Command::Put { key, .. } => key, + // | Command::Swap { key, .. } + // | Command::Delete { key } + // | Command::Cas { key, .. } => Some(key.clone()), + // Command::Scan { .. } => None, + } + } +} + +fn ok(op_name: &'static str, k: String, v: Value<'static>) -> (Value<'static>, Value<'static>) { + ( + v, + literal!({ + "kv": { + "op": op_name, + "ok": k + } + }), + ) +} +fn oks( + op_name: &'static str, + k: String, + v: Value<'static>, +) -> Vec<(Value<'static>, Value<'static>)> { + vec![ok(op_name, k, v)] +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +pub(crate) struct Config {} + +impl tremor_config::Impl for Config {} + +#[derive(Debug, Default)] +pub(crate) struct Builder {} + +impl Builder {} +#[async_trait::async_trait] +impl ConnectorBuilder for Builder { + fn connector_type(&self) -> ConnectorType { + "cluster_kv".into() + } + async fn build( + &self, + _id: &alias::Connector, + _: &ConnectorConfig, + ) -> Result> { + let (tx, rx) = bounded(qsize()); + Ok(Box::new(Kv { + rx: Some(rx), + tx, + source_is_connected: Arc::default(), + })) + } +} + +/// Key value store connector +/// +/// Receiving commands via its sink and emitting responses to those commands via its source. +pub(crate) struct Kv { + rx: Option>, + tx: Sender, + source_is_connected: Arc, +} + +#[async_trait::async_trait] +impl Connector for Kv { + async fn create_source( + &mut self, + ctx: SourceContext, + builder: SourceManagerBuilder, + ) -> Result> { + let source = ChannelSource::from_channel( + self.tx.clone(), + self.rx.take().ok_or_else(already_created_error)?, + self.source_is_connected.clone(), + ); + Ok(Some(builder.spawn(source, ctx))) + } + + async fn create_sink( + &mut self, + ctx: SinkContext, + builder: SinkManagerBuilder, + ) -> Result> { + let codec = Json::default(); + let origin_uri = EventOriginUri { + scheme: "tremor-cluster-kv".to_string(), + host: hostname(), + port: None, + path: vec![], + }; + let sink = KvSink { + alias: ctx.alias().clone(), + app_ctx: ctx.app_ctx().clone(), + raft: ctx.raft().clone(), + tx: self.tx.clone(), + codec, + origin_uri, + source_is_connected: self.source_is_connected.clone(), + }; + Ok(Some(builder.spawn(sink, ctx))) + } + + fn codec_requirements(&self) -> CodecReq { + CodecReq::Structured + } +} + +struct KvSink { + alias: alias::Connector, + app_ctx: AppContext, + raft: raft::Cluster, + tx: Sender, + codec: Json, + origin_uri: EventOriginUri, + source_is_connected: Arc, +} + +impl KvSink { + async fn decode(&mut self, mut v: Option>, ingest_ns: u64) -> Result> { + if let Some(v) = v.as_mut() { + let data: &mut [u8] = v.as_mut_slice(); + // TODO: We could optimize this + Ok(self + .codec + .decode(data, ingest_ns, Value::const_null()) + .await? + .unwrap_or_default() + .0 + .into_static()) + } else { + Ok(Value::null()) + } + } + async fn encode<'v>(&mut self, v: &Value<'v>) -> Result> { + Ok(self.codec.encode(v, &Value::const_null()).await?) + } + async fn execute<'a>( + &mut self, + cmd: Command, + op_name: &'static str, + value: &Value<'a>, + _ingest_ns: u64, + ) -> Result, Value<'static>)>> { + match cmd { + Command::Get { key, strict } => { + let key_parts = vec![ + self.app_ctx.id().to_string(), + self.app_ctx.instance().to_string(), + self.alias.alias().to_string(), + key.clone(), + ]; + let combined_key = key_parts.join("."); + + let value = if strict { + self.raft.kv_get(combined_key).await? + } else { + self.raft.kv_get_local(combined_key).await? + } + .map_or(Value::const_null(), Value::from); + Ok(oks(op_name, key, value)) + } + Command::Put { key } => { + // return the new value + let value_vec = self.encode(value).await?; + let key_parts = vec![ + self.app_ctx.id().to_string(), + self.app_ctx.instance().to_string(), + self.alias.alias().to_string(), + key.clone(), + ]; + let combined_key = key_parts.join("."); + self.raft.kv_set(combined_key, value_vec).await?; + Ok(oks(op_name, key, value.clone_static())) + } // Command::Swap { key } => { + // // return the old value + // let value = self.encode(value)?; + // let v = self.db.insert(&key, value)?; + // self.decode(v, ingest_ns) + // .map(|old_value| oks(op_name, key, old_value)) + // } + // Command::Delete { key } => self + // .decode(self.db.remove(&key)?, ingest_ns) + // .map(|v| oks(op_name, key, v)), + // Command::Cas { key, old } => { + // let vec = self.encode(value)?; + // let old = old.map(|v| self.encode(v)).transpose()?; + // if let Err(CompareAndSwapError { current, proposed }) = + // self.db.compare_and_swap(&key, old, Some(vec))? + // { + // Err(format!( + // "CAS error: expected {} but found {}.", + // self.decode(proposed, ingest_ns)?, + // self.decode(current, ingest_ns)?, + // ) + // .into()) + // } else { + // Ok(oks(op_name, key, Value::null())) + // } + // } + // Command::Scan { start, end } => { + // let i = match end { + // None => self.db.range(start..), + // Some(end) => self.db.range(start..end), + // }; + // let mut res = Vec::with_capacity(i.size_hint().0); + // for e in i { + // let (key, e) = e?; + // let key: &[u8] = &key; + + // res.push(ok(op_name, key.to_vec(), self.decode(Some(e), ingest_ns)?)); + // } + // Ok(res) + // } + } + } +} + +#[async_trait::async_trait] +impl Sink for KvSink { + async fn on_event( + &mut self, + _input: &str, + event: Event, + ctx: &SinkContext, + _serializer: &mut EventSerializer, + _start: u64, + ) -> Result { + let ingest_ns = tremor_common::time::nanotime(); + let send_replies = self.source_is_connected.load(Ordering::Acquire); + + let mut r = SinkReply::ACK; + for (v, m) in event.value_meta_iter() { + let correlation = m.get("correlation"); + let executed = match Command::try_from(m) { + Ok(cmd) => { + let name = cmd.op_name(); + let key = cmd.key().to_string(); + self.execute(cmd, name, v, ingest_ns) + .await + .map_err(|e| (Some(name), Some(key), e)) + } + Err(e) => { + error!("{ctx} Invalid KV command: {e}"); + Err((None, None, e)) + } + }; + match executed { + Ok(res) => { + if send_replies { + for (data, mut meta) in res { + if let Some(correlation) = correlation { + meta.try_insert("correlation", correlation.clone_static()); + } + let reply = SourceReply::Structured { + origin_uri: self.origin_uri.clone(), + payload: (data, meta).into(), + stream: DEFAULT_STREAM_ID, + port: Some(OUT), + }; + if let Err(e) = self.tx.send(reply).await { + error!("{ctx}, Failed to send to source: {e}"); + }; + } + } + } + Err((op, key, e)) => { + error!("{ctx} Error: {e}"); + if send_replies { + // send ERR response and log err + let mut meta = literal!({ + "error": e.to_string(), + "kv": op.map(|op| literal!({ "op": op, "key": key })) + }); + if let Some(correlation) = correlation { + meta.try_insert("correlation", correlation.clone_static()); + } + let reply = SourceReply::Structured { + origin_uri: self.origin_uri.clone(), + payload: ((), meta).into(), + stream: DEFAULT_STREAM_ID, + port: Some(ERR), + }; + ctx.swallow_err(self.tx.send(reply).await, "Failed to send to source"); + } + + r = SinkReply::FAIL; + } + } + } + Ok(r) + } + + fn auto_ack(&self) -> bool { + false + } +} diff --git a/src/connectors/impls/crononome.rs b/src/connectors/impls/crononome.rs index d8987fd517..e468bd35e1 100644 --- a/src/connectors/impls/crononome.rs +++ b/src/connectors/impls/crononome.rs @@ -115,7 +115,7 @@ mod handler; -use crate::{connectors::prelude::*, errors::err_connector_def, system::KillSwitch}; +use crate::connectors::prelude::*; use handler::{ChronomicQueue, CronEntryInt}; use serde_yaml::Value as YamlValue; use tremor_common::time::nanotime; @@ -144,7 +144,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, raw: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let raw = Config::new(raw)?; let payload = if let Some(yaml_entries) = raw.entries { diff --git a/src/connectors/impls/crononome/handler.rs b/src/connectors/impls/crononome/handler.rs index ad3051d9d6..7e5688c078 100644 --- a/src/connectors/impls/crononome/handler.rs +++ b/src/connectors/impls/crononome/handler.rs @@ -151,7 +151,6 @@ impl TemporalPriorityQueue { self.q.push(Reverse(at)); } - #[cfg(not(feature = "tarpaulin-exclude"))] #[cfg(test)] pub(crate) fn pop(&mut self) -> Option> { let Reverse(x) = self.q.peek()?; @@ -176,7 +175,6 @@ impl TemporalPriorityQueue { self.q.pop().map(|Reverse(x)| x) } - #[cfg(not(feature = "tarpaulin-exclude"))] #[cfg(test)] pub(crate) fn drain(&mut self) -> Vec> { let now = Utc::now().timestamp(); @@ -228,7 +226,6 @@ impl ChronomicQueue { } } - #[cfg(not(feature = "tarpaulin-exclude"))] #[cfg(test)] pub(crate) fn drain(&mut self) -> Vec<(String, Option>)> { let due = self.tpq.drain(); @@ -240,7 +237,6 @@ impl ChronomicQueue { } trigger } - #[cfg(not(feature = "tarpaulin-exclude"))] #[cfg(test)] pub(crate) fn next(&mut self) -> Option<(String, Option>)> { self.tpq.pop().map(|ti| { @@ -260,11 +256,8 @@ impl ChronomicQueue { #[cfg(test)] mod tests { use super::*; - use std::convert::TryFrom; - - #[cfg(not(feature = "tarpaulin-exclude"))] use chrono::DateTime; - #[cfg(not(feature = "tarpaulin-exclude"))] + use std::convert::TryFrom; use std::time::Duration; #[test] @@ -287,7 +280,6 @@ mod tests { // This tight requirements for timing are extremely problematic in tests // and lead to frequent issues with the test being flaky or unrelaibale - #[cfg(not(feature = "tarpaulin-exclude"))] #[tokio::test(flavor = "multi_thread")] async fn test_tpq_fill_drain() -> Result<()> { use chrono::prelude::Utc; @@ -322,7 +314,6 @@ mod tests { // This tight requirements for timing are extremely problematic in tests // and lead to frequent issues with the test being flaky or unrelaibale - #[cfg(not(feature = "tarpaulin-exclude"))] #[tokio::test(flavor = "multi_thread")] async fn test_tpq_fill_pop() -> Result<()> { use chrono::prelude::Utc; @@ -358,7 +349,6 @@ mod tests { // This tight requirements for timing are extremely problematic in tests // and lead to frequent issues with the test being flaky or unrelaibale - #[cfg(not(feature = "tarpaulin-exclude"))] #[tokio::test(flavor = "multi_thread")] async fn test_cq_fill_drain_refill() -> Result<()> { let mut cq = ChronomicQueue::default(); @@ -393,7 +383,6 @@ mod tests { // This tight requirements for timing are extremely problematic in tests // and lead to frequent issues with the test being flaky or unrelaibale - #[cfg(not(feature = "tarpaulin-exclude"))] #[tokio::test(flavor = "multi_thread")] async fn test_cq_fill_pop_refill() -> Result<()> { let mut cq = ChronomicQueue::default(); diff --git a/src/connectors/impls/discord.rs b/src/connectors/impls/discord.rs index 3c49afbf16..46c6126852 100644 --- a/src/connectors/impls/discord.rs +++ b/src/connectors/impls/discord.rs @@ -41,10 +41,7 @@ mod handler; mod utils; use crate::channel::{bounded, Receiver, Sender}; -use crate::{ - connectors::{prelude::*, spawn_task}, - system::KillSwitch, -}; +use crate::connectors::prelude::*; use handler::Handler; use serenity::prelude::*; use tokio::task::JoinHandle; @@ -73,7 +70,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config: Config = Config::new(config)?; diff --git a/src/connectors/impls/dns/client.rs b/src/connectors/impls/dns/client.rs index c70ca95a46..59b697dbcc 100644 --- a/src/connectors/impls/dns/client.rs +++ b/src/connectors/impls/dns/client.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::channel::{bounded, Receiver, Sender}; -use crate::{connectors::prelude::*, system::KillSwitch}; +use crate::connectors::prelude::*; use std::sync::Arc; use std::{boxed::Box, sync::atomic::AtomicBool}; use trust_dns_resolver::{ @@ -35,7 +35,6 @@ impl ConnectorBuilder for Builder { &self, _id: &alias::Connector, _raw_config: &ConnectorConfig, - _kill_switch: &KillSwitch, ) -> Result> { let (tx, rx) = bounded(128); Ok(Box::new(Client { diff --git a/src/connectors/impls/elastic.rs b/src/connectors/impls/elastic.rs index cab364d2a2..5d3aa69cc7 100644 --- a/src/connectors/impls/elastic.rs +++ b/src/connectors/impls/elastic.rs @@ -230,7 +230,6 @@ //! [optimistic concurrency control]: https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html#bulk-optimistic-concurrency-control use super::http::auth::Auth; -use crate::system::KillSwitch; use crate::{ connectors::{ impls::http::utils::Header, prelude::*, sink::concurrency_cap::ConcurrencyCap, @@ -313,7 +312,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, raw_config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(raw_config)?; if config.nodes.is_empty() { @@ -1174,9 +1172,10 @@ impl CertValidation { } #[cfg(test)] mod tests { + use elasticsearch::http::request::Body; + use super::*; use crate::config::Connector as ConnectorConfig; - use elasticsearch::http::request::Body; #[tokio::test(flavor = "multi_thread")] async fn connector_builder_empty_nodes() -> Result<()> { @@ -1185,17 +1184,14 @@ mod tests { "nodes": [] } }); - let alias = alias::Connector::new("flow", "my_elastic"); + let alias = alias::Connector::new("my_elastic"); let builder = super::Builder::default(); let connector_config = ConnectorConfig::from_config(&alias, builder.connector_type(), &config)?; - let kill_switch = KillSwitch::dummy(); assert_eq!( - String::from( - "Invalid Definition for connector \"flow::my_elastic\": empty nodes provided" - ), + String::from("Invalid Definition for connector \"my_elastic\": empty nodes provided"), builder - .build(&alias, &connector_config, &kill_switch) + .build(&alias, &connector_config,) .await .err() .map(|e| e.to_string()) @@ -1214,15 +1210,14 @@ mod tests { ] } }); - let alias = alias::Connector::new("snot", "my_elastic"); + let alias = alias::Connector::new("my_elastic"); let builder = super::Builder::default(); let connector_config = ConnectorConfig::from_config(&alias, builder.connector_type(), &config)?; - let kill_switch = KillSwitch::dummy(); assert_eq!( String::from("empty host"), builder - .build(&alias, &connector_config, &kill_switch) + .build(&alias, &connector_config,) .await .err() .map(|e| e.to_string()) diff --git a/src/connectors/impls/exit.rs b/src/connectors/impls/exit.rs index e5ece68776..b28e273946 100644 --- a/src/connectors/impls/exit.rs +++ b/src/connectors/impls/exit.rs @@ -129,21 +129,20 @@ impl ConnectorBuilder for Builder { &self, _id: &alias::Connector, config: &ConnectorConfig, - kill_switch: &KillSwitch, ) -> Result> { let config = match config.config.as_ref() { Some(raw_config) => Config::new(raw_config)?, None => Config::default(), }; - Ok(Box::new(Exit { - kill_switch: kill_switch.clone(), - config, - done: false, - })) + Ok(Box::new(Exit { config })) } } #[derive(Clone)] pub(crate) struct Exit { + config: Config, +} + +pub(crate) struct ExitSink { kill_switch: KillSwitch, config: Config, done: bool, @@ -156,9 +155,14 @@ impl Connector for Exit { ctx: SinkContext, builder: SinkManagerBuilder, ) -> Result> { - let sink = self.clone(); - - Ok(Some(builder.spawn(sink, ctx))) + Ok(Some(builder.spawn( + ExitSink { + config: self.config.clone(), + done: false, + kill_switch: ctx.killswitch(), + }, + ctx, + ))) } fn codec_requirements(&self) -> CodecReq { @@ -166,13 +170,13 @@ impl Connector for Exit { } } -impl Exit { +impl ExitSink { const DELAY: &'static str = "delay"; const GRACEFUL: &'static str = "graceful"; } #[async_trait::async_trait()] -impl Sink for Exit { +impl Sink for ExitSink { fn auto_ack(&self) -> bool { true } diff --git a/src/connectors/impls/file.rs b/src/connectors/impls/file.rs index c8d6690296..b1920483c6 100644 --- a/src/connectors/impls/file.rs +++ b/src/connectors/impls/file.rs @@ -220,7 +220,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; Ok(Box::new(File { config })) diff --git a/src/connectors/impls/gbq/writer.rs b/src/connectors/impls/gbq/writer.rs index 6f7b08b1f3..6b97e538c6 100644 --- a/src/connectors/impls/gbq/writer.rs +++ b/src/connectors/impls/gbq/writer.rs @@ -32,8 +32,8 @@ pub(crate) struct Config { } impl tremor_config::Impl for Config {} +/// 10MB fn default_request_size_limit() -> usize { - // 10MB 10 * 1024 * 1024 } @@ -75,7 +75,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; Ok(Box::new(Gbq { config })) @@ -85,12 +84,11 @@ impl ConnectorBuilder for Builder { #[cfg(test)] mod tests { use tokio::sync::broadcast; - use tremor_common::ids::SinkId; use super::*; - use crate::connectors::reconnect::ConnectionLostNotifier; use crate::connectors::sink::builder; - use crate::connectors::{metrics::SinkReporter, utils::quiescence::QuiescenceBeacon}; + use crate::connectors::{reconnect::ConnectionLostNotifier, utils::metrics::SinkReporter}; + use crate::{connectors::utils::quiescence::QuiescenceBeacon, system::flow::AppContext}; #[tokio::test(flavor = "multi_thread")] pub async fn can_spawn_sink() -> Result<()> { @@ -103,22 +101,25 @@ mod tests { token: TokenSrc::dummy(), }, }; - + let app_ctx = AppContext::default(); let sink_address = connector .create_sink( SinkContext::new( - SinkId::default(), - alias::Connector::new("a", "b"), + SinkUId::default(), + alias::Connector::new("b"), ConnectorType::default(), QuiescenceBeacon::default(), ConnectionLostNotifier::new(crate::channel::bounded(128).0), + app_ctx.clone(), + KillSwitch::dummy(), ), builder( &ConnectorConfig::default(), CodecReq::Structured, - &alias::Connector::new("a", "b"), + &alias::Connector::new("b"), SinkReporter::new( - alias::Connector::new("a", "b"), + app_ctx, + alias::Connector::new("b"), broadcast::channel(1).0, None, ), diff --git a/src/connectors/impls/gbq/writer/sink.rs b/src/connectors/impls/gbq/writer/sink.rs index e88a168610..19379083ba 100644 --- a/src/connectors/impls/gbq/writer/sink.rs +++ b/src/connectors/impls/gbq/writer/sink.rs @@ -29,6 +29,7 @@ use googapis::google::cloud::bigquery::storage::v1::{ }; use prost::encoding::WireType; use prost_types::{field_descriptor_proto, DescriptorProto, FieldDescriptorProto}; +use simd_json_derive::Serialize; use std::collections::hash_map::Entry; use std::marker::PhantomData; use std::{collections::HashMap, time::Duration}; @@ -93,6 +94,7 @@ fn map_field( raw_fields: &Vec, ctx: &SinkContext, ) -> (DescriptorProto, HashMap) { + // see: https://cloud.google.com/bigquery/docs/write-api#data_type_conversions // The capacity for nested_types isn't known here, as it depends on the number of fields that have the struct type let mut nested_types = vec![]; let mut proto_fields = Vec::with_capacity(raw_fields.len()); @@ -147,7 +149,7 @@ fn map_field( } TableType::Unspecified => { - warn!("{} Found a field of unspecified type: {}", ctx, raw_field.name); + warn!("{ctx} Found a field of unspecified type: {}", raw_field.name); continue; } }; @@ -202,20 +204,17 @@ fn encode_field(val: &Value, field: &Field, result: &mut Vec) -> Result<()> match field.table_type { TableType::Double => prost::encoding::double::encode( tag, - &val.as_f64() - .ok_or_else(|| ErrorKind::BigQueryTypeMismatch("f64", val.value_type()))?, + &val.try_as_f64()?, result, ), TableType::Int64 => prost::encoding::int64::encode( tag, - &val.as_i64() - .ok_or_else(|| ErrorKind::BigQueryTypeMismatch("i64", val.value_type()))?, + &val.try_as_i64()?, result, ), TableType::Bool => prost::encoding::bool::encode( tag, - &val.as_bool() - .ok_or_else(|| ErrorKind::BigQueryTypeMismatch("bool", val.value_type()))?, + &val.try_as_bool()?, result, ), TableType::String @@ -229,8 +228,7 @@ fn encode_field(val: &Value, field: &Field, result: &mut Vec) -> Result<()> | TableType::Geography => { prost::encoding::string::encode( tag, - &val.as_str() - .ok_or_else(|| ErrorKind::BigQueryTypeMismatch("string", val.value_type()))? + &val.try_as_str()? .to_string(), result, ); @@ -238,8 +236,7 @@ fn encode_field(val: &Value, field: &Field, result: &mut Vec) -> Result<()> TableType::Struct => { let mut struct_buf: Vec = vec![]; for (k, v) in val - .as_object() - .ok_or_else(|| ErrorKind::BigQueryTypeMismatch("object", val.value_type()))? + .try_as_object()? { let subfield_description = field.subfields.get(&k.to_string()); @@ -259,16 +256,16 @@ fn encode_field(val: &Value, field: &Field, result: &mut Vec) -> Result<()> TableType::Bytes => { prost::encoding::bytes::encode( tag, - &Vec::from( - val.as_bytes().ok_or_else(|| { - ErrorKind::BigQueryTypeMismatch("bytes", val.value_type()) - })?, - ), + &val.try_as_bytes()?.to_vec(), result, ); } TableType::Json => { - warn!("Found a field of type JSON, this is not supported, ignoring."); + prost::encoding::string::encode( + tag, + &val.json_string()?, + result, + ); } TableType::Interval => { warn!("Found a field of type Interval, this is not supported, ignoring."); @@ -295,7 +292,6 @@ impl JsonToProtobufMapping { pub fn map(&self, value: &Value) -> Result> { if let Some(obj) = value.as_object() { let mut result = Vec::with_capacity(obj.len()); - for (key, val) in obj { let k: &str = key; if let Some(field) = self.fields.get(k) { @@ -311,6 +307,7 @@ impl JsonToProtobufMapping { &self.descriptor } } + impl, TChannelError: GbqChannelError> GbqSink { @@ -629,9 +626,7 @@ mod test { use crate::connectors::{ google::{tests::TestTokenProvider, TokenSrc}, impls::gbq, - reconnect::ConnectionLostNotifier, tests::ConnectorHarness, - utils::quiescence::QuiescenceBeacon, }; use bytes::Bytes; use futures::future::Ready; @@ -643,14 +638,14 @@ mod test { }; use http::{HeaderMap, HeaderValue}; use prost::Message; - use std::collections::VecDeque; - use std::fmt::{Display, Formatter}; - use std::sync::{Arc, RwLock}; - use std::task::Poll; + use std::{ + collections::VecDeque, + fmt::{Display, Formatter}, + sync::{Arc, RwLock}, + task::Poll, + }; use tonic::body::BoxBody; use tonic::codegen::Service; - use tremor_common::ids::SinkId; - use value_trait::StaticNode; struct HardcodedChannelFactory { channel: Channel, @@ -738,8 +733,6 @@ mod test { #[test] fn skips_unknown_field_types() { - let (rx, _tx) = crate::channel::bounded(1024); - let result = map_field( "name", &vec![TableFieldSchema { @@ -752,13 +745,7 @@ mod test { precision: 0, scale: 0, }], - &SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ), + &SinkContext::dummy("gbq_writer"), ); assert_eq!(result.0.field.len(), 0); @@ -767,8 +754,6 @@ mod test { #[test] fn skips_fields_of_unspecified_type() { - let (rx, _tx) = bounded(1024); - let result = map_field( "name", &vec![TableFieldSchema { @@ -781,13 +766,7 @@ mod test { precision: 0, scale: 0, }], - &SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ), + &SinkContext::dummy("gbq_writer"), ); assert_eq!(result.0.field.len(), 0); @@ -805,8 +784,6 @@ mod test { ]; for item in data { - let (rx, _tx) = bounded(1024); - let result = map_field( "name", &vec![TableFieldSchema { @@ -819,13 +796,7 @@ mod test { precision: 0, scale: 0, }], - &SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ), + &SinkContext::dummy("gbq_writer"), ); assert_eq!(result.1.len(), 1); @@ -836,8 +807,6 @@ mod test { #[test] fn can_map_a_struct() { - let (rx, _tx) = bounded(1024); - let result = map_field( "name", &vec![TableFieldSchema { @@ -859,13 +828,7 @@ mod test { precision: 0, scale: 0, }], - &SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ), + &SinkContext::dummy("gbq_writer"), ); assert_eq!(result.1.len(), 1); @@ -885,7 +848,7 @@ mod test { fn encode_fails_on_type_mismatch() { let data = [ ( - Value::String("asdf".into()), + Value::from("asdf"), Field { table_type: TableType::Int64, tag: 1, @@ -893,7 +856,7 @@ mod test { }, ), ( - Value::Static(StaticNode::F64(1.243)), + Value::from(1.243), Field { table_type: TableType::String, tag: 2, @@ -930,7 +893,7 @@ mod test { let mut result = vec![]; assert!( encode_field( - &Value::String("I".into()), + &Value::from("I"), &Field { table_type: item, tag: 123, @@ -984,7 +947,7 @@ mod test { #[test] pub fn can_encode_a_double() { - let value = Value::Static(StaticNode::F64(1.2345)); + let value = Value::from(1.2345); let field = Field { table_type: TableType::Double, tag: 2, @@ -1002,7 +965,7 @@ mod test { #[test] pub fn can_encode_boolean() { - let value = Value::Static(StaticNode::Bool(false)); + let value = Value::from(false); let field = Field { table_type: TableType::Bool, tag: 43, @@ -1041,14 +1004,13 @@ mod test { let mut result = Vec::new(); assert!(encode_field(&value, &field, &mut result).is_ok()); - // json is currently not supported, so we expect the field to be skipped - assert_eq!([] as [u8; 0], result[..]); + assert_eq!([10, 2, 123, 125] as [u8; 4], result[..]); } #[test] pub fn can_encode_interval() { - let value = Value::String("".into()); + let value = Value::from(""); let field = Field { table_type: TableType::Interval, tag: 1, @@ -1064,7 +1026,7 @@ mod test { #[test] pub fn can_skips_unspecified() { - let value = Value::String("".into()); + let value = Value::from(""); let field = Field { table_type: TableType::Unspecified, tag: 1, @@ -1080,15 +1042,7 @@ mod test { #[test] pub fn mapping_generates_a_correct_descriptor() { - let (rx, _tx) = bounded(1024); - - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ); + let ctx = SinkContext::dummy("gbq_writer"); let mapping = JsonToProtobufMapping::new( &vec![ TableFieldSchema { @@ -1129,15 +1083,7 @@ mod test { #[test] pub fn can_map_json_to_protobuf() -> Result<()> { - let (rx, _tx) = bounded(1024); - - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ); + let ctx = SinkContext::dummy("gbq_writer"); let mapping = JsonToProtobufMapping::new( &vec![ TableFieldSchema { @@ -1174,15 +1120,7 @@ mod test { #[test] fn map_field_ignores_fields_that_are_not_in_definition() -> Result<()> { - let (rx, _tx) = bounded(1024); - - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ); + let ctx = SinkContext::dummy("gbq_writer"); let mapping = JsonToProtobufMapping::new( &vec![ TableFieldSchema { @@ -1220,15 +1158,7 @@ mod test { #[test] fn map_field_ignores_struct_fields_that_are_not_in_definition() -> Result<()> { - let (rx, _tx) = bounded(1024); - - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ); + let ctx = SinkContext::dummy("gbq_writer"); let mapping = JsonToProtobufMapping::new( &vec![TableFieldSchema { name: "a".to_string(), @@ -1264,15 +1194,7 @@ mod test { #[test] fn fails_on_bytes_type_mismatch() { - let (rx, _tx) = bounded(1024); - - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ); + let ctx = SinkContext::dummy("gbq_writer"); let mapping = JsonToProtobufMapping::new( &vec![TableFieldSchema { name: "a".to_string(), @@ -1290,7 +1212,7 @@ mod test { fields.try_insert("a", 12); let result = mapping.map(&fields); - if let Err(Error(ErrorKind::BigQueryTypeMismatch("bytes", x), _)) = result { + if let Err(Error(ErrorKind::TypeError(ValueType::Custom("bytes"), x), _)) = result { assert_eq!(x, ValueType::I64); } else { panic!("Bytes conversion did not fail on type mismatch"); @@ -1299,15 +1221,7 @@ mod test { #[test] fn fails_if_the_event_is_not_an_object() { - let (rx, _tx) = bounded(1024); - - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ); + let ctx = SinkContext::dummy("gbq_writer"); let mapping = JsonToProtobufMapping::new( &vec![TableFieldSchema { name: "a".to_string(), @@ -1321,9 +1235,9 @@ mod test { }], &ctx, ); - let result = mapping.map(&Value::Static(StaticNode::I64(123))); + let result = mapping.map(&Value::from(123_i64)); - if let Err(Error(ErrorKind::BigQueryTypeMismatch("object", x), _)) = result { + if let Err(Error(ErrorKind::TypeError(ValueType::Object, x), _)) = result { assert_eq!(x, ValueType::I64); } else { panic!("Mapping did not fail on non-object event"); @@ -1347,7 +1261,6 @@ mod test { #[tokio::test(flavor = "multi_thread")] async fn on_event_fails_if_client_is_not_conected() -> Result<()> { - let (rx, _tx) = bounded(1024); let config = Config::new(&literal!({ "token": {"file": file!().to_string()}, "table_id": "doesnotmatter", @@ -1362,20 +1275,8 @@ mod test { .on_event( "", Event::signal_tick(), - &SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ), - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &alias::Connector::new("flow", "connector"), - )?, + &SinkContext::dummy("gbq_writer"), + &mut EventSerializer::dummy(None)?, 0, ) .await; @@ -1386,7 +1287,6 @@ mod test { #[tokio::test(flavor = "multi_thread")] async fn on_event_fails_if_write_stream_is_not_conected() -> Result<()> { - let (rx, _tx) = bounded(1024); let config = Config::new(&literal!({ "token": {"file": file!().to_string()}, "table_id": "doesnotmatter", @@ -1405,20 +1305,8 @@ mod test { .on_event( "", Event::signal_tick(), - &SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ), - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &alias::Connector::new("flow", "connector"), - )?, + &SinkContext::dummy("gbq_writer"), + &mut EventSerializer::dummy(None)?, 0, ) .await; @@ -1491,13 +1379,7 @@ mod test { }), ); - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(crate::channel::bounded(1024).0), - ); + let ctx = SinkContext::dummy("gbq_writer"); sink.connect(&ctx, &Attempt::default()).await?; @@ -1510,19 +1392,7 @@ mod test { event.transactional = true; let result = sink - .on_event( - "", - event, - &ctx, - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &alias::Connector::new("flow", "connector"), - )?, - 0, - ) + .on_event("", event, &ctx, &mut EventSerializer::dummy(None)?, 0) .await?; assert_eq!(result.ack, SinkAck::Fail); assert_eq!(result.cb, CbAction::None); @@ -1581,30 +1451,20 @@ mod test { }), ); - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("flow", "connector"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(crate::channel::bounded(1024).0), - ); + let ctx = SinkContext::dummy("gbq_writer"); sink.connect(&ctx, &Attempt::default()).await?; let value = literal!([ { - "data": { - "value": { - "a": "a".repeat(15*1024) - }, - "meta": {} - } + "data": { + "value": {"a": "a".repeat(15*1024)}, + "meta": {} + } }, { "data": { - "value": { - "a": "b".repeat(15*1024) - }, + "value": {"a": "b".repeat(15*1024)}, "meta": {} } } @@ -1621,13 +1481,7 @@ mod test { ..Default::default() }, &ctx, - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &alias::Connector::new("flow", "connector"), - )?, + &mut EventSerializer::dummy(None)?, 0, ) .await?; diff --git a/src/connectors/impls/gcl/writer.rs b/src/connectors/impls/gcl/writer.rs index ea560ba293..936ecf7814 100644 --- a/src/connectors/impls/gcl/writer.rs +++ b/src/connectors/impls/gcl/writer.rs @@ -270,7 +270,6 @@ impl ConnectorBuilder for Builder { _id: &alias::Connector, _: &ConnectorConfig, raw: &Value, - _kill_switch: &KillSwitch, ) -> Result> { Ok(Box::new(Gcl { config: Config::new(raw)?, diff --git a/src/connectors/impls/gcl/writer/sink.rs b/src/connectors/impls/gcl/writer/sink.rs index 7b1213612d..0cf62f066c 100644 --- a/src/connectors/impls/gcl/writer/sink.rs +++ b/src/connectors/impls/gcl/writer/sink.rs @@ -243,30 +243,23 @@ where mod test { #![allow(clippy::cast_possible_wrap)] use super::*; - use crate::connectors::impls::gcl; - use crate::connectors::tests::ConnectorHarness; - use crate::connectors::ConnectionLostNotifier; use crate::connectors::{ - google::tests::TestTokenProvider, utils::quiescence::QuiescenceBeacon, + google::tests::TestTokenProvider, google::TokenSrc, impls::gcl, tests::ConnectorHarness, }; - use crate::{channel::bounded, connectors::google::TokenSrc}; use bytes::Bytes; use futures::future::Ready; - use googapis::google::logging::r#type::LogSeverity; - use googapis::google::logging::v2::WriteLogEntriesResponse; + use googapis::google::logging::{r#type::LogSeverity, v2::WriteLogEntriesResponse}; use http::{HeaderMap, HeaderValue}; use http_body::Body; use prost::Message; - use std::task::Poll; use std::{ collections::HashMap, fmt::{Debug, Display, Formatter}, + task::Poll, }; use tonic::body::BoxBody; use tonic::codegen::Service; - use tremor_common::ids::SinkId; - use tremor_pipeline::CbAction::Trigger; - use tremor_pipeline::EventId; + use tremor_pipeline::{CbAction::Trigger, EventId}; use tremor_value::{literal, structurize}; #[derive(Debug)] @@ -342,7 +335,6 @@ mod test { #[tokio::test(flavor = "multi_thread")] async fn on_event_can_send_an_event() -> Result<()> { let (tx, mut rx) = crate::channel::unbounded(); - let (connection_lost_tx, _connection_lost_rx) = bounded(10); let mut sink = GclSink::::new( Config { @@ -360,15 +352,9 @@ mod test { tx, MockChannelFactory, ); - let ctx = SinkContext::new( - SinkId::default(), - alias::Connector::new("a", "b"), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(connection_lost_tx), - ); + let sink_context = SinkContext::dummy("gcl_writer"); - sink.connect(&ctx, &Attempt::default()).await?; + sink.connect(&sink_context, &Attempt::default()).await?; let event = Event { id: EventId::new(1, 2, 3, 4), @@ -385,14 +371,8 @@ mod test { sink.on_event( "", event.clone(), - &ctx, - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &"a".into(), - &alias::Connector::new("a", "b"), - )?, + &sink_context, + &mut EventSerializer::dummy(None)?, 0, ) .await?; @@ -443,7 +423,6 @@ mod test { #[tokio::test(flavor = "multi_thread")] async fn on_event_fails_if_client_is_not_connected() -> Result<()> { - let (rx, _tx) = bounded(1024); let (reply_tx, _reply_rx) = crate::channel::unbounded(); let config = Config::new(&literal!({ "token": {"file": file!().to_string()}, @@ -456,20 +435,8 @@ mod test { .on_event( "", Event::signal_tick(), - &SinkContext::new( - SinkId::default(), - alias::Connector::new("", ""), - ConnectorType::default(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(rx), - ), - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &alias::Connector::new("", ""), - )?, + &SinkContext::dummy("gcl_writer"), + &mut EventSerializer::dummy(None)?, 0, ) .await; diff --git a/src/connectors/impls/gcs/streamer.rs b/src/connectors/impls/gcs/streamer.rs index 7088227aa2..24d0441f12 100644 --- a/src/connectors/impls/gcs/streamer.rs +++ b/src/connectors/impls/gcs/streamer.rs @@ -29,7 +29,6 @@ use crate::{ }, }, errors::err_gcs, - system::KillSwitch, }; use std::time::Duration; use tremor_common::time::nanotime; @@ -120,7 +119,6 @@ impl ConnectorBuilder for Builder { alias: &alias::Connector, _config: &ConnectorConfig, connector_config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let mut config = Config::new(connector_config)?; config.normalize(alias); @@ -407,17 +405,13 @@ pub(crate) mod tests { config::Reconnect, connectors::{ impls::gcs::{resumable_upload_client::ResumableUploadClient, streamer::Mode}, - reconnect::ConnectionLostNotifier, - utils::{ - object_storage::{BufferPart, ObjectId}, - quiescence::QuiescenceBeacon, - }, + utils::object_storage::{BufferPart, ObjectId}, }, - errors::err_gcs, + system::flow::AppContext, }; use halfbrown::HashMap; use std::sync::atomic::AtomicUsize; - use tremor_common::ids::{ConnectorIdGen, SinkId}; + use tremor_common::uids::ConnectorUIdGen; use tremor_pipeline::EventId; use tremor_value::literal; @@ -529,7 +523,7 @@ pub(crate) mod tests { }); let mut config = Config::new(&raw_config).expect("config should be valid"); - let alias = alias::Connector::new("flow", "conn"); + let alias = alias::Connector::new("conn"); config.normalize(&alias); assert_eq!(256 * 1024, config.buffer_size); } @@ -568,23 +562,8 @@ pub(crate) mod tests { ChunkedBuffer, > = YoloSink::new(sink_impl); - let (connection_lost_tx, _) = bounded(10); - - let alias = alias::Connector::new("a", "b"); - let context = SinkContext::new( - SinkId::default(), - alias.clone(), - "gcs_streamer".into(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(connection_lost_tx), - ); - let mut serializer = EventSerializer::new( - Some(tremor_codec::Config::from("json")), - CodecReq::Required, - vec![], - &"gcs_streamer".into(), - &alias, - )?; + let context = SinkContext::dummy("gcs_streamer"); + let mut serializer = EventSerializer::dummy(Some(CodecConfig::from("json")))?; // simulate sink lifecycle sink.on_start(&context).await?; @@ -754,23 +733,8 @@ pub(crate) mod tests { ChunkedBuffer, > = YoloSink::new(sink_impl); - let (connection_lost_tx, _) = bounded(10); - - let alias = alias::Connector::new("a", "b"); - let context = SinkContext::new( - SinkId::default(), - alias.clone(), - "gcs_streamer".into(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(connection_lost_tx), - ); - let mut serializer = EventSerializer::new( - Some(tremor_codec::Config::from("json")), - CodecReq::Required, - vec![], - &"gcs_streamer".into(), - &alias, - )?; + let context = SinkContext::dummy("gcs_streamer"); + let mut serializer = EventSerializer::dummy(Some(CodecConfig::from("json")))?; // simulate sink lifecycle sink.on_start(&context).await?; @@ -928,23 +892,8 @@ pub(crate) mod tests { ChunkedBuffer, > = YoloSink::new(sink_impl); - let (connection_lost_tx, _) = bounded(10); - - let alias = alias::Connector::new("a", "b"); - let context = SinkContext::new( - SinkId::default(), - alias.clone(), - "gcs_streamer".into(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(connection_lost_tx), - ); - let mut serializer = EventSerializer::new( - Some(tremor_codec::Config::from("json")), - CodecReq::Required, - vec![], - &"gcs_streamer".into(), - &alias, - )?; + let context = SinkContext::dummy("gcs_streamer"); + let mut serializer = EventSerializer::dummy(Some(CodecConfig::from("json")))?; // simulate sink lifecycle sink.on_start(&context).await?; @@ -1023,16 +972,7 @@ pub(crate) mod tests { ChunkedBuffer, > = YoloSink::new(sink_impl); - let (connection_lost_tx, _) = bounded(10); - - let alias = alias::Connector::new("a", "b"); - let context = SinkContext::new( - SinkId::default(), - alias.clone(), - "gcs_streamer".into(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(connection_lost_tx), - ); + let context = SinkContext::dummy("gcs_streamer"); // simulate sink lifecycle sink.on_start(&context).await?; @@ -1077,23 +1017,8 @@ pub(crate) mod tests { ChunkedBuffer, > = ConsistentSink::new(sink_impl); - let (connection_lost_tx, _) = bounded(10); - - let alias = alias::Connector::new("a", "b"); - let context = SinkContext::new( - SinkId::default(), - alias.clone(), - "gcs_streamer".into(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(connection_lost_tx), - ); - let mut serializer = EventSerializer::new( - Some(tremor_codec::Config::from("json")), - CodecReq::Required, - vec![], - &"gcs_streamer".into(), - &alias, - )?; + let context = SinkContext::dummy("gcs_streamer"); + let mut serializer = EventSerializer::dummy(Some(CodecConfig::from("json")))?; // simulate standard sink lifecycle sink.on_start(&context).await?; @@ -1302,23 +1227,8 @@ pub(crate) mod tests { ChunkedBuffer, > = ConsistentSink::new(sink_impl); - let (connection_lost_tx, _) = bounded(10); - - let alias = alias::Connector::new("a", "b"); - let context = SinkContext::new( - SinkId::default(), - alias.clone(), - "gcs_streamer".into(), - QuiescenceBeacon::default(), - ConnectionLostNotifier::new(connection_lost_tx), - ); - let mut serializer = EventSerializer::new( - Some(tremor_codec::Config::from("json")), - CodecReq::Required, - vec![], - &"gcs_streamer".into(), - &alias, - )?; + let context = SinkContext::dummy("gcs_streamer"); + let mut serializer = EventSerializer::dummy(Some(CodecConfig::from("json")))?; // simulate standard sink lifecycle sink.on_start(&context).await?; @@ -1453,13 +1363,19 @@ pub(crate) mod tests { metrics_interval_s: None, }; let kill_switch = KillSwitch::dummy(); - let alias = alias::Connector::new("snot", "badger"); - let mut connector_id_gen = ConnectorIdGen::default(); + let alias = alias::Connector::new("badger"); + let mut connector_id_gen = ConnectorUIdGen::default(); // lets cover create-sink here - let addr = - crate::connectors::spawn(&alias, &mut connector_id_gen, &builder, cfg, &kill_switch) - .await?; + let addr = crate::connectors::spawn( + &alias, + &mut connector_id_gen, + &builder, + cfg, + &kill_switch, + AppContext::default(), + ) + .await?; let (tx, mut rx) = bounded(1); addr.stop(tx).await?; assert!(rx.recv().await.expect("rx empty").res.is_ok()); @@ -1484,13 +1400,19 @@ pub(crate) mod tests { metrics_interval_s: None, }; let kill_switch = KillSwitch::dummy(); - let alias = alias::Connector::new("snot", "badger"); - let mut connector_id_gen = ConnectorIdGen::default(); + let alias = alias::Connector::new("badger"); + let mut connector_id_gen = ConnectorUIdGen::default(); // lets cover create-sink here - let addr = - crate::connectors::spawn(&alias, &mut connector_id_gen, &builder, cfg, &kill_switch) - .await?; + let addr = crate::connectors::spawn( + &alias, + &mut connector_id_gen, + &builder, + cfg, + &kill_switch, + AppContext::default(), + ) + .await?; let (tx, mut rx) = bounded(1); addr.stop(tx).await?; assert!(rx.recv().await.expect("rx empty").res.is_ok()); diff --git a/src/connectors/impls/gpubsub/consumer.rs b/src/connectors/impls/gpubsub/consumer.rs index d586b8328d..6723f67034 100644 --- a/src/connectors/impls/gpubsub/consumer.rs +++ b/src/connectors/impls/gpubsub/consumer.rs @@ -93,7 +93,6 @@ impl ConnectorBuilder for Builder { alias: &alias::Connector, _: &ConnectorConfig, raw: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(raw)?; let url = Url::::parse(config.url.as_str())?; diff --git a/src/connectors/impls/gpubsub/producer.rs b/src/connectors/impls/gpubsub/producer.rs index ecaf206089..66856584c3 100644 --- a/src/connectors/impls/gpubsub/producer.rs +++ b/src/connectors/impls/gpubsub/producer.rs @@ -64,7 +64,6 @@ impl ConnectorBuilder for Builder { _alias: &alias::Connector, _config: &ConnectorConfig, raw_config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(raw_config)?; diff --git a/src/connectors/impls/http/auth.rs b/src/connectors/impls/http/auth.rs index 9b6052deb2..cd7fd1dee8 100644 --- a/src/connectors/impls/http/auth.rs +++ b/src/connectors/impls/http/auth.rs @@ -14,7 +14,7 @@ use crate::errors::Result; use std::io::Write; -use tremor_common::base64::{Engine, BASE64}; +use tremor_common::base64::{encode, BASE64}; /// Authorization methods #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] @@ -44,14 +44,14 @@ impl Auth { ref username, ref password, } => { - let encoded = BASE64.encode(format!("{username}:{password}")); + let encoded = encode(format!("{username}:{password}")); Ok(Some(format!("Basic {}", &encoded))) } Auth::Bearer(token) => Ok(Some(format!("Bearer {}", &token))), Auth::ElasticsearchApiKey { id, api_key } => { let mut header_value = "ApiKey ".to_string(); let mut writer = - base64::write::EncoderStringWriter::from_consumer(&mut header_value, &BASE64); + ::base64::write::EncoderStringWriter::from_consumer(&mut header_value, &BASE64); write!(writer, "{id}:")?; write!(writer, "{api_key}")?; writer.into_inner(); // release the reference, so header-value is accessible again diff --git a/src/connectors/impls/http/client.rs b/src/connectors/impls/http/client.rs index d128cd66df..189ecfbd7f 100644 --- a/src/connectors/impls/http/client.rs +++ b/src/connectors/impls/http/client.rs @@ -114,7 +114,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _connector_config: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; diff --git a/src/connectors/impls/http/meta.rs b/src/connectors/impls/http/meta.rs index 12200709bf..3645dce0d5 100644 --- a/src/connectors/impls/http/meta.rs +++ b/src/connectors/impls/http/meta.rs @@ -300,6 +300,7 @@ pub(super) fn extract_response_meta(response: &Response) -> Result Result<()> { @@ -310,13 +311,7 @@ mod test { "cake": ["black forst", "cheese"], "pie": "key lime" }}); - let mut s = EventSerializer::new( - None, - CodecReq::Optional("json"), - vec![], - &ConnectorType("http".into()), - &alias::Connector::new("flow", "http"), - )?; + let mut s = EventSerializer::dummy(Some(tremor_codec::Config::from("json")))?; let config = client::Config::new(&c)?; let mut b = HttpRequestBuilder::new(request_id, meta, &codec_map, &config)?; diff --git a/src/connectors/impls/http/server.rs b/src/connectors/impls/http/server.rs index 6a56b9fce7..ad86ce5b38 100644 --- a/src/connectors/impls/http/server.rs +++ b/src/connectors/impls/http/server.rs @@ -44,7 +44,7 @@ use std::{ sync::Arc, }; use tokio::{sync::oneshot, task::JoinHandle}; -use tremor_common::ids::Id; +use tremor_common::uids::UId; #[derive(Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] @@ -81,7 +81,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _raw_config: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; let tls_server_config = config.tls.clone(); diff --git a/src/connectors/impls/kafka.rs b/src/connectors/impls/kafka.rs index 03551fdb5c..bb605a2542 100644 --- a/src/connectors/impls/kafka.rs +++ b/src/connectors/impls/kafka.rs @@ -288,7 +288,7 @@ pub(crate) mod consumer; pub(crate) mod producer; -use crate::connectors::prelude::*; +use crate::connectors::{prelude::*, utils::metrics::make_metrics_payload}; use beef::Cow; use rdkafka::{error::KafkaError, ClientContext, Statistics}; use rdkafka_sys::RDKafkaErrorCode; @@ -636,7 +636,7 @@ mod tests { &literal!({ "measurement": "kafka_consumer_stats", "tags": { - "connector": "fake::fake" + "connector": "fake" }, "fields": { "rx_msgs": 42, diff --git a/src/connectors/impls/kafka/consumer.rs b/src/connectors/impls/kafka/consumer.rs index e8f9a589ba..36adc743be 100644 --- a/src/connectors/impls/kafka/consumer.rs +++ b/src/connectors/impls/kafka/consumer.rs @@ -265,7 +265,6 @@ impl ConnectorBuilder for Builder { alias: &alias::Connector, config: &ConnectorConfig, raw_config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let metrics_interval_s = config.metrics_interval_s; let config = Config::new(raw_config)?; diff --git a/src/connectors/impls/kafka/producer.rs b/src/connectors/impls/kafka/producer.rs index 425432778d..ee995c6e64 100644 --- a/src/connectors/impls/kafka/producer.rs +++ b/src/connectors/impls/kafka/producer.rs @@ -82,7 +82,6 @@ impl ConnectorBuilder for Builder { alias: &alias::Connector, config: &ConnectorConfig, raw_config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let metrics_interval_s = config.metrics_interval_s; let config = Config::new(raw_config)?; diff --git a/src/connectors/impls/kv.rs b/src/connectors/impls/kv.rs index 3b038e671c..4ba07b71f2 100644 --- a/src/connectors/impls/kv.rs +++ b/src/connectors/impls/kv.rs @@ -288,7 +288,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config: Config = Config::new(config)?; if !PathBuf::from(&config.path).is_dir() { diff --git a/src/connectors/impls/metrics.rs b/src/connectors/impls/metrics.rs index c6c6d78e87..3effc532c9 100644 --- a/src/connectors/impls/metrics.rs +++ b/src/connectors/impls/metrics.rs @@ -152,7 +152,7 @@ use crate::connectors::prelude::*; use beef::Cow; use tokio::sync::broadcast::{error::RecvError, Receiver, Sender}; -use tremor_pipeline::{MetricsMsg, METRICS_CHANNEL}; +use tremor_pipeline::{MetricsChannel, MetricsMsg, MetricsSender}; use tremor_script::utils::hostname; const MEASUREMENT: Cow<'static, str> = Cow::const_str("measurement"); @@ -170,13 +170,13 @@ const TIMESTAMP: Cow<'static, str> = Cow::const_str("timestamp"); /// There should be only one instance around all the time, identified by `tremor://localhost/connector/system::metrics/system` /// pub(crate) struct MetricsConnector { - tx: Sender, + tx: MetricsSender, } impl MetricsConnector { pub(crate) fn new() -> Self { Self { - tx: METRICS_CHANNEL.tx(), + tx: MetricsChannel::new(128).tx(), } } } @@ -194,7 +194,6 @@ impl ConnectorBuilder for Builder { &self, _id: &alias::Connector, _config: &ConnectorConfig, - _kill_switch: &KillSwitch, ) -> Result> { Ok(Box::new(MetricsConnector::new())) } @@ -202,7 +201,8 @@ impl ConnectorBuilder for Builder { #[async_trait::async_trait()] impl Connector for MetricsConnector { - async fn connect(&mut self, _ctx: &ConnectorContext, _attempt: &Attempt) -> Result { + async fn connect(&mut self, ctx: &ConnectorContext, _attempt: &Attempt) -> Result { + self.tx = ctx.app_ctx.metrics.tx(); Ok(true) } diff --git a/src/connectors/impls/metronome.rs b/src/connectors/impls/metronome.rs index 70d8b6155f..e2868e98f3 100644 --- a/src/connectors/impls/metronome.rs +++ b/src/connectors/impls/metronome.rs @@ -127,7 +127,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, raw: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(raw)?; let origin_uri = EventOriginUri { @@ -228,7 +227,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn missing_config() -> Result<()> { - let alias = alias::Connector::new("flow", "connector"); + let alias = alias::Connector::new("connector"); let builder = super::Builder::default(); let connector_config = super::ConnectorConfig { connector_type: builder.connector_type(), @@ -239,12 +238,8 @@ mod tests { reconnect: Reconnect::None, metrics_interval_s: Some(5), }; - let kill_switch = KillSwitch::dummy(); assert!(matches!( - builder - .build(&alias, &connector_config, &kill_switch) - .await - .err(), + builder.build(&alias, &connector_config,).await.err(), Some(Error(ErrorKind::MissingConfiguration(_), _)) )); Ok(()) diff --git a/src/connectors/impls/null.rs b/src/connectors/impls/null.rs index 8ff56da4c9..6953c04302 100644 --- a/src/connectors/impls/null.rs +++ b/src/connectors/impls/null.rs @@ -100,7 +100,6 @@ impl ConnectorBuilder for Builder { &self, _alias: &alias::Connector, _config: &ConnectorConfig, - _kill_switch: &KillSwitch, ) -> Result> { Ok(Box::new(Null {})) } diff --git a/src/connectors/impls/oneshot.rs b/src/connectors/impls/oneshot.rs new file mode 100644 index 0000000000..c54eb4a282 --- /dev/null +++ b/src/connectors/impls/oneshot.rs @@ -0,0 +1,98 @@ +// Copyright 2021, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::connectors::prelude::*; +use tremor_value::Value; + +#[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +struct Config { + value: simd_json::OwnedValue, +} +impl tremor_config::Impl for Config {} + +#[derive(Debug, Default)] +pub(crate) struct Builder {} + +#[async_trait::async_trait] +impl ConnectorBuilder for Builder { + fn connector_type(&self) -> ConnectorType { + "oneshot".into() + } + async fn build_cfg( + &self, + _: &alias::Connector, + _: &ConnectorConfig, + config: &Value, + ) -> Result> { + let config = Config::new(config)?; + let value = tremor_value::to_value(&config.value)?; + Ok(Box::new(Oneshot { value: Some(value) })) + } +} + +struct Oneshot { + value: Option>, +} + +#[async_trait::async_trait] +impl Connector for Oneshot { + async fn create_source( + &mut self, + source_context: SourceContext, + builder: SourceManagerBuilder, + ) -> Result> { + let source = OneshotSource { + value: self.value.take(), + }; + Ok(Some(builder.spawn(source, source_context))) + } + + fn codec_requirements(&self) -> CodecReq { + CodecReq::Structured + } +} + +struct OneshotSource { + value: Option>, +} + +#[async_trait::async_trait] +impl Source for OneshotSource { + async fn pull_data(&mut self, _pull_id: &mut u64, _ctx: &SourceContext) -> Result { + if let Some(value) = self.value.take() { + Ok(SourceReply::Structured { + origin_uri: EventOriginUri { + scheme: "tremor-oneshot".to_string(), + host: "localhost".to_string(), + port: None, + path: vec![], + }, + payload: (value, Value::object()).into(), + stream: DEFAULT_STREAM_ID, + port: None, + }) + } else { + Ok(SourceReply::Finished) + } + } + + fn is_transactional(&self) -> bool { + false + } + + fn asynchronous(&self) -> bool { + false + } +} diff --git a/src/connectors/impls/otel/client.rs b/src/connectors/impls/otel/client.rs index 1039adb60e..b4b0ab3187 100644 --- a/src/connectors/impls/otel/client.rs +++ b/src/connectors/impls/otel/client.rs @@ -71,7 +71,6 @@ impl ConnectorBuilder for Builder { _id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; let origin_uri = EventOriginUri { @@ -212,7 +211,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn otel_client_builder() -> Result<()> { - let alias = alias::Connector::new("flow", "my_otel_client"); + let alias = alias::Connector::new("my_otel_client"); let with_processors = literal!({ "config": { "url": "localhost:4317", @@ -225,8 +224,7 @@ mod tests { )?; let builder = super::Builder::default(); - let kill_switch = KillSwitch::dummy(); - let _connector = builder.build(&alias, &config, &kill_switch).await?; + let _connector = builder.build(&alias, &config).await?; Ok(()) } diff --git a/src/connectors/impls/otel/server.rs b/src/connectors/impls/otel/server.rs index 4732ba7770..39dff27e40 100644 --- a/src/connectors/impls/otel/server.rs +++ b/src/connectors/impls/otel/server.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::net::ToSocketAddrs; + use super::{common::OtelDefaults, logs, metrics, trace}; use crate::{connectors::prelude::*, errors::already_created_error}; use async_std::channel::{bounded, Receiver, Sender}; @@ -71,7 +73,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let origin_uri = EventOriginUri { scheme: "tremor-otel-server".to_string(), @@ -129,7 +130,10 @@ impl Connector for Server { .url .port() .ok_or("Missing prot for otel server")?; - let endpoint = format!("{host}:{port}").parse()?; + let endpoint = format!("{host}:{port}") + .to_socket_addrs()? + .next() + .ok_or("badaddr")?; if let Some(previous_handle) = self.accept_task.take() { previous_handle.abort(); @@ -195,12 +199,10 @@ impl Source for OtelSource { #[cfg(test)] mod tests { use super::*; - // use env_logger; - // use http_types::Method; #[tokio::test(flavor = "multi_thread")] async fn otel_client_builder() -> Result<()> { - let alias = alias::Connector::new("test", "my_otel_server"); + let alias = alias::Connector::new("my_otel_server"); let with_processors = literal!({ "config": { "url": "localhost:4317", @@ -211,11 +213,10 @@ mod tests { ConnectorType("otel_server".into()), &with_processors, )?; - let alias = alias::Connector::new("flow", "my_otel_server"); + let alias = alias::Connector::new("my_otel_server"); let builder = super::Builder::default(); - let kill_switch = KillSwitch::dummy(); - let _connector = builder.build(&alias, &config, &kill_switch).await?; + let _connector = builder.build(&alias, &config).await?; Ok(()) } diff --git a/src/connectors/impls/s3/reader.rs b/src/connectors/impls/s3/reader.rs index 42d9038fcb..1c55de4d7a 100644 --- a/src/connectors/impls/s3/reader.rs +++ b/src/connectors/impls/s3/reader.rs @@ -98,7 +98,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; diff --git a/src/connectors/impls/s3/streamer.rs b/src/connectors/impls/s3/streamer.rs index 2739b26370..14978f92f8 100644 --- a/src/connectors/impls/s3/streamer.rs +++ b/src/connectors/impls/s3/streamer.rs @@ -90,7 +90,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let mut config = Config::new(config)?; config.normalize(id); diff --git a/src/connectors/impls/stdio.rs b/src/connectors/impls/stdio.rs index 78753baa82..a330059385 100644 --- a/src/connectors/impls/stdio.rs +++ b/src/connectors/impls/stdio.rs @@ -98,7 +98,6 @@ impl ConnectorBuilder for Builder { &self, _id: &alias::Connector, _raw_config: &ConnectorConfig, - _kill_switch: &KillSwitch, ) -> Result> { Ok(Box::new(StdStreamConnector {})) } diff --git a/src/connectors/impls/tcp/client.rs b/src/connectors/impls/tcp/client.rs index 7eaf5a3694..fb27205e45 100644 --- a/src/connectors/impls/tcp/client.rs +++ b/src/connectors/impls/tcp/client.rs @@ -81,7 +81,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; if config.url.port().is_none() { diff --git a/src/connectors/impls/tcp/server.rs b/src/connectors/impls/tcp/server.rs index bc50289042..944f0312f6 100644 --- a/src/connectors/impls/tcp/server.rs +++ b/src/connectors/impls/tcp/server.rs @@ -77,7 +77,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> crate::errors::Result> { let config = Config::new(config)?; if config.url.port().is_none() { diff --git a/src/connectors/impls/udp/client.rs b/src/connectors/impls/udp/client.rs index e3a86dd6db..5b95afbbfa 100644 --- a/src/connectors/impls/udp/client.rs +++ b/src/connectors/impls/udp/client.rs @@ -51,7 +51,6 @@ impl ConnectorBuilder for Builder { _id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config: Config = Config::new(config)?; if config.url.port().is_none() { diff --git a/src/connectors/impls/udp/server.rs b/src/connectors/impls/udp/server.rs index e58bbfae7e..011d789bf8 100644 --- a/src/connectors/impls/udp/server.rs +++ b/src/connectors/impls/udp/server.rs @@ -54,7 +54,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, raw: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(raw)?; Ok(Box::new(UdpServer { config })) diff --git a/src/connectors/impls/unix_socket/client.rs b/src/connectors/impls/unix_socket/client.rs index f9ebc4eb07..4b505a2ea1 100644 --- a/src/connectors/impls/unix_socket/client.rs +++ b/src/connectors/impls/unix_socket/client.rs @@ -49,7 +49,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, conf: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(conf)?; let (source_tx, source_rx) = bounded(qsize()); diff --git a/src/connectors/impls/unix_socket/server.rs b/src/connectors/impls/unix_socket/server.rs index c37150e503..1f1cba55d0 100644 --- a/src/connectors/impls/unix_socket/server.rs +++ b/src/connectors/impls/unix_socket/server.rs @@ -68,7 +68,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; let (sink_tx, sink_rx) = bounded(qsize()); diff --git a/src/connectors/impls/wal.rs b/src/connectors/impls/wal.rs index 52d6270925..fb677deb00 100644 --- a/src/connectors/impls/wal.rs +++ b/src/connectors/impls/wal.rs @@ -72,7 +72,6 @@ impl ConnectorBuilder for Builder { _: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config: Config = Config::new(config)?; diff --git a/src/connectors/impls/ws/client.rs b/src/connectors/impls/ws/client.rs index adad81f916..4ab569d2f6 100644 --- a/src/connectors/impls/ws/client.rs +++ b/src/connectors/impls/ws/client.rs @@ -67,7 +67,6 @@ impl ConnectorBuilder for Builder { id: &alias::Connector, _: &ConnectorConfig, config: &Value, - _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; let host = config diff --git a/src/connectors/impls/ws/server.rs b/src/connectors/impls/ws/server.rs index 99bf5c6cca..aadd9dd3ce 100644 --- a/src/connectors/impls/ws/server.rs +++ b/src/connectors/impls/ws/server.rs @@ -72,7 +72,6 @@ impl ConnectorBuilder for Builder { _id: &alias::Connector, _: &ConnectorConfig, raw_config: &Value, - _kill_switch: &KillSwitch, ) -> crate::errors::Result> { let config = Config::new(raw_config)?; diff --git a/src/connectors/prelude.rs b/src/connectors/prelude.rs index 1ffa9a62ae..dc32dca4ba 100644 --- a/src/connectors/prelude.rs +++ b/src/connectors/prelude.rs @@ -15,7 +15,6 @@ pub(crate) use crate::{ channel::{bounded, Receiver, Sender}, connectors::{ - metrics::make_metrics_payload, sink::{ channel_sink::{ChannelSink, ChannelSinkRuntime}, AsyncSinkReply, ContraflowData, EventSerializer, ReplySender, Sink, SinkAck, SinkAddr, @@ -37,9 +36,12 @@ pub(crate) use crate::{ Event, }; pub(crate) use std::sync::atomic::Ordering; -pub use tremor_common::alias; + pub(crate) use tremor_common::{ + alias, ports::{Port, ERR, IN, OUT}, + time::nanotime, + uids::{SinkUId, SourceUId}, url::{Defaults, HttpsDefaults, Url}, }; pub(crate) use tremor_config::Impl; diff --git a/src/connectors/sink.rs b/src/connectors/sink.rs index 95ac4cb8d4..b7d90e67bb 100644 --- a/src/connectors/sink.rs +++ b/src/connectors/sink.rs @@ -19,13 +19,15 @@ pub(crate) mod channel_sink; /// Utility for limiting concurrency (by sending `CB::Close` messages when a maximum concurrency value is reached) pub(crate) mod concurrency_cap; +use super::{utils::metrics::SinkReporter, ConnectionLostNotifier, Msg, QuiescenceBeacon}; use crate::{ channel::{bounded, unbounded, Receiver, Sender, UnboundedReceiver, UnboundedSender}, config::Connector as ConnectorConfig, connectors::prelude::*, pipeline, primerge::PriorityMerge, - qsize, + qsize, raft, + system::flow::AppContext, }; use futures::StreamExt; // for .next() on PriorityMerge use std::{ @@ -37,11 +39,7 @@ use std::{ use tokio::task; use tokio_stream::wrappers::{ReceiverStream, UnboundedReceiverStream}; use tremor_codec::{self as codec, Codec}; -use tremor_common::{ - ids::{SinkId, SourceId}, - ports::Port, - time::nanotime, -}; +use tremor_common::time::nanotime; use tremor_interceptor::postprocessor::{ self, finish, make_postprocessors, postprocess, Postprocessors, }; @@ -49,8 +47,6 @@ use tremor_pipeline::{CbAction, Event, EventId, OpMeta, SignalKind, DEFAULT_STRE use tremor_script::{ast::DeployEndpoint, EventPayload}; use tremor_value::Value; -use super::{metrics::SinkReporter, ConnectionLostNotifier, Msg, QuiescenceBeacon}; - pub(crate) type ReplySender = UnboundedSender; /// Result for a sink function that may provide insights or response. @@ -255,7 +251,7 @@ pub(crate) trait SinkRuntime: Send + Sync { #[derive(Clone)] pub(crate) struct SinkContextInner { /// the connector unique identifier - pub(crate) uid: SinkId, + pub(crate) uid: SinkUId, /// the connector alias pub(crate) alias: alias::Connector, /// the connector type @@ -266,22 +262,46 @@ pub(crate) struct SinkContextInner { /// notifier the connector runtime if we lost a connection pub(crate) notifier: ConnectionLostNotifier, + + /// sender for raft requests + /// Application Context + pub(crate) app_ctx: AppContext, + + pub(crate) killswitch: KillSwitch, } #[derive(Clone)] pub(crate) struct SinkContext(Arc); impl SinkContext { - pub(crate) fn uid(&self) -> SinkId { + #[cfg(test)] + pub(crate) fn dummy(ct: &str) -> Self { + let (tx, _rx) = bounded(1024); + Self::new( + SinkUId::default(), + Alias::from(ct), + ConnectorType::from(ct), + QuiescenceBeacon::default(), + ConnectionLostNotifier::new(tx), + AppContext::default(), + KillSwitch::dummy(), + ) + } + pub(crate) fn killswitch(&self) -> KillSwitch { + self.0.killswitch.clone() + } + pub(crate) fn uid(&self) -> SinkUId { self.0.uid } pub(crate) fn notifier(&self) -> &ConnectionLostNotifier { &self.0.notifier } pub(crate) fn new( - uid: SinkId, + uid: SinkUId, alias: alias::Connector, connector_type: ConnectorType, quiescence_beacon: QuiescenceBeacon, notifier: ConnectionLostNotifier, + app_ctx: AppContext, + killswitch: KillSwitch, ) -> SinkContext { Self(Arc::new(SinkContextInner { uid, @@ -289,35 +309,43 @@ impl SinkContext { connector_type, quiescence_beacon, notifier, + app_ctx, + killswitch, })) } } +impl Display for SinkContextInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}[Sink::{}]", self.app_ctx, self.alias) + } +} + impl Display for SinkContext { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "[Sink::{}]", &self.0.alias) + self.0.fmt(f) } } impl Context for SinkContext { - // fn uid(&self) -> &SinkId { - // &self.0.uid - // } fn alias(&self) -> &alias::Connector { &self.0.alias } - fn quiescence_beacon(&self) -> &QuiescenceBeacon { &self.0.quiescence_beacon } - fn notifier(&self) -> &ConnectionLostNotifier { &self.0.notifier } - fn connector_type(&self) -> &ConnectorType { &self.0.connector_type } + fn raft(&self) -> &raft::Cluster { + &self.0.app_ctx.raft + } + fn app_ctx(&self) -> &AppContext { + &self.0.app_ctx + } } /// messages a sink can receive @@ -413,7 +441,6 @@ pub(crate) fn builder( config: &ConnectorConfig, connector_codec_requirement: CodecReq, alias: &alias::Connector, - metrics_reporter: SinkReporter, ) -> Result { // resolve codec and processors @@ -454,6 +481,21 @@ pub(crate) struct EventSerializer { } impl EventSerializer { + #[cfg(test)] + pub(crate) fn dummy(codec_config: Option) -> Result { + let default = if codec_config.is_none() { + CodecReq::Structured + } else { + CodecReq::Required + }; + Self::new( + codec_config, + default, + vec![], + &ConnectorType::from("dummy"), + &Alias::from("dummy"), + ) + } pub(crate) fn new( codec_config: Option, default_codec: CodecReq, @@ -626,9 +668,9 @@ where // pipelines connected to IN port pipelines: Vec<(DeployEndpoint, pipeline::Addr)>, // set of source ids we received start signals from - starts_received: HashSet, + starts_received: HashSet, // set of connector ids we received drain signals from - drains_received: HashSet, // TODO: use a bitset for both? + drains_received: HashSet, // TODO: use a bitset for both? drain_channel: Option>, state: SinkState, } diff --git a/src/connectors/sink/channel_sink.rs b/src/connectors/sink/channel_sink.rs index ff2f663d95..b85efde41a 100644 --- a/src/connectors/sink/channel_sink.rs +++ b/src/connectors/sink/channel_sink.rs @@ -35,7 +35,7 @@ use tokio::{ task::{self, JoinHandle}, time::timeout, }; -use tremor_common::{ids::Id, time::nanotime}; +use tremor_common::{time::nanotime, uids::UId}; use tremor_pipeline::{CbAction, Event, SignalKind}; use tremor_value::Value; diff --git a/src/connectors/source.rs b/src/connectors/source.rs index 764058f11e..097f6b4194 100644 --- a/src/connectors/source.rs +++ b/src/connectors/source.rs @@ -17,30 +17,29 @@ /// A simple source that is fed with `SourceReply` via a channel. pub mod channel_source; -use super::{CodecReq, Connectivity}; -use crate::channel::{unbounded, Sender, UnboundedReceiver, UnboundedSender}; -use crate::connectors::{ - metrics::SourceReporter, - prelude::*, - utils::reconnect::{Attempt, ConnectionLostNotifier}, - ConnectorType, Context, Msg, QuiescenceBeacon, StreamDone, -}; use crate::errors::empty_error; -use crate::errors::{Error, Result}; +use crate::errors::Result; use crate::pipeline; use crate::pipeline::InputTarget; +use crate::{ + channel::{unbounded, Sender, UnboundedReceiver, UnboundedSender}, + raft, +}; +use crate::{ + connectors::{ + prelude::*, + utils::reconnect::{Attempt, ConnectionLostNotifier}, + CodecReq, Connectivity, ConnectorType, Context, Msg, QuiescenceBeacon, StreamDone, + }, + system::flow::AppContext, +}; pub(crate) use channel_source::{ChannelSource, ChannelSourceRuntime}; use hashbrown::HashSet; use std::collections::{btree_map::Entry, BTreeMap}; use std::fmt::Display; use tokio::task; use tremor_codec::{self as codec, Codec}; -use tremor_common::{ - alias, - ids::{Id, SinkId, SourceId}, - ports::{Port, ERR, OUT}, - time::nanotime, -}; +use tremor_common::uids::UId; use tremor_config::NameWithConfig; use tremor_interceptor::preprocessor; use tremor_interceptor::preprocessor::{finish, make_preprocessors, preprocess, Preprocessors}; @@ -48,7 +47,8 @@ use tremor_pipeline::{ CbAction, Event, EventId, EventIdGenerator, EventOriginUri, DEFAULT_STREAM_ID, }; use tremor_script::{ast::DeployEndpoint, prelude::BaseExpr, EventPayload, ValueAndMeta}; -use tremor_value::{literal, Value}; + +use super::utils::metrics::SourceReporter; #[derive(Debug)] /// Messages a Source can receive @@ -274,7 +274,7 @@ pub(crate) trait StreamReader: Send { #[derive(Clone)] pub(crate) struct SourceContext { /// connector uid - pub uid: SourceId, + pub(crate) uid: SourceUId, /// connector alias pub(crate) alias: alias::Connector, @@ -285,11 +285,61 @@ pub(crate) struct SourceContext { /// tool to notify the connector when the connection is lost pub(crate) notifier: ConnectionLostNotifier, + + /// Application Context + pub(crate) app_ctx: AppContext, + + /// kill switch + pub(crate) killswitch: KillSwitch, + + /// Precomputed prefix for logging + pub(crate) prefix: alias::SourceContext, +} + +impl SourceContext { + pub(crate) fn new( + uid: SourceUId, + alias: alias::Connector, + connector_type: ConnectorType, + quiescence_beacon: QuiescenceBeacon, + notifier: ConnectionLostNotifier, + app_ctx: AppContext, + killswitch: KillSwitch, + ) -> Self { + let s = Self { + uid, + alias, + connector_type, + quiescence_beacon, + notifier, + app_ctx, + killswitch, + prefix: alias::SourceContext::new(format!("[Source::{app_ctx}::{alias}]",)), + }; + } + pub(crate) fn killswitch(&self) -> KillSwitch { + self.killswitch.clone() + } + #[cfg(test)] + pub(crate) fn dummy() -> Self { + Self::new( + SourceUId::default(), + Alias::new("test"), + ConnectorType::default(), + QuiescenceBeacon::default(), + ConnectionLostNotifier::dummy(), + AppContext::default(), + KillSwitch::dummy(), + ) + } + pub(crate) fn prefix(&self) -> &alias::SourceContext { + &self.prefix + } } impl Display for SourceContext { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "[Source::{}]", &self.alias) + f.write_str(&self.prefix) } } @@ -297,18 +347,21 @@ impl Context for SourceContext { fn alias(&self) -> &alias::Connector { &self.alias } - fn quiescence_beacon(&self) -> &QuiescenceBeacon { &self.quiescence_beacon } - fn notifier(&self) -> &ConnectionLostNotifier { &self.notifier } - fn connector_type(&self) -> &ConnectorType { &self.connector_type } + fn raft(&self) -> &raft::Cluster { + &self.app_ctx.raft + } + fn app_ctx(&self) -> &AppContext { + &self.app_ctx + } } /// address of a source @@ -381,8 +434,8 @@ impl SourceManagerBuilder { /// # Errors /// - on invalid connector configuration pub(crate) fn builder( - source_uid: SourceId, - config: &super::ConnectorConfig, + source_uid: SourceUId, + config: &ConnectorConfig, connector_default_codec: CodecReq, source_metrics_reporter: SourceReporter, ) -> Result { @@ -418,7 +471,7 @@ pub(crate) fn builder( /// maintaining stream state // TODO: there is optimization potential here for reusing codec and preprocessors after a stream got ended struct Streams { - uid: SourceId, + uid: SourceUId, codec_config: tremor_codec::Config, preprocessor_configs: Vec, states: BTreeMap, @@ -430,7 +483,7 @@ impl Streams { } /// constructor fn new( - uid: SourceId, + uid: SourceUId, codec_config: tremor_codec::Config, preprocessor_configs: Vec, ) -> Self { @@ -487,7 +540,7 @@ impl Streams { /// build a stream fn build_stream( - source_uid: SourceId, + source_uid: SourceUId, stream_id: u64, codec_config: &tremor_codec::Config, codec_overwrite: Option, @@ -558,7 +611,7 @@ where /// Gather all the sinks that reported being started. /// This will give us some knowledge on the topology and most importantly /// on how many `Drained` messages to wait. Assumes a static topology. - started_sinks: HashSet, + started_sinks: HashSet, num_started_sinks: u64, /// is counted up for each call to `pull_data` in order to identify the pull call /// an event is originating from. We can only ack or fail pulls. @@ -1017,7 +1070,7 @@ where let mut ingest_ns = nanotime(); if let Some(mut stream_state) = self.streams.end_stream(stream_id) { let results = build_last_events( - &self.ctx.alias, + &self.ctx, &mut stream_state, &mut ingest_ns, pull_id, @@ -1088,7 +1141,7 @@ where if let Some(stream) = stream { let stream_state = self.streams.get_or_create_stream(stream, &self.ctx)?; let results = build_events( - &self.ctx.alias, + &self.ctx, stream_state, &mut ingest_ns, pull_id, @@ -1116,7 +1169,7 @@ where let mut stream_state = self.streams.create_anonymous_stream(codec_overwrite)?; let meta = meta.unwrap_or_else(Value::object); let mut results = build_events( - &self.ctx.alias, + &self.ctx, &mut stream_state, &mut ingest_ns, pull_id, @@ -1129,7 +1182,7 @@ where .await; // finish up the stream immediately let mut last_events = build_last_events( - &self.ctx.alias, + &self.ctx, &mut stream_state, &mut ingest_ns, pull_id, @@ -1250,7 +1303,7 @@ where /// preprocessor or codec errors are turned into events to the ERR port of the source/connector #[allow(clippy::too_many_arguments)] async fn build_events( - alias: &alias::Connector, + ctx: &SourceContext, stream_state: &mut StreamState, ingest_ns: &mut u64, pull_id: u64, @@ -1265,7 +1318,7 @@ async fn build_events( ingest_ns, data, meta.clone(), - alias, + ctx.prefix(), ) { Ok(processed) => { let mut res = Vec::with_capacity(processed.len()); @@ -1283,13 +1336,7 @@ async fn build_events( Err(None) => continue, Err(Some(e)) => ( ERR, - make_error( - alias, - &Error::from(e), - stream_state.stream_id, - pull_id, - meta, - ), + make_error(ctx, &e, stream_state.stream_id, pull_id, meta), ), }; let event = build_event( @@ -1307,13 +1354,7 @@ async fn build_events( } Err(e) => { // preprocessor error - let err_payload = make_error( - alias, - &Error::from(e), - stream_state.stream_id, - pull_id, - meta, - ); + let err_payload = make_error(ctx, &e, stream_state.stream_id, pull_id, meta); let event = build_event( stream_state, pull_id, @@ -1331,7 +1372,7 @@ async fn build_events( /// preprocessor or codec errors are turned into events to the ERR port of the source/connector #[allow(clippy::too_many_arguments)] async fn build_last_events( - alias: &alias::Connector, + ctx: &SourceContext, stream_state: &mut StreamState, ingest_ns: &mut u64, pull_id: u64, @@ -1340,7 +1381,7 @@ async fn build_last_events( meta: Value<'static>, is_transactional: bool, ) -> Vec<(Port<'static>, Event)> { - match finish(stream_state.preprocessors.as_mut_slice(), alias) { + match finish(stream_state.preprocessors.as_mut_slice(), ctx.prefix()) { Ok(processed) => { let mut res = Vec::with_capacity(processed.len()); for (chunk, meta) in processed { @@ -1356,13 +1397,7 @@ async fn build_last_events( Err(None) => continue, Err(Some(e)) => ( ERR, - make_error( - alias, - &Error::from(e), - stream_state.stream_id, - pull_id, - meta, - ), + make_error(ctx, &e, stream_state.stream_id, pull_id, meta), ), }; let event = build_event( @@ -1379,13 +1414,7 @@ async fn build_last_events( } Err(e) => { // preprocessor error - let err_payload = make_error( - alias, - &Error::from(e), - stream_state.stream_id, - pull_id, - meta, - ); + let err_payload = make_error(ctx, &e, stream_state.stream_id, pull_id, meta); let event = build_event( stream_state, pull_id, @@ -1400,9 +1429,9 @@ async fn build_last_events( } /// create an error payload -fn make_error( - connector_alias: &alias::Connector, - error: &Error, +fn make_error( + ctx: &SourceContext, + error: &E, stream_id: u64, pull_id: u64, mut meta: Value<'static>, @@ -1410,7 +1439,7 @@ fn make_error( let e_string = error.to_string(); let data = literal!({ "error": e_string.clone(), - "source": connector_alias.to_string(), + "source": ctx.to_string(), "stream_id": stream_id, "pull_id": pull_id }); diff --git a/src/connectors/tests.rs b/src/connectors/tests.rs index 860e9af060..ecc6693f36 100644 --- a/src/connectors/tests.rs +++ b/src/connectors/tests.rs @@ -60,6 +60,7 @@ use super::{prelude::KillSwitch, sink::SinkMsg}; use crate::{ channel::{bounded, unbounded, Receiver, UnboundedReceiver}, errors::empty_error, + system::flow::AppContext, }; use crate::{ config, @@ -74,8 +75,8 @@ use std::{collections::HashMap, time::Instant}; use tokio::{sync::oneshot, task, time::timeout}; use tremor_common::alias::{self}; use tremor_common::{ - ids::{ConnectorIdGen, Id, SourceId}, ports::{Port, ERR, IN, OUT}, + uids::{ConnectorUIdGen, SourceUId, UId}, }; use tremor_pipeline::{CbAction, EventId}; use tremor_script::{ast::DeployEndpoint, lexer::Location, NodeMeta}; @@ -95,8 +96,8 @@ impl ConnectorHarness { input_ports: Vec>, output_ports: Vec>, ) -> Result { - let alias = alias::Connector::new("test", alias); - let mut connector_id_gen = ConnectorIdGen::new(); + let alias = alias::Connector::new(alias); + let mut connector_id_gen = ConnectorUIdGen::new(); let mut known_connectors = HashMap::new(); for builder in builtin_connector_types() { @@ -109,6 +110,7 @@ impl ConnectorHarness { builder, raw_config, &kill_switch, + AppContext::default(), ) .await?; let mut pipes = HashMap::new(); @@ -196,7 +198,7 @@ impl ConnectorHarness { // ensure we notify the connector that its sink part is connected self.addr .send_sink(SinkMsg::Signal { - signal: Event::signal_start(SourceId::new(1)), + signal: Event::signal_start(SourceUId::new(1)), }) .await?; @@ -219,7 +221,7 @@ impl ConnectorHarness { let cr = rx.recv().await.ok_or_else(empty_error)?; debug!("Stop result received."); cr.res?; - //self.handle.cancel().await; + //self.handle.abort(); let out_events = self .pipes .get_mut(&OUT) @@ -374,12 +376,11 @@ impl TestPipeline { self.addr.send_mgmt(pipeline::MgmtMsg::Stop).await } pub(crate) fn new(alias: String) -> Self { - let flow_id = alias::Flow::new("test"); let qsize = qsize(); let (tx, rx) = bounded(qsize); let (tx_cf, rx_cf) = unbounded(); let (tx_mgmt, mut rx_mgmt) = bounded(qsize); - let pipeline_id = alias::Pipeline::new(flow_id, alias); + let pipeline_id = alias::Pipeline::new(alias); let addr = pipeline::Addr::new(tx, tx_cf, tx_mgmt, pipeline_id); task::spawn(async move { @@ -526,6 +527,7 @@ pub(crate) mod free_port { loop { if let Ok(listener) = TcpListener::bind(("127.0.0.1", candidate)).await { let port = listener.local_addr()?.port(); + assert_eq!(candidate, port); drop(listener); return Ok(port); } diff --git a/src/connectors/tests/bench.rs b/src/connectors/tests/bench.rs index 31b1f3812c..86ad57c64d 100644 --- a/src/connectors/tests/bench.rs +++ b/src/connectors/tests/bench.rs @@ -14,7 +14,7 @@ use super::ConnectorHarness; use crate::{ connectors::{impls::bench, prelude::KillSwitch, sink::SinkMsg}, errors::Result, - system::{flow_supervisor, World, WorldConfig}, + system::{flow_supervisor, Runtime, WorldConfig}, }; use std::{io::Write, time::Duration}; use tempfile::NamedTempFile; @@ -40,7 +40,7 @@ async fn stop_after_events() -> Result<()> { "iters": 2 } }); - let (world, world_handle) = World::start(WorldConfig::default()).await?; + let (world, world_handle) = Runtime::start(WorldConfig::default()).await?; let mut harness = ConnectorHarness::new_with_kill_switch( function_name!(), &bench::Builder::default(), @@ -123,7 +123,7 @@ async fn stop_after_secs() -> Result<()> { // the bench connector should trigger the kill switch let two_secs = Duration::from_secs(2); let msg = timeout(two_secs, rx.recv()).await?.expect("failed to recv"); - assert!(matches!(msg, flow_supervisor::Msg::Stop)); + assert!(matches!(msg, flow_supervisor::Msg::Terminate)); info!("Flow supervisor finished"); info!("Harness stopped"); handle.abort(); // stopping the pipeline after the connector to ensure it is draining the source diff --git a/src/connectors/tests/elastic.rs b/src/connectors/tests/elastic.rs index 8edc575c66..7bca6e90fb 100644 --- a/src/connectors/tests/elastic.rs +++ b/src/connectors/tests/elastic.rs @@ -15,8 +15,8 @@ use std::time::{Duration, Instant}; use super::{setup_for_tls, ConnectorHarness}; -use crate::connectors::impls::elastic; use crate::connectors::impls::http::auth::Auth; +use crate::connectors::{impls::elastic, tests::free_port}; use crate::errors::{Error, Result}; use elasticsearch::auth::{ClientCertificate, Credentials}; use elasticsearch::cert::{Certificate, CertificateValidation}; @@ -57,7 +57,7 @@ async fn connector_elastic() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); let docker = clients::Cli::default(); - let port = super::free_port::find_free_tcp_port().await?; + let port = free_port::find_free_tcp_port().await?; let image = RunnableImage::from( default_image() .with_env_var("xpack.security.enabled", "false") @@ -479,7 +479,7 @@ async fn elastic_routing() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); let docker = clients::Cli::default(); - let port = super::free_port::find_free_tcp_port().await?; + let port = free_port::find_free_tcp_port().await?; let image = RunnableImage::from( default_image() .with_env_var("xpack.security.enabled", "false") @@ -801,7 +801,7 @@ async fn elastic_routing() -> Result<()> { async fn auth_basic() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); let docker = clients::Cli::default(); - let port = super::free_port::find_free_tcp_port().await?; + let port = free_port::find_free_tcp_port().await?; let password = "snot"; let image = RunnableImage::from( default_image() @@ -863,7 +863,7 @@ async fn auth_basic() -> Result<()> { async fn auth_api_key() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); let docker = clients::Cli::default(); - let port = super::free_port::find_free_tcp_port().await?; + let port = free_port::find_free_tcp_port().await?; let password = "snot"; let image = RunnableImage::from( default_image() @@ -953,7 +953,7 @@ async fn auth_client_cert() -> Result<()> { }; let docker = clients::Cli::default(); - let port = super::free_port::find_free_tcp_port().await?; + let port = free_port::find_free_tcp_port().await?; let password = "snot"; let image = RunnableImage::from( default_image() @@ -1091,7 +1091,7 @@ async fn elastic_https() -> Result<()> { }; let docker = clients::Cli::default(); - let port = super::free_port::find_free_tcp_port().await?; + let port = free_port::find_free_tcp_port().await?; let password = "snot"; let image = RunnableImage::from( default_image() diff --git a/src/connectors/tests/http/client.rs b/src/connectors/tests/http/client.rs index 9a34697b1a..997194fee0 100644 --- a/src/connectors/tests/http/client.rs +++ b/src/connectors/tests/http/client.rs @@ -683,7 +683,7 @@ async fn missing_tls_config_https() -> Result<()> { .map(|e| e.to_string()) .unwrap_or_default(); - assert_eq!("Invalid Definition for connector \"test::missing_tls_config_https\": missing tls config with 'https' url. Set 'tls' to 'true' or provide a full tls config.", res); + assert_eq!("Invalid Definition for connector \"missing_tls_config_https\": missing tls config with 'https' url. Set 'tls' to 'true' or provide a full tls config.", res); Ok(()) } diff --git a/src/connectors/tests/kafka.rs b/src/connectors/tests/kafka.rs index 55f43f9fb6..b458e7d889 100644 --- a/src/connectors/tests/kafka.rs +++ b/src/connectors/tests/kafka.rs @@ -14,7 +14,7 @@ mod consumer; mod producer; -use crate::connectors::tests::free_port::find_free_tcp_port; +use super::free_port::find_free_tcp_port; use crate::errors::Result; use std::time::Duration; use testcontainers::{ diff --git a/src/connectors/tests/kafka/consumer.rs b/src/connectors/tests/kafka/consumer.rs index 682641ef83..ad90291ce7 100644 --- a/src/connectors/tests/kafka/consumer.rs +++ b/src/connectors/tests/kafka/consumer.rs @@ -16,7 +16,7 @@ use crate::{ connectors::{ impls::kafka, tests::{ - free_port, + free_port::find_free_tcp_port, kafka::{redpanda_container, PRODUCE_TIMEOUT}, ConnectorHarness, }, @@ -226,7 +226,7 @@ async fn transactional_retry() -> Result<()> { assert_eq!( &literal!({ "error": "SIMD JSON error: InternalError(TapeError) at character 0 ('}')", - "source": "test::transactional_retry", + "source": "[Source::[Node::0][default/]::transactional_retry]", "stream_id": 8_589_934_592_u64, "pull_id": 1u64 }), @@ -443,7 +443,7 @@ async fn custom_no_retry() -> Result<()> { assert_eq!( &literal!({ "error": "SIMD JSON error: InternalError(TapeError) at character 0 ('}')", - "source": "test::custom_no_retry", + "source": "[Source::[Node::0][default/]::custom_no_retry]", "stream_id": 8_589_934_592_u64, "pull_id": 1u64 }), @@ -649,7 +649,7 @@ async fn performance() -> Result<()> { assert_eq!( &literal!({ "error": "SIMD JSON error: InternalError(TapeError) at character 0 ('}')", - "source": "test::performance", + "source": "[Source::[Node::0][default/]::performance]", "stream_id": 8_589_934_592_u64, "pull_id": 1u64 }), @@ -696,7 +696,7 @@ async fn performance() -> Result<()> { #[tokio::test(flavor = "multi_thread")] #[serial(kafka)] async fn connector_kafka_consumer_unreachable() -> Result<()> { - let kafka_port = free_port::find_free_tcp_port().await?; + let kafka_port = find_free_tcp_port().await?; let _: std::result::Result<_, _> = env_logger::try_init(); let connector_config = literal!({ "reconnect": { @@ -734,7 +734,7 @@ async fn connector_kafka_consumer_unreachable() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn invalid_rdkafka_options() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); - let kafka_port = free_port::find_free_tcp_port().await?; + let kafka_port = find_free_tcp_port().await?; let broker = format!("127.0.0.1:{kafka_port}"); let topic = "tremor_test_pause_resume"; let group_id = "invalid_rdkafka_options"; diff --git a/src/connectors/tests/s3.rs b/src/connectors/tests/s3.rs index 825aaafb33..07bcdaf075 100644 --- a/src/connectors/tests/s3.rs +++ b/src/connectors/tests/s3.rs @@ -24,6 +24,7 @@ use std::time::{Duration, Instant}; use testcontainers::{clients::Cli, images::generic::GenericImage, Container, RunnableImage}; use super::free_port::find_free_tcp_port; + const IMAGE: &str = "minio/minio"; const VERSION: &str = "RELEASE.2023-01-12T02-06-16Z"; diff --git a/src/connectors/tests/s3/streamer.rs b/src/connectors/tests/s3/streamer.rs index fafecd0dd6..3e5f52ed5a 100644 --- a/src/connectors/tests/s3/streamer.rs +++ b/src/connectors/tests/s3/streamer.rs @@ -312,8 +312,9 @@ async fn get_object(client: &Client, bucket: &str, key: &str) -> Vec { async fn get_object_value(client: &Client, bucket: &str, key: &str) -> value::Value<'static> { let mut v = get_object(client, bucket, key).await; - let obj = value::parse_to_value(&mut v).expect("parse failed"); - return obj.into_static(); + value::parse_to_value(&mut v) + .expect("parse failed") + .into_static() } fn get_unbatched_event() -> (Event, value::Value<'static>) { diff --git a/src/connectors/tests/tcp/client.rs b/src/connectors/tests/tcp/client.rs index 6d869d6c86..65e94fae89 100644 --- a/src/connectors/tests/tcp/client.rs +++ b/src/connectors/tests/tcp/client.rs @@ -15,7 +15,7 @@ use crate::{ connectors::{ impls::tcp, - tests::{free_port, setup_for_tls, tcp::EchoServer, ConnectorHarness}, + tests::{free_port::find_free_tcp_port, setup_for_tls, tcp::EchoServer, ConnectorHarness}, }, errors::Result, }; @@ -40,7 +40,7 @@ async fn tcp_client() -> Result<()> { async fn tcp_client_test(use_tls: bool) -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); - let free_port = free_port::find_free_tcp_port().await?; + let free_port = find_free_tcp_port().await?; let server_addr = format!("localhost:{free_port}"); diff --git a/src/connectors/tests/wal.rs b/src/connectors/tests/wal.rs index faec88c2dc..3804c1f276 100644 --- a/src/connectors/tests/wal.rs +++ b/src/connectors/tests/wal.rs @@ -15,8 +15,8 @@ use super::ConnectorHarness; use crate::{connectors::impls::wal, errors::Result}; use std::time::Duration; use tremor_common::{ - ids::{Id, SourceId}, ports::IN, + uids::{SourceUId, UId}, }; use tremor_pipeline::{CbAction, Event, EventIdGenerator}; use tremor_value::{literal, prelude::*, Value}; @@ -40,7 +40,7 @@ async fn wal() -> Result<()> { harness.wait_for_connected().await?; harness.consume_initial_sink_contraflow().await?; - let source_id = SourceId::new(1); + let source_id = SourceUId::new(1); let mut id_gen = EventIdGenerator::new(source_id); let value = Value::from(42_u64); let meta = Value::object(); diff --git a/src/connectors/utils/metrics.rs b/src/connectors/utils/metrics.rs index 381fea0a3b..9366fe166d 100644 --- a/src/connectors/utils/metrics.rs +++ b/src/connectors/utils/metrics.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::system::flow::AppContext; use beef::Cow; use simd_json::ObjectHasher; use tremor_common::{ @@ -23,6 +24,8 @@ use tremor_pipeline::MetricsSender; use tremor_script::EventPayload; use tremor_value::prelude::*; +const APP: Cow<'static, str> = Cow::const_str("app"); +const INSTANCE: Cow<'static, str> = Cow::const_str("instance"); const FLOW: Cow<'static, str> = Cow::const_str("flow"); const CONNECTOR: Cow<'static, str> = Cow::const_str("connector"); const PORT: Cow<'static, str> = Cow::const_str("port"); @@ -30,6 +33,7 @@ const CONNECTOR_EVENTS: Cow<'static, str> = Cow::const_str("connector_events"); /// metrics reporter for connector sources pub(crate) struct SourceReporter { + app_ctx: AppContext, alias: alias::Connector, metrics_out: u64, metrics_err: u64, @@ -40,11 +44,13 @@ pub(crate) struct SourceReporter { impl SourceReporter { pub(crate) fn new( + app_ctx: AppContext, alias: alias::Connector, tx: MetricsSender, flush_interval_s: Option, ) -> Self { Self { + app_ctx, alias, metrics_out: 0, metrics_err: 0, @@ -67,10 +73,20 @@ impl SourceReporter { pub(crate) fn periodic_flush(&mut self, timestamp: u64) -> Option { if let Some(interval) = self.flush_interval_ns { if timestamp >= self.last_flush_ns + interval { - let payload_out = - make_event_count_metrics_payload(timestamp, OUT, self.metrics_out, &self.alias); - let payload_err = - make_event_count_metrics_payload(timestamp, ERR, self.metrics_err, &self.alias); + let payload_out = make_event_count_metrics_payload( + timestamp, + OUT, + self.metrics_out, + &self.app_ctx, + &self.alias, + ); + let payload_err = make_event_count_metrics_payload( + timestamp, + ERR, + self.metrics_err, + &self.app_ctx, + &self.alias, + ); send(&self.tx, payload_out, &self.alias); send(&self.tx, payload_err, &self.alias); self.last_flush_ns = timestamp; @@ -90,6 +106,7 @@ impl SourceReporter { /// metrics reporter for connector sinks pub(crate) struct SinkReporter { + app_ctx: AppContext, alias: alias::Connector, metrics_in: u64, tx: MetricsSender, @@ -99,11 +116,13 @@ pub(crate) struct SinkReporter { impl SinkReporter { pub(crate) fn new( + app_ctx: AppContext, alias: alias::Connector, tx: MetricsSender, flush_interval_s: Option, ) -> Self { Self { + app_ctx, alias, metrics_in: 0, tx, @@ -119,8 +138,13 @@ impl SinkReporter { pub(crate) fn periodic_flush(&mut self, timestamp: u64) -> Option { if let Some(interval) = self.flush_interval_ns { if timestamp >= self.last_flush_ns + interval { - let payload = - make_event_count_metrics_payload(timestamp, IN, self.metrics_in, &self.alias); + let payload = make_event_count_metrics_payload( + timestamp, + IN, + self.metrics_in, + &self.app_ctx, + &self.alias, + ); send(&self.tx, payload, &self.alias); self.last_flush_ns = timestamp; return Some(timestamp); @@ -153,10 +177,13 @@ pub(crate) fn make_event_count_metrics_payload( timestamp: u64, port: Port<'static>, count: u64, + app_ctx: &AppContext, connector_id: &alias::Connector, ) -> EventPayload { - let mut tags = Object::with_capacity_and_hasher(2, ObjectHasher::default()); - tags.insert_nocheck(FLOW, Value::from(connector_id.flow_alias().to_string())); + let mut tags = Object::with_capacity_and_hasher(5, ObjectHasher::default()); + tags.insert_nocheck(APP, Value::from(app_ctx.id().to_string())); + tags.insert_nocheck(INSTANCE, Value::from(app_ctx.instance().to_string())); + tags.insert_nocheck(FLOW, Value::from(connector_id.to_string())); tags.insert_nocheck(CONNECTOR, connector_id.to_string().into()); tags.insert_nocheck(PORT, port.into()); diff --git a/src/connectors/utils/pb.rs b/src/connectors/utils/pb.rs index ecb286d5a6..f72153d5f4 100644 --- a/src/connectors/utils/pb.rs +++ b/src/connectors/utils/pb.rs @@ -18,7 +18,7 @@ use crate::connectors::prelude::*; use crate::errors::{Error, ErrorKind, Result}; use simd_json::StaticNode; use std::collections::BTreeMap; -use tremor_common::base64::{Engine, BASE64}; +use tremor_common::base64::encode; use tremor_otelapis::opentelemetry::proto::metrics::v1; pub(crate) fn maybe_string_to_pb(data: Option<&Value<'_>>) -> Result { @@ -155,7 +155,7 @@ pub(crate) fn value_to_prost_value(json: &Value) -> Result { } } Value::Bytes(v) => { - let encoded = BASE64.encode(v); + let encoded = encode(v); prost_types::Value { kind: Some(Kind::StringValue(encoded)), } diff --git a/src/connectors/utils/reconnect.rs b/src/connectors/utils/reconnect.rs index c381d48fb4..b35e0c251f 100644 --- a/src/connectors/utils/reconnect.rs +++ b/src/connectors/utils/reconnect.rs @@ -211,6 +211,11 @@ impl ConnectionLostNotifier { self.0.send(Msg::ConnectionLost).await?; Ok(()) } + #[cfg(test)] + pub(crate) fn dummy() -> Self { + let (tx, _) = bounded(128); + Self(tx) + } } impl ReconnectRuntime { @@ -388,6 +393,7 @@ mod tests { use crate::{ connectors::{utils::quiescence::QuiescenceBeacon, CodecReq}, qsize, + system::flow::AppContext, }; /// does not connect @@ -447,7 +453,7 @@ mod tests { async fn failfast_runtime() -> Result<()> { let (tx, _rx) = bounded(qsize()); let notifier = ConnectionLostNotifier::new(tx.clone()); - let alias = alias::Connector::new("flow", "test"); + let alias = alias::Connector::new("test"); let addr = Addr { alias: alias.clone(), source: None, @@ -465,6 +471,7 @@ mod tests { connector_type: "fake".into(), quiescence_beacon: qb, notifier: runtime.notifier(), + app_ctx: AppContext::default(), }; // failing attempt assert_eq!( @@ -479,7 +486,7 @@ mod tests { async fn backoff_runtime() -> Result<()> { let (tx, mut rx) = bounded(qsize()); let notifier = ConnectionLostNotifier::new(tx.clone()); - let alias = alias::Connector::new("flow", "test"); + let alias = alias::Connector::new("test"); let addr = Addr { alias: alias.clone(), source: None, @@ -502,6 +509,7 @@ mod tests { connector_type: "fake".into(), quiescence_beacon: qb, notifier: runtime.notifier(), + app_ctx: AppContext::default(), }; // 1st failing attempt assert!(matches!( diff --git a/src/errors.rs b/src/errors.rs index e05f58147e..dfc2ac7187 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -186,11 +186,13 @@ error_chain! { JsonAccessError(value_trait::AccessError); JsonError(simd_json::Error); KafkaError(rdkafka::error::KafkaError); + SerdeJsonError(serde_json::Error); MimeParsingError(mime::FromStrError); ModeParseError(file_mode::ModeParseError); OneShotRecv(tokio::sync::oneshot::error::RecvError); ParseFloatError(std::num::ParseFloatError); ParseIntError(std::num::ParseIntError); + RaftAPIError(crate::raft::api::client::Error); RegexError(regex::Error); ReqwestError(reqwest::Error); RustlsError(rustls::Error); @@ -210,9 +212,15 @@ error_chain! { WalJson(qwal::Error); Ws(tokio_tungstenite::tungstenite::Error); YamlError(serde_yaml::Error) #[doc = "Error during yaml parsing"]; + CheckIsLeaderError(openraft::error::RaftError>); + ClientWriteError(openraft::error::RaftError>); } errors { + RaftNotRunning { + description("Raft is not running") + display("Raft is not running") + } TypeError(expected: ValueType, found: ValueType) { description("Type error") display("Type error: Expected {}, found {}", expected, found) @@ -293,6 +301,10 @@ error_chain! { description("No socket available") display("No socket available. Probably not connected yet.") } + FlowFailed(flow: String) { + description("Flow entered failed state") + display("Flow {flow} entered failed state") + } DeployFlowError(flow: String, err: String) { description("Error deploying Flow") display("Error deploying Flow {}: {}", flow, err) @@ -335,6 +347,7 @@ error_chain! { display("Type in the message does not match BigQuery type. Expected: {}, actual: {:?}", expected, actual) } + NoClickHouseClientAvailable { description("The ClickHouse adapter has no client available") display("The ClickHouse adapter has no client available") diff --git a/src/instance.rs b/src/instance.rs index b5bce814cc..7362b6dae2 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -17,7 +17,7 @@ // different artefact types // -use std::fmt::Display; +use std::fmt::{Display, Formatter}; /// Possible lifecycle states of an instance #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -46,7 +46,7 @@ impl State { } impl Display for State { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_str(match self { Self::Initializing => "initialized", Self::Running => "running", @@ -57,3 +57,33 @@ impl Display for State { }) } } + +/// Representing the state an instance should be in +/// subset of `State` as those are the only states instances can be transitioned to +#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Default)] +pub enum IntendedState { + #[default] + Running, + Paused, + Stopped, +} + +impl Display for IntendedState { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Running => "running", + Self::Paused => "paused", + Self::Stopped => "stopped", + }) + } +} + +impl From for State { + fn from(intended: IntendedState) -> Self { + match intended { + IntendedState::Paused => State::Paused, + IntendedState::Running => State::Running, + IntendedState::Stopped => State::Stopped, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index f1a1eb7964..f6939cdda2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,6 @@ //! Tremor runtime #![deny(warnings)] -#![deny(missing_docs)] #![recursion_limit = "1024"] #![deny( clippy::all, @@ -65,30 +64,27 @@ pub mod version; /// Instance management pub mod instance; +pub mod raft; + /// Metrics instance name pub static mut INSTANCE: &str = "tremor"; -use std::sync::atomic::AtomicUsize; - -use crate::errors::{Error, Result}; - pub(crate) use crate::config::Connector; -use system::World; -pub use tremor_pipeline::Event; -use tremor_script::{ - deploy::Deploy, highlighter::Dumb as ToStringHighlighter, highlighter::Term as TermHighlighter, +use crate::{ + errors::{Error, Result}, + system::Runtime, }; -use tremor_script::{highlighter::Highlighter, FN_REGISTRY}; +use std::{io::Read, sync::atomic::AtomicUsize}; +pub use tremor_pipeline::Event; +use tremor_script::highlighter::Dumb as ToStringHighlighter; /// Operator Config pub type OpConfig = tremor_value::Value<'static>; pub(crate) mod channel; -lazy_static! { - /// Default Q Size - static ref QSIZE: AtomicUsize = AtomicUsize::new(128); -} +/// Default Q Size +static QSIZE: AtomicUsize = AtomicUsize::new(128); pub(crate) fn qsize() -> usize { QSIZE.load(std::sync::atomic::Ordering::Relaxed) @@ -98,8 +94,7 @@ pub(crate) fn qsize() -> usize { /// /// # Errors /// Fails if the file can not be loaded -pub async fn load_troy_file(world: &World, file_name: &str) -> Result { - use std::io::Read; +pub async fn load_troy_file(world: &Runtime, file_name: &str) -> Result { info!("Loading troy from {file_name}"); let mut file = tremor_common::file::open(&file_name)?; @@ -107,28 +102,7 @@ pub async fn load_troy_file(world: &World, file_name: &str) -> Result { file.read_to_string(&mut src) .map_err(|e| Error::from(format!("Could not open file {file_name} => {e}")))?; - let aggr_reg = tremor_script::registry::aggr(); - - let deployable = Deploy::parse(&src, &*FN_REGISTRY.read()?, &aggr_reg); - let mut h = TermHighlighter::stderr(); - let deployable = match deployable { - Ok(deployable) => { - deployable.format_warnings_with(&mut h)?; - deployable - } - Err(e) => { - log_error!(h.format_error(&e), "Error: {e}"); - - return Err(format!("failed to load troy file: {file_name}").into()); - } - }; - - let mut count = 0; - for flow in deployable.iter_flows() { - world.start_flow(flow).await?; - count += 1; - } - Ok(count) + world.load_troy(file_name, &src).await } /// Logs but ignores an error @@ -153,7 +127,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_load_troy_file() -> Result<()> { - let (world, handle) = World::start(WorldConfig::default()).await?; + let (world, handle) = Runtime::start(WorldConfig::default()).await?; let troy_file = tempfile::NamedTempFile::new()?; troy_file.as_file().write_all( r#" diff --git a/src/pipeline.rs b/src/pipeline.rs index a77f240c51..fd735f26bc 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -13,19 +13,23 @@ // limitations under the License. use crate::{ channel::{bounded, unbounded, Receiver, Sender, UnboundedReceiver, UnboundedSender}, - qsize, -}; -use crate::{ connectors::{self, sink::SinkMsg, source::SourceMsg}, errors::{pipe_send_e, Result}, instance::State, primerge::PriorityMerge, + qsize, + system::flow::AppContext, }; use futures::StreamExt; use std::{fmt, time::Duration}; use tokio::task::{self, JoinHandle}; use tokio_stream::wrappers::{ReceiverStream, UnboundedReceiverStream}; -use tremor_common::{alias, ids::OperatorIdGen, ports::Port, time::nanotime}; +use tremor_common::{ + alias::{self, Generic}, + ports::Port, + time::nanotime, + uids::OperatorUIdGen, +}; use tremor_pipeline::{ errors::ErrorKind as PipelineErrorKind, CbAction, Event, ExecutableGraph, SignalKind, }; @@ -146,12 +150,13 @@ impl TryFrom for OutputTarget { } pub(crate) fn spawn( + app_ctx: AppContext, pipeline_alias: alias::Pipeline, config: &tremor_pipeline::query::Query, - operator_id_gen: &mut OperatorIdGen, + operator_id_gen: &mut OperatorUIdGen, ) -> Result { let qsize = qsize(); - let mut pipeline = config.to_executable_graph(operator_id_gen)?; + let mut pipeline = config.to_executable_graph(operator_id_gen, &app_ctx.metrics)?; pipeline.optimize(); let (tx, rx) = bounded::>(qsize); @@ -179,6 +184,7 @@ pub(crate) fn spawn( let addr = Addr::new(tx, cf_tx, mgmt_tx, pipeline_alias.clone()); task::spawn(pipeline_task( + app_ctx, pipeline_alias, pipeline, rx, @@ -391,13 +397,13 @@ async fn send_signal(own_id: &alias::Pipeline, signal: Event, dests: &mut Dests) let first = destinations.next(); for (id, dest) in destinations { // if we are connected to ourselves we should not forward signals - if matches!(dest, OutputTarget::Sink(_)) || id.alias() != own_id.pipeline_alias() { + if matches!(dest, OutputTarget::Sink(_)) || id.alias() != own_id.alias() { dest.send_signal(signal.clone()).await?; } } if let Some((id, dest)) = first { // if we are connected to ourselves we should not forward signals - if matches!(dest, OutputTarget::Sink(_)) || id.alias() != own_id.pipeline_alias() { + if matches!(dest, OutputTarget::Sink(_)) || id.alias() != own_id.alias() { dest.send_signal(signal).await?; } } @@ -474,31 +480,30 @@ fn maybe_send(r: Result<()>) { /// /// currently only used for printing struct PipelineContext { + app_context: AppContext, alias: alias::Pipeline, } -impl std::fmt::Display for PipelineContext { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "[Pipeline::{}]", &self.alias) - } -} - -impl From<&alias::Pipeline> for PipelineContext { - fn from(alias: &alias::Pipeline) -> Self { - Self { - alias: alias.clone(), - } +impl PipelineContext { + fn new(app_context: AppContext, alias: alias::Pipeline) -> Self { + Self { app_context, alias } } } -impl From for PipelineContext { - fn from(alias: alias::Pipeline) -> Self { - Self { alias } +impl std::fmt::Display for PipelineContext { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "[Node:{}][Pipeline::{}]", + self.app_context.raft.id(), + &self.alias + ) } } #[allow(clippy::too_many_lines)] pub(crate) async fn pipeline_task( + app_ctx: AppContext, id: alias::Pipeline, mut pipeline: ExecutableGraph, rx: Receiver>, @@ -507,8 +512,8 @@ pub(crate) async fn pipeline_task( tick_handler: JoinHandle<()>, ) -> Result<()> { pipeline.id = id.to_string(); - - let ctx = PipelineContext::from(&id); + let node_id = app_ctx.raft.id(); + let ctx = PipelineContext::new(app_ctx.clone(), id.clone()); let mut dests: Dests = halfbrown::HashMap::new(); let mut inputs: Inputs = halfbrown::HashMap::new(); @@ -528,7 +533,7 @@ pub(crate) async fn pipeline_task( match msg { AnyMsg::Contraflow(msg) => handle_cf_msg(msg, &mut pipeline, &inputs), AnyMsg::Flow(Msg::Event { input, event }) => { - match pipeline.enqueue(input.clone(), event, &mut eventset) { + match pipeline.enqueue(node_id, input.clone(), event, &mut eventset) { Ok(()) => { handle_insights(&mut pipeline, &inputs); maybe_send(send_events(&mut eventset, &mut dests).await); @@ -547,7 +552,7 @@ pub(crate) async fn pipeline_task( } } AnyMsg::Flow(Msg::Signal(signal)) => { - if let Err(e) = pipeline.enqueue_signal(signal.clone(), &mut eventset) { + if let Err(e) = pipeline.enqueue_signal(node_id, signal.clone(), &mut eventset) { let err_str = if let PipelineErrorKind::Script(script_kind) = e.0 { let script_error = tremor_script::errors::Error(script_kind, e.1); Dumb::error_to_string(&script_error)? @@ -688,9 +693,9 @@ mod tests { }; use std::time::Instant; use tremor_common::{ - ids::Id as _, - ids::SourceId, ports::{IN, OUT}, + uids::SourceUId, + uids::UId as _, }; use tremor_pipeline::{EventId, OpMeta}; use tremor_script::{aggr_registry, lexer::Location, NodeMeta, FN_REGISTRY}; @@ -699,23 +704,27 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn report() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); - let mut operator_id_gen = OperatorIdGen::new(); + let mut operator_id_gen = OperatorUIdGen::new(); + let trickle = r#"select event from in into out;"#; let aggr_reg = aggr_registry(); let query = - tremor_pipeline::query::Query::parse(trickle, &*FN_REGISTRY.read()?, &aggr_reg)?; + tremor_pipeline::query::Query::parse(&trickle, &*FN_REGISTRY.read()?, &aggr_reg)?; let addr = spawn( - alias::Pipeline::new("report", "test-pipe1"), + AppContext::default(), + alias::Pipeline::new("test-pipe1"), &query, &mut operator_id_gen, )?; let addr2 = spawn( - alias::Pipeline::new("report", "test-pipe2"), + AppContext::default(), + alias::Pipeline::new("test-pipe2"), &query, &mut operator_id_gen, )?; let addr3 = spawn( - alias::Pipeline::new("report", "test-pipe3"), + AppContext::default(), + alias::Pipeline::new("test-pipe3"), &query, &mut operator_id_gen, )?; @@ -805,13 +814,18 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn pipeline_spawn() -> Result<()> { let _: std::result::Result<_, _> = env_logger::try_init(); - let mut operator_id_gen = OperatorIdGen::new(); + let mut operator_id_gen = OperatorUIdGen::new(); let trickle = r#"select event from in into out;"#; let aggr_reg = aggr_registry(); - let pipeline_id = alias::Pipeline::new("flow", "test-pipe"); + let pipeline_id = alias::Pipeline::new("test-pipe"); let query = - tremor_pipeline::query::Query::parse(trickle, &*FN_REGISTRY.read()?, &aggr_reg)?; - let addr = spawn(pipeline_id, &query, &mut operator_id_gen)?; + tremor_pipeline::query::Query::parse(&trickle, &*FN_REGISTRY.read()?, &aggr_reg)?; + let addr = spawn( + AppContext::default(), + pipeline_id, + &query, + &mut operator_id_gen, + )?; let (tx, mut rx) = bounded(1); addr.send_mgmt(MgmtMsg::Inspect(tx.clone())).await?; @@ -895,7 +909,7 @@ mod tests { } // send a signal - addr.send(Box::new(Msg::Signal(Event::signal_drain(SourceId::new( + addr.send(Box::new(Msg::Signal(Event::signal_drain(SourceUId::new( 42, ))))) .await?; diff --git a/src/preprocessor.rs b/src/preprocessor.rs deleted file mode 100644 index 0fb4327d49..0000000000 --- a/src/preprocessor.rs +++ /dev/null @@ -1,760 +0,0 @@ -// Copyright 2020-2021, The Tremor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod base64; -mod decompress; -pub(crate) mod gelf_chunking; -mod ingest_ns; -mod length_prefixed; -mod remove_empty; -pub(crate) mod separate; -mod textual_length_prefixed; -pub(crate) mod prelude { - pub use super::Preprocessor; - pub use crate::errors::Result; - pub use tremor_value::Value; - use value_trait::prelude::*; -} -use self::prelude::*; -use crate::{alias::Connector, config::Preprocessor as PreprocessorConfig, errors::Result}; - -//pub type Lines = lines::Lines; - -/// A set of preprocessors -pub type Preprocessors = Vec>; - -/// Preprocessor trait -pub trait Preprocessor: Sync + Send { - /// Canonical name for this preprocessor - fn name(&self) -> &str; - /// process data - /// - /// # Errors - /// - /// * Errors if the data can not processed - fn process( - &mut self, - ingest_ns: &mut u64, - data: &[u8], - meta: Value<'static>, - ) -> Result, Value<'static>)>>; - - /// Finish processing data and emit anything that might be left. - /// Takes a `data` buffer of input data, that is potentially empty, - /// especially if this is the first preprocessor in a chain. - /// - /// # Errors - /// - /// * if finishing fails for some reason lol - fn finish( - &mut self, - _data: Option<&[u8]>, - _meta: Option>, - ) -> Result, Value<'static>)>> { - Ok(vec![]) - } -} - -/// Lookup a preprocessor implementation via its configuration -/// -/// # Errors -/// -/// * Errors if the preprocessor is not known -pub fn lookup_with_config(config: &PreprocessorConfig) -> Result> { - match config.name.as_str() { - "separate" => Ok(Box::new(separate::Separate::from_config(&config.config)?)), - "base64" => Ok(Box::::default()), - "decompress" => Ok(Box::new(decompress::Decompress::from_config( - config.config.as_ref(), - )?)), - "remove-empty" => Ok(Box::::default()), - "gelf-chunking" => Ok(Box::::default()), - "ingest-ns" => Ok(Box::::default()), - "length-prefixed" => Ok(Box::::default()), - "textual-length-prefixed" => { - Ok(Box::::default()) - } - name => Err(format!("Preprocessor '{name}' not found.").into()), - } -} - -/// Lookup a preprocessor implementation via its unique id -/// -/// # Errors -/// -/// * if the preprocessor with `name` is not known -pub fn lookup(name: &str) -> Result> { - lookup_with_config(&PreprocessorConfig::from(name)) -} - -/// Given the slice of preprocessor names: Look them up and return them as `Preprocessors`. -/// -/// # Errors -/// -/// * If the preprocessor is not known. -pub fn make_preprocessors(preprocessors: &[PreprocessorConfig]) -> Result { - preprocessors.iter().map(lookup_with_config).collect() -} - -/// Canonical way to preprocess data before it is fed to a codec for decoding. -/// -/// Preprocessors might split up the given data in multiple chunks. Each of those -/// chunks must be seperately decoded by a `Codec`. -/// -/// # Errors -/// -/// * If a preprocessor failed -pub fn preprocess( - preprocessors: &mut [Box], - ingest_ns: &mut u64, - data: Vec, - meta: Value<'static>, - alias: &Alias, -) -> Result, Value<'static>)>> { - let mut data = vec![(data, meta)]; - let mut data1 = Vec::new(); - for pp in preprocessors { - for (i, (d, m)) in data.drain(..).enumerate() { - match pp.process(ingest_ns, &d, m) { - Ok(mut r) => data1.append(&mut r), - Err(e) => { - error!("[Connector::{alias}] Preprocessor [{i}] error: {e}"); - return Err(e); - } - } - } - std::mem::swap(&mut data, &mut data1); - } - Ok(data) -} - -/// Canonical way to finish preprocessors up -/// -/// # Errors -/// -/// * If a preprocessor failed -pub fn finish( - preprocessors: &mut [Box], - alias: &Alias, -) -> Result, Value<'static>)>> { - if let Some((head, tail)) = preprocessors.split_first_mut() { - let mut data = match head.finish(None, None) { - Ok(d) => d, - Err(e) => { - error!( - "[Connector::{alias}] Preprocessor '{}' finish error: {e}", - head.name() - ); - return Err(e); - } - }; - let mut data1 = Vec::new(); - for pp in tail { - for (d, m) in data.drain(..) { - match pp.finish(Some(&d), Some(m)) { - Ok(mut r) => data1.append(&mut r), - Err(e) => { - error!( - "[Connector::{alias}] Preprocessor '{}' finish error: {e}", - pp.name() - ); - return Err(e); - } - } - } - std::mem::swap(&mut data, &mut data1); - } - Ok(data) - } else { - Ok(vec![]) - } -} - -#[cfg(test)] -mod test { - #![allow(clippy::ignored_unit_patterns)] - use super::*; - use crate::postprocessor::{self as post, separate::Separate as SeparatePost, Postprocessor}; - use crate::Result; - - #[test] - fn ingest_ts() -> Result<()> { - let mut pre_p = ingest_ns::ExtractIngestTs {}; - let mut post_p = post::ingest_ns::IngestNs {}; - - let data = vec![1_u8, 2, 3]; - - let encoded = post_p.process(42, 23, &data)?.pop().ok_or("no data")?; - - let mut in_ns = 0u64; - let decoded = pre_p - .process(&mut in_ns, &encoded, Value::object())? - .pop() - .ok_or("no data")? - .0; - - assert!(pre_p.finish(None, None)?.is_empty()); - - assert_eq!(data, decoded); - assert_eq!(in_ns, 42); - - // data too short - assert!(pre_p.process(&mut in_ns, &[0_u8], Value::object()).is_err()); - Ok(()) - } - - fn textual_prefix(len: usize) -> String { - format!("{len} {}", String::from_utf8_lossy(&vec![b'O'; len])) - } - - use proptest::prelude::*; - - // generate multiple chopped length-prefixed strings - fn multiple_textual_lengths(max_elements: usize) -> BoxedStrategy<(Vec, Vec)> { - proptest::collection::vec(".+", 1..max_elements) // generator for Vec of arbitrary strings, maximum length of vector: `max_elements` - .prop_map(|ss| { - let s: (Vec, Vec) = ss - .into_iter() - .map(|s| (s.len(), format!("{} {s}", s.len()))) // for each string, extract the length, and create a textual length prefix - .unzip(); - s - }) - .prop_map(|tuple| (tuple.0, tuple.1.join(""))) // generator for a tuple of 1. the sizes of the length prefixed strings, 2. the concatenated length prefixed strings as one giant string - .prop_map(|tuple| { - // here we chop the big string into up to 4 bits - let mut chopped = Vec::with_capacity(4); - let mut giant_string: String = tuple.1.clone(); - while !giant_string.is_empty() && chopped.len() < 4 { - // verify we are at a char boundary - let mut indices = giant_string.char_indices(); - let num_chars = giant_string.chars().count(); - if let Some((index, _)) = indices.nth(num_chars / 2) { - let mut splitted = giant_string.split_off(index); - std::mem::swap(&mut splitted, &mut giant_string); - chopped.push(splitted); - } else { - break; - } - } - chopped.push(giant_string); - (tuple.0, chopped) - }) - .boxed() - } - - proptest! { - #[test] - fn textual_length_prefix_prop((lengths, datas) in multiple_textual_lengths(5)) { - let mut pre_p = textual_length_prefixed::TextualLengthPrefixed::default(); - let mut in_ns = 0_u64; - let res: Vec<_> = datas.into_iter().flat_map(|data| { - pre_p.process(&mut in_ns, data.as_bytes(), Value::object()).unwrap_or_default() - }).collect(); - assert_eq!(lengths.len(), res.len()); - for (processed, expected_len) in res.iter().zip(lengths) { - assert_eq!(expected_len, processed.0.len()); - } - } - - #[test] - fn textual_length_pre_post(length in 1..100_usize) { - let data = vec![1_u8; length]; - let mut pre_p = textual_length_prefixed::TextualLengthPrefixed::default(); - let mut post_p = post::textual_length_prefixed::TextualLengthPrefixed::default(); - let encoded = post_p.process(0, 0, &data).unwrap_or_default().pop().unwrap_or_default(); - let mut in_ns = 0_u64; - let mut res = pre_p.process(&mut in_ns, &encoded, Value::object()).unwrap_or_default(); - assert_eq!(1, res.len()); - let payload = res.pop().unwrap_or_default().0; - assert_eq!(length, payload.len()); - } - } - - #[test] - fn textual_prefix_length_loop() { - let datas = vec![ - "24 \'?\u{d617e}ѨR\u{202e}\u{f8f7c}\u{ede29}\u{ac784}36 ?{Â¥?MȺ\r\u{bac41}9\u{5bbbb}\r\u{1c46c}\u{4ba79}Â¥\u{7f}*?:\u{0}$i", - "60 %\u{a825a}\u{a4269}\u{39e0c}\u{b3e21}<ì\u{f6c20}ѨÛ`HW\u{9523f}V", - "\u{3}\u{605fe}%Fq\u{89b5e}\u{93780}Q3", - "Â¥?\u{feff}9", - " \'�2\u{4269b}", - ]; - let lengths: Vec = vec![24, 36, 60, 9]; - let mut pre_p = textual_length_prefixed::TextualLengthPrefixed::default(); - let mut in_ns = 0_u64; - let res: Vec<_> = datas - .into_iter() - .flat_map(|data| { - pre_p - .process(&mut in_ns, data.as_bytes(), Value::object()) - .unwrap_or_default() - }) - .collect(); - assert_eq!(lengths.len(), res.len()); - for (processed, expected_len) in res.iter().zip(lengths) { - assert_eq!(expected_len, processed.0.len()); - } - } - - #[test] - fn textual_length_prefix() { - let mut pre_p = textual_length_prefixed::TextualLengthPrefixed::default(); - let data = textual_prefix(42); - let mut in_ns = 0_u64; - let mut res = pre_p - .process(&mut in_ns, data.as_bytes(), Value::object()) - .unwrap_or_default(); - assert_eq!(1, res.len()); - let payload = res.pop().unwrap_or_default().0; - assert_eq!(42, payload.len()); - } - - #[test] - fn empty_textual_prefix() { - let data = ("").as_bytes(); - let mut pre_p = textual_length_prefixed::TextualLengthPrefixed::default(); - let mut post_p = post::textual_length_prefixed::TextualLengthPrefixed::default(); - let mut in_ns = 0_u64; - let res = pre_p - .process(&mut in_ns, data, Value::object()) - .unwrap_or_default(); - assert_eq!(0, res.len()); - - let data_empty = vec![]; - let encoded = post_p - .process(42, 23, &data_empty) - .unwrap_or_default() - .pop() - .unwrap_or_default(); - assert_eq!("0 ", String::from_utf8_lossy(&encoded)); - let mut res2 = pre_p - .process(&mut in_ns, &encoded, Value::object()) - .unwrap_or_default(); - assert_eq!(1, res2.len()); - let payload = res2.pop().unwrap_or_default().0; - assert_eq!(0, payload.len()); - } - - #[test] - fn length_prefix() -> Result<()> { - let mut it = 0; - - let pre_p = length_prefixed::LengthPrefixed::default(); - let mut post_p = post::length_prefixed::LengthPrefixed::default(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; - let wire = post_p.process(0, 0, &data)?; - let (start, end) = wire[0].split_at(7); - let alias = Alias::new("test", "test"); - let mut pps: Vec> = vec![Box::new(pre_p)]; - let recv = preprocess( - pps.as_mut_slice(), - &mut it, - start.to_vec(), - Value::object(), - &alias, - )?; - assert!(recv.is_empty()); - let recv = preprocess( - pps.as_mut_slice(), - &mut it, - end.to_vec(), - Value::object(), - &alias, - )?; - assert_eq!(recv[0].0, data); - - // incomplete data - let processed = preprocess( - pps.as_mut_slice(), - &mut it, - start.to_vec(), - Value::object(), - &alias, - )?; - assert!(processed.is_empty()); - // not emitted upon finish - let finished = finish(pps.as_mut_slice(), &alias)?; - assert!(finished.is_empty()); - - Ok(()) - } - - const LOOKUP_TABLE: [&str; 8] = [ - "separate", - "base64", - "decompress", - "remove-empty", - "gelf-chunking", - "ingest-ns", - "length-prefixed", - "textual-length-prefixed", - ]; - - #[test] - fn test_lookup() { - for t in &LOOKUP_TABLE { - assert!(lookup(t).is_ok()); - } - let t = "snot"; - assert!(lookup(t).is_err()); - - assert!(lookup("bad_lookup").is_err()); - } - - #[test] - fn test_filter_empty() { - let mut pre = remove_empty::RemoveEmpty::default(); - assert_eq!(Ok(vec![]), pre.process(&mut 0_u64, &[], Value::object())); - assert_eq!(Ok(vec![]), pre.finish(None, None)); - } - - #[test] - fn test_filter_null() { - let mut pre = remove_empty::RemoveEmpty::default(); - assert_eq!(Ok(vec![]), pre.process(&mut 0_u64, &[], Value::object())); - assert_eq!(Ok(vec![]), pre.finish(None, None)); - } - - #[test] - fn test_lines() -> Result<()> { - let int = "snot\nbadger".as_bytes(); - let enc = "snot\nbadger\n".as_bytes(); // First event ( event per line ) - let out = "snot".as_bytes(); - - let mut post = SeparatePost::default(); - let mut pre = separate::Separate::default(); - - let mut ingest_ns = 0_u64; - let egress_ns = 1_u64; - - let r = post.process(ingest_ns, egress_ns, int); - assert!(r.is_ok(), "Expected Ok(...), Got: {r:?}"); - let ext = &r?[0]; - let ext = ext.as_slice(); - // Assert actual encoded form is as expected - assert_eq!(enc, ext); - - let r = pre.process(&mut ingest_ns, ext, Value::object()); - let out2 = &r?[0].0; - let out2 = out2.as_slice(); - // Assert actual decoded form is as expected - assert_eq!(out, out2); - - // assert empty finish, no leftovers - assert!(pre.finish(None, None)?.is_empty()); - Ok(()) - } - - #[test] - fn test_separate_buffered() -> Result<()> { - let input = "snot\nbadger\nwombat\ncapybara\nquagga".as_bytes(); - let mut pre = separate::Separate::new(b'\n', 1000, true); - let mut ingest_ns = 0_u64; - let mut res = pre.process(&mut ingest_ns, input, Value::object())?; - let splitted = input - .split(|c| *c == b'\n') - .map(|v| (v.to_vec(), Value::object())) - .collect::>(); - assert_eq!(splitted[..splitted.len() - 1].to_vec(), res); - let mut finished = pre.finish(None, None)?; - res.append(&mut finished); - assert_eq!(splitted, res); - Ok(()) - } - - macro_rules! assert_separate_no_buffer { - ($inbound:expr, $outbound1:expr, $outbound2:expr, $case_number:expr, $separator:expr) => { - let mut ingest_ns = 0_u64; - let r = separate::Separate::new($separator, 0, false).process( - &mut ingest_ns, - $inbound, - Value::object(), - ); - - let out = &r?; - // Assert preprocessor output is as expected - assert!( - 2 == out.len(), - "Test case : {} => expected output = {}, actual output = {}", - $case_number, - "2", - out.len() - ); - assert!( - $outbound1 == out[0].0.as_slice(), - "Test case : {} => expected output = \"{}\", actual output = \"{}\"", - $case_number, - std::str::from_utf8($outbound1).unwrap(), - std::str::from_utf8(out[0].0.as_slice()).unwrap() - ); - assert!( - $outbound2 == out[1].0.as_slice(), - "Test case : {} => expected output = \"{}\", actual output = \"{}\"", - $case_number, - std::str::from_utf8($outbound2).unwrap(), - std::str::from_utf8(out[1].0.as_slice()).unwrap() - ); - }; - } - - #[allow(clippy::type_complexity)] - #[test] - fn test_separate_no_buffer_no_maxlength() -> Result<()> { - let test_data: [(&'static [u8], &'static [u8], &'static [u8], &'static str); 4] = [ - (b"snot\nbadger", b"snot", b"badger", "0"), - (b"snot\n", b"snot", b"", "1"), - (b"\nsnot", b"", b"snot", "2"), - (b"\n", b"", b"", "3"), - ]; - for case in &test_data { - assert_separate_no_buffer!(case.0, case.1, case.2, case.3, b'\n'); - } - - Ok(()) - } - - #[allow(clippy::type_complexity)] - #[test] - fn test_carriage_return_no_buffer_no_maxlength() -> Result<()> { - let test_data: [(&'static [u8], &'static [u8], &'static [u8], &'static str); 4] = [ - (b"snot\rbadger", b"snot", b"badger", "0"), - (b"snot\r", b"snot", b"", "1"), - (b"\rsnot", b"", b"snot", "2"), - (b"\r", b"", b"", "3"), - ]; - for case in &test_data { - assert_separate_no_buffer!(case.0, case.1, case.2, case.3, b'\r'); - } - - Ok(()) - } - - #[test] - fn test_base64() -> Result<()> { - let int = "snot badger".as_bytes(); - let enc = "c25vdCBiYWRnZXI=".as_bytes(); - - let mut pre = base64::Base64::default(); - let mut post = post::base64::Base64::default(); - - // Fake ingest_ns and egress_ns - let mut ingest_ns = 0_u64; - let egress_ns = 1_u64; - - let r = post.process(ingest_ns, egress_ns, int); - let ext = &r?[0]; - let ext = ext.as_slice(); - // Assert actual encoded form is as expected - assert_eq!(&enc, &ext); - - let r = pre.process(&mut ingest_ns, ext, Value::object()); - let out = &r?[0].0; - let out = out.as_slice(); - // Assert actual decoded form is as expected - assert_eq!(&int, &out); - - // assert empty finish, no leftovers - assert!(pre.finish(None, None)?.is_empty()); - Ok(()) - } - - struct BadPreprocessor {} - impl Preprocessor for BadPreprocessor { - fn name(&self) -> &'static str { - "chucky" - } - - fn process( - &mut self, - _ingest_ns: &mut u64, - _data: &[u8], - _meta: Value<'static>, - ) -> Result, Value<'static>)>> { - Err("chucky".into()) - } - fn finish( - &mut self, - _data: Option<&[u8]>, - _meta: Option>, - ) -> Result, Value<'static>)>> { - Ok(vec![]) - } - } - - struct BadFinisher {} - impl Preprocessor for BadFinisher { - fn name(&self) -> &'static str { - "chucky" - } - - fn process( - &mut self, - _ingest_ns: &mut u64, - _data: &[u8], - _meta: Value<'static>, - ) -> Result, Value<'static>)>> { - Ok(vec![]) - } - - fn finish( - &mut self, - _data: Option<&[u8]>, - _meta: Option>, - ) -> Result, Value<'static>)>> { - Err("chucky revenge".into()) - } - } - - struct NoOp {} - impl Preprocessor for NoOp { - fn name(&self) -> &'static str { - "nily" - } - - fn process( - &mut self, - _ingest_ns: &mut u64, - _data: &[u8], - meta: Value<'static>, - ) -> Result, Value<'static>)>> { - Ok(vec![(b"non".to_vec(), meta)]) - } - fn finish( - &mut self, - _data: Option<&[u8]>, - meta: Option>, - ) -> Result, Value<'static>)>> { - Ok(vec![(b"nein".to_vec(), meta.unwrap_or_else(Value::object))]) - } - } - - #[test] - fn badly_behaved_process() { - let mut pre = Box::new(BadPreprocessor {}); - assert_eq!("chucky", pre.name()); - - let mut ingest_ns = 0_u64; - let r = pre.process(&mut ingest_ns, b"foo", Value::object()); - assert!(r.is_err()); - - let r = pre.finish(Some(b"foo"), Some(Value::object())); - assert!(r.is_ok()); - } - - #[test] - fn badly_behaved_finish() { - let mut pre = Box::new(BadFinisher {}); - assert_eq!("chucky", pre.name()); - - let mut ingest_ns = 0_u64; - let r = pre.process(&mut ingest_ns, b"foo", Value::object()); - assert!(r.is_ok()); - - let r = pre.finish(Some(b"foo"), Some(Value::object())); - assert!(r.is_err()); - } - - #[test] - fn single_pre_process_head_ok() { - let pre = Box::new(BadPreprocessor {}); - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); - let mut ingest_ns = 0_u64; - let r = preprocess( - &mut [pre], - &mut ingest_ns, - b"foo".to_vec(), - Value::object(), - &alias, - ); - assert!(r.is_err()); - } - - #[test] - fn single_pre_process_tail_err() { - let noop = Box::new(NoOp {}); - assert_eq!("nily", noop.name()); - let pre = Box::new(BadPreprocessor {}); - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); - let mut ingest_ns = 0_u64; - let r = preprocess( - &mut [noop, pre], - &mut ingest_ns, - b"foo".to_vec(), - Value::object(), - &alias, - ); - assert!(r.is_err()); - } - - #[test] - fn single_pre_finish_ok() { - let pre = Box::new(BadPreprocessor {}); - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); - let r = finish(&mut [pre], &alias); - assert!(r.is_ok()); - } - - #[test] - fn direct_pre_finish_err() { - let mut pre = Box::new(BadFinisher {}); - let r = pre.finish(Some(b"foo"), Some(Value::object())); - assert!(r.is_err()); - } - - #[test] - fn preprocess_finish_head_fail() { - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); - let pre = Box::new(BadFinisher {}); - let r = finish(&mut [pre], &alias); - assert!(r.is_err()); - } - - #[test] - fn preprocess_finish_tail_fail() { - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); - let noop = Box::new(NoOp {}); - let pre = Box::new(BadFinisher {}); - let r = finish(&mut [noop, pre], &alias); - assert!(r.is_err()); - } - - #[test] - fn preprocess_finish_multi_ok() { - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("xyz"), - "xyz".to_string(), - ); - let noop1 = Box::new(NoOp {}); - let noop2 = Box::new(NoOp {}); - let noop3 = Box::new(NoOp {}); - let r = finish(&mut [noop1, noop2, noop3], &alias); - assert!(r.is_ok()); - } -} diff --git a/src/raft.rs b/src/raft.rs new file mode 100644 index 0000000000..e85ef7d178 --- /dev/null +++ b/src/raft.rs @@ -0,0 +1,210 @@ +// Copyright 2023, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod api; +pub mod archive; +pub(crate) mod manager; +pub mod network; +pub mod node; +pub mod store; + +#[cfg(test)] +mod test; + +use crate::raft::node::Addr; + +pub(crate) use self::manager::Cluster; +use api::client::Error; +use network::raft_network_impl::Network; +use openraft::{ + error::{Fatal, InitializeError, RaftError}, + metrics::WaitError, + storage::Adaptor, + Config, ConfigError, Raft, TokioRuntime, +}; +use std::{ + fmt::{Display, Formatter}, + io::Cursor, + sync::Mutex, +}; +use store::{TremorRequest, TremorResponse}; +use tokio::task::JoinError; + +/// A `NodeId` +pub type NodeId = u64; + +/// load a default raft config +/// # Errors +/// When the config isn't valid +pub fn config() -> ClusterResult { + let config = Config { + heartbeat_interval: 150, + election_timeout_min: 299, + ..Default::default() + }; + Ok(config.validate()?) +} + +openraft::declare_raft_types!( + /// Declare the type configuration for example K/V store. + pub TremorRaftConfig: D = TremorRequest, R = TremorResponse, NodeId = NodeId, Node = node::Addr, + Entry = openraft::Entry, + SnapshotData = Cursor>, + AsyncRuntime = TokioRuntime +); + +pub type LogStore = Adaptor; +pub type StateMachineStore = Adaptor; + +pub type TremorRaftImpl = Raft; + +#[derive(Debug)] +pub enum ClusterError { + Other(String), + Rocks(rocksdb::Error), + Io(std::io::Error), + Store(store::Error), + Initialize(RaftError>), + MsgPackEncode(rmp_serde::encode::Error), + MsgPackDecode(rmp_serde::decode::Error), + Config(ConfigError), + Client(Error), + JoinError(JoinError), + Fatal(Fatal), + WaitError(WaitError), + // TODO: this is a horrible hack + Runtime(Mutex), +} + +impl From for ClusterError { + fn from(e: store::Error) -> Self { + ClusterError::Store(e) + } +} + +impl From for ClusterError { + fn from(e: std::io::Error) -> Self { + ClusterError::Io(e) + } +} + +impl From for ClusterError { + fn from(e: rocksdb::Error) -> Self { + ClusterError::Rocks(e) + } +} +impl From<&str> for ClusterError { + fn from(e: &str) -> Self { + ClusterError::Other(e.to_string()) + } +} + +impl From for ClusterError { + fn from(e: String) -> Self { + ClusterError::Other(e) + } +} + +impl From>> for ClusterError { + fn from(e: RaftError>) -> Self { + ClusterError::Initialize(e) + } +} + +impl From for ClusterError { + fn from(e: ConfigError) -> Self { + ClusterError::Config(e) + } +} + +impl From for ClusterError { + fn from(e: Error) -> Self { + Self::Client(e) + } +} + +impl From for ClusterError { + fn from(e: crate::Error) -> Self { + ClusterError::Runtime(Mutex::new(e)) + } +} + +impl From for ClusterError { + fn from(e: rmp_serde::encode::Error) -> Self { + ClusterError::MsgPackEncode(e) + } +} + +impl From for ClusterError { + fn from(e: rmp_serde::decode::Error) -> Self { + ClusterError::MsgPackDecode(e) + } +} + +impl From for ClusterError { + fn from(e: JoinError) -> Self { + ClusterError::JoinError(e) + } +} + +impl From> for ClusterError { + fn from(e: Fatal) -> Self { + ClusterError::Fatal(e) + } +} + +impl From for ClusterError { + fn from(e: WaitError) -> Self { + ClusterError::WaitError(e) + } +} + +impl Display for ClusterError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ClusterError::Other(e) => e.fmt(f), + ClusterError::Rocks(e) => e.fmt(f), + ClusterError::Io(e) => e.fmt(f), + ClusterError::Store(e) => e.fmt(f), + ClusterError::Initialize(e) => e.fmt(f), + ClusterError::Config(e) => e.fmt(f), + ClusterError::Runtime(e) => write!(f, "{:?}", e.lock()), + ClusterError::MsgPackEncode(e) => e.fmt(f), + ClusterError::MsgPackDecode(e) => e.fmt(f), + ClusterError::Client(e) => e.fmt(f), + ClusterError::JoinError(e) => e.fmt(f), + ClusterError::Fatal(e) => e.fmt(f), + ClusterError::WaitError(e) => e.fmt(f), + } + } +} + +impl std::error::Error for ClusterError {} + +type ClusterResult = Result; + +/// Removes a node from a cluster +/// # Errors +/// When the node can't be removed +pub async fn remove_node( + node_id: NodeId, + api_addr: &T, +) -> Result<(), crate::errors::Error> { + let client = api::client::Tremor::new(api_addr)?; + client.demote_voter(&node_id).await?; + client.remove_learner(&node_id).await?; + client.remove_node(&node_id).await?; + println!("Membership updated: node {node_id} removed."); + Ok(()) +} diff --git a/src/raft/api.rs b/src/raft/api.rs new file mode 100644 index 0000000000..4f77360967 --- /dev/null +++ b/src/raft/api.rs @@ -0,0 +1,456 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains the HTTP API exposed externally. +//! for inter-node communication look into the `network` module + +pub(crate) mod apps; +pub mod client; +mod cluster; +pub mod kv; +pub(crate) mod worker; + +use self::apps::AppState; +use crate::{ + channel::{OneShotSender, Receiver, Sender}, + raft::{ + node::Addr, + store::{self, StateApp, Store, TremorResponse}, + NodeId, TremorRaftImpl, + }, +}; +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, Router, +}; +use http::{header::LOCATION, HeaderMap, Uri}; +use openraft::{ + error::{ + ChangeMembershipError, ClientWriteError, Fatal, ForwardToLeader, QuorumNotEnough, RaftError, + }, + StorageError, +}; +use std::collections::HashMap; +use std::{collections::BTreeSet, num::ParseIntError, sync::Arc, time::Duration}; +use tokio::{task::JoinHandle, time::timeout}; +use tremor_common::alias; + +pub(crate) type APIRequest = Arc; +pub type APIResult = Result; + +const API_WORKER_TIMEOUT: Duration = Duration::from_secs(5); + +pub(crate) type ReplySender = OneShotSender; + +#[derive(Debug)] +pub enum APIStoreReq { + GetApp(alias::App, ReplySender>), + GetApps(ReplySender>), + KVGet(String, ReplySender>>), + GetNode(NodeId, ReplySender>), + GetNodes(ReplySender>), + GetNodeId(Addr, ReplySender>), + GetLastMembership(ReplySender>), +} + +pub(crate) struct ServerState { + id: NodeId, + addr: Addr, + raft: TremorRaftImpl, + raft_manager: super::Cluster, +} + +impl ServerState { + pub(crate) async fn ensure_leader(&self, uri: Option) -> Result<(), APIError> { + self.raft.is_leader().await.map_err(|e| match e { + RaftError::APIError(e) => match e { + openraft::error::CheckIsLeaderError::ForwardToLeader(e) => { + // forward_to_leader(e, uri, state).await + e.leader_id + .zip(e.leader_node) + .map_or(APIError::NoLeader, |(node_id, addr)| { + APIError::ForwardToLeader { node_id, addr, uri } + }) + } + openraft::error::CheckIsLeaderError::QuorumNotEnough(e) => APIError::NoQuorum(e), + }, + RaftError::Fatal(e) => APIError::Fatal(e), + }) + } + pub(crate) fn id(&self) -> NodeId { + self.id + } + pub(crate) fn addr(&self) -> &Addr { + &self.addr + } +} + +pub(crate) fn initialize( + id: NodeId, + addr: Addr, + raft: TremorRaftImpl, + store: Store, + store_tx: Sender, + store_rx: Receiver, +) -> (JoinHandle<()>, Arc) { + let handle = tokio::task::spawn(worker::api_worker(store, store_rx)); + let state = Arc::new(ServerState { + id, + addr, + raft: raft.clone(), + raft_manager: super::Cluster::new(id, store_tx, raft), + }); + (handle, state) +} + +pub(crate) fn endpoints() -> Router { + Router::new() + .nest("/v1/cluster", cluster::endpoints()) + .nest("/v1/api/apps", apps::endpoints()) + .nest("/v1/api/kv", kv::endpoints()) +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub enum ArgsError { + Missing(String), + Invalid(String), +} + +impl std::fmt::Display for ArgsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Missing(m) => write!(f, "Missing required argument \"{m}\""), + Self::Invalid(inv) => write!(f, "Unknown argument \"{inv}\""), + } + } +} + +#[derive(Debug, Serialize)] +pub enum AppError { + /// app is already installed + AlreadyInstalled(alias::App), + + /// app not found + AppNotFound(alias::App), + FlowNotFound(alias::Flow), + InstanceNotFound(alias::Flow), + /// App has at least 1 running instances (and thus cannot be uninstalled) + HasInstances(alias::App, Vec), + InstanceAlreadyExists(alias::Flow), + /// provided flow args are invalid + InvalidArgs { + flow: alias::Flow, + instance: alias::Flow, + errors: Vec, + }, +} + +impl std::error::Error for AppError {} + +impl std::fmt::Display for AppError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::AlreadyInstalled(app_id) => write!(f, "App \"{app_id}\" is already installed."), + Self::AppNotFound(app_id) => write!(f, "App \"{app_id}\" not found."), + Self::HasInstances(app_id, instances) => write!( + f, + "App \"{app_id}\" has running instances: {}", + instances + .iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ), + Self::InstanceAlreadyExists(instance_id) => write!( + f, + "App \"{}\" already has an instance \"{}\"", + instance_id.app_id(), + instance_id.instance_id() + ), + Self::InstanceNotFound(instance_id) => write!( + f, + "No instance \"{}\" in App \"{}\" found", + instance_id.instance_id(), + instance_id.app_id() + ), + Self::FlowNotFound(flow_id) => { + write!(f, "Flow \"{flow_id}\" not found.") + } + Self::InvalidArgs { + flow, + instance, + errors, + } => write!( + f, + "Invalid Arguments provided for instance \"{instance}\" of Flow \"{flow}\": {}", + errors + .iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ), + } + } +} + +#[allow(clippy::trivially_copy_pass_by_ref)] +fn round_sc(x: &StatusCode, s: S) -> Result +where + S: serde::Serializer, +{ + s.serialize_u16(x.as_u16()) +} + +#[derive(Debug, Serialize)] +pub enum APIError { + /// We need to send this API request to the leader_url + ForwardToLeader { + node_id: NodeId, + addr: Addr, + #[serde(skip)] + uri: Option, + }, + /// HTTP related error + HTTP { + #[serde(serialize_with = "round_sc")] + status: StatusCode, + message: String, + }, + /// raft fatal error, includes StorageError + Fatal(Fatal), + /// We don't have a quorum to read + NoQuorum(QuorumNotEnough), + /// We don't have a leader + NoLeader, + /// Error when changing a membership + ChangeMembership(ChangeMembershipError), + /// Error from our store/statemachine + Store(String), + /// openraft storage error + Storage(StorageError), + /// Errors around tremor apps + App(AppError), + /// Error in the runtime + Runtime(String), + // Timeout + Timeout, + // recv error + Recv, + /// fallback error type + Other(String), +} +impl IntoResponse for APIError { + fn into_response(self) -> Response { + let status = match &self { + APIError::ForwardToLeader { .. } => StatusCode::TEMPORARY_REDIRECT, + APIError::ChangeMembership(ChangeMembershipError::LearnerNotFound(_)) => { + StatusCode::NOT_FOUND + } + APIError::ChangeMembership(ChangeMembershipError::EmptyMembership(_)) => { + StatusCode::BAD_REQUEST + } + APIError::Other(_) + | APIError::ChangeMembership(_) + | APIError::Store(_) + | APIError::Storage(_) + | APIError::Fatal(_) + | APIError::Runtime(_) + | APIError::Recv + | APIError::App(_) => StatusCode::INTERNAL_SERVER_ERROR, + APIError::NoLeader | APIError::NoQuorum(_) => StatusCode::SERVICE_UNAVAILABLE, + APIError::Timeout => StatusCode::GATEWAY_TIMEOUT, + APIError::HTTP { status, .. } => *status, + }; + + if let APIError::ForwardToLeader { addr, uri, .. } = self { + let mut headers = HeaderMap::new(); + let uri = if let Some(uri) = uri { + let path_and_query = if let Some(query) = uri.query() { + format!("{}?{}", uri.path(), query) + } else { + uri.path().to_string() + }; + http::uri::Builder::new() + .scheme(uri.scheme_str().unwrap_or("http")) + .authority(addr.api()) + .path_and_query(path_and_query) + .build() + .unwrap_or_default() + } else { + Uri::default() + }; + + if let Ok(v) = uri.to_string().parse() { + headers.insert(LOCATION, v); + } + (status, headers).into_response() + } else { + (status, Json(self)).into_response() + } + } +} + +#[async_trait::async_trait] +trait ToAPIResult +where + T: serde::Serialize + serde::Deserialize<'static>, +{ + async fn to_api_result(self, uri: &Uri, req: &APIRequest) -> APIResult; +} + +#[async_trait::async_trait()] +impl + Send> ToAPIResult + for Result>> +{ + // we need the request context here to construct the redirect url properly + async fn to_api_result(self, uri: &Uri, state: &APIRequest) -> APIResult { + match self { + Ok(response) => Ok(response), + Err(RaftError::APIError(ClientWriteError::ForwardToLeader(e))) => { + forward_to_leader(e, uri, state).await + } + Err(RaftError::APIError(ClientWriteError::ChangeMembershipError(e))) => { + Err(APIError::ChangeMembership(e)) + } + Err(RaftError::Fatal(e)) => Err(APIError::Fatal(e)), + } + } +} + +#[allow(clippy::unused_async)] +async fn forward_to_leader( + e: ForwardToLeader, + uri: &Uri, + state: &APIRequest, +) -> APIResult +where + T: serde::Serialize + serde::Deserialize<'static>, +{ + Err(if let Some(leader_id) = e.leader_id { + // we can only forward to the leader if we have the node in our state machine + if let Some(leader_addr) = + timeout(API_WORKER_TIMEOUT, state.raft_manager.get_node(leader_id)).await?? + { + debug!("Forwarding to leader: {uri}"); + // we don't care about fragment + + APIError::ForwardToLeader { + node_id: leader_id, + addr: leader_addr, + uri: Some(uri.clone()), + } + } else { + APIError::Other(format!("Leader {leader_id} not known")) + } + } else { + dbg!(); + APIError::NoLeader + }) +} + +impl IntoResponse for TremorResponse { + fn into_response(self) -> Response { + (StatusCode::OK, Json(self)).into_response() + } +} + +impl std::fmt::Display for APIError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + APIError::ForwardToLeader { + addr: leader_addr, + node_id, + uri, + } => f + .debug_struct("ForwardToLeader") + .field("leader_addr", leader_addr) + .field("node_id", node_id) + .field("uri", uri) + .finish(), + APIError::Other(s) | Self::Store(s) => write!(f, "{s}"), + APIError::HTTP { message, status } => write!(f, "HTTP {status} {message}"), + APIError::Fatal(e) => write!(f, "Fatal Error: {e}"), + APIError::ChangeMembership(e) => write!(f, "Error changing cluster membership: {e}"), + APIError::NoQuorum(e) => write!(f, "Quorum: {e}"), + APIError::Storage(e) => write!(f, "Storage: {e}"), + APIError::Runtime(e) => write!(f, "Runtime: {e}"), + APIError::App(e) => write!(f, "App: {e}"), + APIError::Timeout => write!(f, "Timeout"), + APIError::Recv => write!(f, "Recv error"), + APIError::NoLeader => write!(f, "No Leader"), + } + } +} +impl std::error::Error for APIError {} + +impl From for APIError { + fn from(e: store::Error) -> Self { + Self::Store(e.to_string()) + } +} + +impl From for APIError { + fn from(e: url::ParseError) -> Self { + Self::Other(e.to_string()) + } +} + +impl From> for APIError { + fn from(e: StorageError) -> Self { + Self::Storage(e) + } +} + +impl From for APIError { + fn from(e: ParseIntError) -> Self { + Self::Other(e.to_string()) + } +} + +impl From for APIError { + fn from(app: AppError) -> Self { + APIError::App(app) + } +} + +impl From for APIError { + fn from(e: crate::Error) -> Self { + APIError::Runtime(e.to_string()) + } +} + +impl From> for APIError { + fn from(e: crate::channel::SendError) -> Self { + Self::Other(e.to_string()) + } +} + +impl From for APIError { + fn from(_: tokio::sync::oneshot::error::RecvError) -> Self { + Self::Recv + } +} + +impl From for APIError { + fn from(_: tokio::time::error::Elapsed) -> Self { + Self::Timeout + } +} + +impl From for APIError { + fn from(e: simd_json::Error) -> Self { + Self::Other(e.to_string()) + } +} diff --git a/src/raft/api/apps.rs b/src/raft/api/apps.rs new file mode 100644 index 0000000000..1b91759d10 --- /dev/null +++ b/src/raft/api/apps.rs @@ -0,0 +1,351 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use crate::{ + instance::IntendedState, + raft::{ + api::{APIRequest, APIResult, AppError, ArgsError, ToAPIResult, API_WORKER_TIMEOUT}, + archive::{get_app, TremorAppDef}, + store::{ + AppsRequest as AppsCmd, FlowInstance, Instances, StateApp, TremorInstanceState, + TremorRequest, TremorResponse, TremorStart, + }, + }, + system::flow::DeploymentType, +}; +use axum::{ + extract::{self, Json, State}, + routing::{delete, post}, + Router, +}; +use std::collections::HashMap; +use std::fmt::Display; +use tokio::time::timeout; +use tremor_common::alias; + +pub(crate) fn endpoints() -> Router { + Router::::new() + .route("/", post(install_app).get(list)) + .route("/:app", delete(uninstall_app)) + .route("/:app/flows/:flow", post(start)) + .route( + "/:app/instances/:instance", + post(manage_instance).delete(stop_instance), + ) +} + +async fn install_app( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + Json(file): Json>, +) -> APIResult> { + let app = get_app(&file)?; + let app_id = app.name().clone(); + + if timeout( + API_WORKER_TIMEOUT, + state.raft_manager.get_app_local(app_id.clone()), + ) + .await?? + .is_some() + { + return Err(AppError::AlreadyInstalled(app.name).into()); + } + let request = TremorRequest::Apps(AppsCmd::InstallApp { + app, + file: file.clone(), + }); + state + .raft + .client_write(request) + .await + .to_api_result(&uri, &state) + .await?; + Ok(Json(app_id)) +} + +async fn uninstall_app( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path(app_id): extract::Path, +) -> APIResult { + let app = timeout( + API_WORKER_TIMEOUT, + state.raft_manager.get_app_local(app_id.clone()), + ) + .await??; + if let Some(app) = app { + if !app.instances.is_empty() { + return Err(AppError::HasInstances( + app_id.clone(), + app.instances + .keys() + .map(|alias| alias::Flow::new(app_id.clone(), alias)) + .collect(), + ) + .into()); + } + } else { + return Err(AppError::AppNotFound(app_id.clone()).into()); + } + let request = TremorRequest::Apps(AppsCmd::UninstallApp { + app: app_id.clone(), + force: false, + }); + state + .raft + .client_write(request) + .await + .to_api_result(&uri, &state) + .await + .map(|d| d.data) +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct AppState { + pub def: TremorAppDef, + pub instances: Instances, +} + +impl From<&StateApp> for AppState { + fn from(state: &StateApp) -> Self { + AppState { + def: state.app.clone(), + instances: state.instances.clone(), + } + } +} + +impl Display for AppState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Name: {}", self.def.name)?; + writeln!(f, "Flows: ")?; + for (name, flow) in &self.def.flows { + writeln!(f, " - {name}")?; + write!(f, " Config:")?; + if flow.args.is_empty() { + writeln!(f, " -")?; + } else { + writeln!(f)?; + for (name, val) in &flow.args { + write!(f, " - {name}: ")?; + if let Some(val) = val { + writeln!(f, "{val}")?; + } else { + writeln!(f, "-")?; + } + } + } + } + writeln!(f, " Instances: ")?; + for ( + name, + FlowInstance { + definition, + config, + state, + .. + }, + ) in &self.instances + { + writeln!(f, " - {name}")?; + writeln!(f, " Flow definition: {definition}")?; + writeln!(f, " Config:")?; + if config.is_empty() { + writeln!(f, " -")?; + } else { + writeln!(f)?; + for (name, val) in config { + writeln!(f, " - {name}: {val}")?; + } + } + writeln!(f, " State: {state}")?; + } + Ok(()) + } +} + +async fn list(State(state): State) -> APIResult>> { + let apps = timeout(API_WORKER_TIMEOUT, state.raft_manager.get_apps_local()).await??; + Ok(Json(apps)) +} + +async fn start( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path((app_id, flow_id)): extract::Path<(alias::App, alias::FlowDefinition)>, + Json(body): Json, +) -> APIResult> { + let instance_id = body.instance.clone(); + let flow_id = alias::Flow::new(app_id, flow_id); + + let app = timeout( + API_WORKER_TIMEOUT, + state.raft_manager.get_app_local(flow_id.app_id().clone()), + ) + .await??; + + if let Some(app) = app { + if app.instances.contains_key(instance_id.instance_id()) { + return Err(AppError::InstanceAlreadyExists(body.instance).into()); + } + if let Some(flow) = app.app.flows.get(flow_id.instance_id()) { + if let Some(errors) = config_errors(&flow.args, &body.config) { + return Err(AppError::InvalidArgs { + flow: flow_id, + instance: body.instance, + errors, + } + .into()); + } + } else { + return Err(AppError::FlowNotFound(flow_id).into()); + } + } else { + return Err(AppError::AppNotFound(flow_id.app_id().clone()).into()); + } + let request = TremorRequest::Apps(AppsCmd::Deploy { + app: flow_id.app_id().clone(), + flow: flow_id.instance_id().into(), + instance: body.instance.clone(), + config: body.config.clone(), + state: body.state(), + deployment_type: if body.single_node { + DeploymentType::OneNode + } else { + DeploymentType::AllNodes + }, + }); + state + .raft + .client_write(request) + .await + .to_api_result(&uri, &state) + .await?; + Ok(Json(instance_id)) +} + +async fn manage_instance( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path((app_id, flow_id)): extract::Path<(alias::App, String)>, + Json(body): Json, +) -> APIResult> { + // FIXME: this is not only for this but all the API functions as we're running in a potentially + // problematic situation here. + // + // The raft algorithm expects all commands and statemachine changes to be excecutable and not fail. + // That means we need to do all the checks before we send the command to the raft core, however + // the checks here are executed potentially first on a follower, then on the leader and then + // send to raft. + // This has two problems: + // 1) if the follow didn't catch up with the leader yet we might get a false negative here in the + // way that the follow claims a command would fail but the leader would accept it. + // + // Example: client sends install to leader, leader accepts it, client sends start to follower, + // the install hasn't been replicated to the follower yet, follower rejects the start. + // + // 2) the leader might change it's state between the command being tested and the command being + // forwarded to the raft algorithm and serialized. + // + // Example: leader gets two uninstall commands in quick succession, it checks the first, sends + // it to raft, the second one arrives and is checked before reft propagated the first + // request so it is accepted as well but fails. + // + // Solution? We might need to put a single process inbetween the API and the raft algorithm that + // serializes all commands to ensure no command is executed before the previous one has been fully + // handled + let instance_id = alias::Flow::new(app_id.clone(), flow_id); + + let app = timeout( + API_WORKER_TIMEOUT, + state.raft_manager.get_app_local(app_id.clone()), + ) + .await??; + if let Some(app) = app { + if !app.instances.contains_key(instance_id.instance_id()) { + return Err(AppError::InstanceNotFound(instance_id).into()); + } + } else { + return Err(AppError::AppNotFound(app_id).into()); + } + let body_state = match body { + TremorInstanceState::Pause => IntendedState::Paused, + TremorInstanceState::Resume => IntendedState::Running, + }; + + let request = TremorRequest::Apps(AppsCmd::InstanceStateChange { + instance: instance_id.clone(), + state: body_state, + }); + state + .raft + .client_write(request) + .await + .to_api_result(&uri, &state) + .await?; + Ok(Json(instance_id)) +} + +async fn stop_instance( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path((app_id, flow_id)): extract::Path<(alias::App, String)>, +) -> APIResult> { + debug!("Stop instance request for {app_id}/{flow_id}"); + let instance_id = alias::Flow::new(app_id.clone(), flow_id); + + if let Some(app) = timeout( + API_WORKER_TIMEOUT, + state.raft_manager.get_app_local(app_id.clone()), + ) + .await?? + { + if !app.instances.contains_key(instance_id.instance_id()) { + return Err(AppError::InstanceNotFound(instance_id).into()); + } + } else { + return Err(AppError::AppNotFound(app_id).into()); + } + let request = TremorRequest::Apps(AppsCmd::Undeploy(instance_id.clone())); + state + .raft + .client_write(request) + .await + .to_api_result(&uri, &state) + .await?; + Ok(Json(instance_id)) +} + +/// check the given `args` for errors according to the specified `config` from the flow definition +fn config_errors( + args: &HashMap>, + config: &HashMap, +) -> Option> { + let mut errors = Vec::new(); + for present_key in config.keys() { + if !args.contains_key(present_key) { + errors.push(ArgsError::Invalid(present_key.to_string())); + } + } + for (key, val) in args { + if val.is_none() && !config.contains_key(key) { + errors.push(ArgsError::Missing(key.to_string())); + } + } + if errors.is_empty() { + None + } else { + Some(errors) + } +} diff --git a/src/raft/api/client.rs b/src/raft/api/client.rs new file mode 100644 index 0000000000..2302dfe350 --- /dev/null +++ b/src/raft/api/client.rs @@ -0,0 +1,455 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tremor Rest API Client +use crate::errors::Result; +use crate::raft::{ + api::apps::AppState, + node::Addr, + store::{TremorInstanceState, TremorStart}, + NodeId, +}; +use halfbrown::HashMap; +use openraft::{LogId, RaftMetrics}; +use reqwest::Method; +use reqwest::{redirect::Policy, Client}; +use serde::{de::DeserializeOwned, Serialize}; +use simd_json::OwnedValue; +use tremor_common::alias; + +use super::kv::KVSet; + +type ClientResult = std::result::Result; + +const DEFAULT_RETRIES: usize = 10; + +pub struct Tremor { + /// The endpoint to send requests to. + pub endpoint: String, + + pub inner: Client, +} + +impl Tremor { + /// Create + /// + /// # Errors + /// if the client cannot be created + pub fn new(endpoint: &T) -> Result { + let inner = reqwest::Client::builder() + .redirect(Policy::limited(DEFAULT_RETRIES)) + .build()?; + Ok(Self { + endpoint: endpoint.to_string(), + inner, + }) + } + // --- Internal methods + + /// Send RPC to specified node. + /// + /// It sends out a POST request if `req` is Some. Otherwise a GET request. + /// The remote endpoint must respond a reply in form of `Result`. + /// An `Err` happened on remote will be wrapped in an [`RPCError::RemoteError`]. + async fn api_req( + &self, + uri: &str, + method: reqwest::Method, + req: Option<&Req>, + ) -> ClientResult + where + Req: Serialize + 'static + ?Sized, + Resp: Serialize + DeserializeOwned, + { + let target_url = { + let target_addr = &self.endpoint; + format!("http://{target_addr}/v1/{uri}") + }; + debug!(">>> client send {method} request to {target_url}"); + let mut request = self.inner.request(method, &target_url); + if let Some(req) = req { + request = request.json(req); + } + // FYI: 307 redirects are followed here with a default redirect limit of 10 + // if we target a non-leader, it will return with a 307 and redirect us to the leader + let resp = request.send().await?; + if resp.status().is_success() { + let result: Resp = resp.json().await?; + if log::log_enabled!(log::Level::Debug) { + if let Ok(json) = serde_json::to_string_pretty(&result) { + debug!("<<< client recv reply from {target_url}: {json}",); + } + } + Ok(result) + } else if let Err(err) = resp.error_for_status_ref() { + error!( + "Received {} with body: {}", + resp.status(), + resp.text().await? + ); + Err(Error::HTTP(err)) + } else { + Err("Heisenerror, not error nor success".into()) + } + } +} + +// --- kv API +impl Tremor { + /// Submit a write request to the raft cluster. + /// + /// The request will be processed by raft protocol: it will be replicated to a quorum and then will be applied to + /// state machine. + /// + /// The written value will be returned as a string. + /// + /// # Errors + /// if the api call fails + pub async fn write(&self, req: &KVSet) -> ClientResult { + self.api_req::("api/kv/write", Method::POST, Some(req)) + .await + } + /// Read value by key, in an inconsistent mode. + /// + /// This method may return stale value because it does not force to read on a legal leader. + /// + /// # Errors + /// if the api call fails + pub async fn read(&self, req: &str) -> ClientResult { + self.api_req("api/kv/read", Method::POST, Some(req)).await + } + + /// Consistent Read value by key. + /// + /// This method MUST return consistent value or `CheckIsLeaderError`. + /// + /// # Errors + /// if the api call fails + pub async fn consistent_read(&self, req: &str) -> ClientResult { + self.api_req("api/kv/consistent_read", Method::POST, Some(req)) + .await + } +} + +// --- Application API +impl Tremor { + /// Submit a write request to the raft cluster. + /// + /// The request will be processed by raft protocol: it will be replicated to a quorum and then will be applied to + /// state machine. + /// + /// The result of applying the request will be returned. + /// + /// # Errors + /// if the api call fails + pub async fn install(&self, req: &Vec) -> ClientResult { + self.api_req::, alias::App>("api/apps", Method::POST, Some(req)) + .await + } + + /// Submit a write request to the raft cluster. + /// + /// The request will be processed by raft protocol: it will be replicated to a quorum and then will be applied to + /// state machine. + /// + /// The result of applying the request will be returned. + /// + /// # Errors + /// if the api call fails + pub async fn uninstall_app(&self, app: &alias::App) -> ClientResult { + self.api_req::<(), alias::App>(&format!("api/apps/{app}"), Method::DELETE, None) + .await + } + + /// Submit a write request to the raft cluster. + /// + /// The request will be processed by raft protocol: it will be replicated to a quorum and then will be applied to + /// state machine. + /// + /// The result of applying the request will be returned. + /// + /// # Errors + /// if the api call fails + pub async fn start( + &self, + flow: &alias::FlowDefinition, + instance: &alias::Flow, + config: std::collections::HashMap, + running: bool, + single_node: bool, + ) -> ClientResult { + let req = TremorStart { + instance: instance.clone(), + config, + running, + single_node, + }; + self.api_req::( + &format!("api/apps/{}/flows/{flow}", instance.app_id()), + Method::POST, + Some(&req), + ) + .await + } + + /// Submit a write request to the raft cluster. + /// + /// The request will be processed by raft protocol: it will be replicated to a quorum and then will be applied to + /// state machine. + /// + /// The result of applying the request will be returned. + /// + /// # Errors + /// if the api call fails + pub async fn change_instance_state( + &self, + instance: &alias::Flow, + state: TremorInstanceState, + ) -> ClientResult { + self.api_req( + &format!( + "api/apps/{}/instances/{}", + instance.app_id(), + instance.instance_id() + ), + Method::POST, + Some(&state), + ) + .await + } + + /// Submit a write request to the raft cluster. + /// + /// The request will be processed by raft protocol: it will be replicated to a quorum and then will be applied to + /// state machine. + /// + /// The result of applying the request will be returned. + /// + /// # Errors + /// if the api call fails + pub async fn stop_instance(&self, instance: &alias::Flow) -> ClientResult { + self.api_req( + &format!( + "api/apps/{}/instances/{}", + instance.app_id(), + instance.instance_id() + ), + Method::DELETE, + None::<&()>, + ) + .await + } + + /// Get the metrics about the cluster. + /// + /// Metrics contains various information about the cluster, such as current leader, + /// membership config, replication status etc. + /// See [`RaftMetrics`]. + /// + /// # Errors + /// if the api call fails + pub async fn list(&self) -> ClientResult> { + self.api_req("api/apps", Method::GET, None::<&()>).await + } +} +// Cluster +impl Tremor { + // --- Cluster management API + + /// Make the given node known to the cluster and assign it a unique `node_id` + /// # Errors + /// If the api call fails + pub async fn add_node(&self, addr: &Addr) -> ClientResult { + self.api_req("cluster/nodes", Method::POST, Some(addr)) + .await + } + + /// Remove a node from the cluster + /// + /// After this call a node is not reachable anymore for all nodes still participating in the cluster + /// # Errors + /// if the api call fails + pub async fn remove_node(&self, node_id: &NodeId) -> ClientResult<()> { + self.api_req( + &format!("cluster/nodes/{node_id}"), + Method::DELETE, + None::<&()>, + ) + .await + } + + /// Get all the nodes with their id and address that are currently known to the cluster + /// + /// # Errors + /// if the api call fails + pub async fn get_nodes(&self) -> ClientResult> { + self.api_req("cluster/nodes", Method::GET, None::<&()>) + .await + } + + /// Add a node as learner. + /// + /// If the node has never been added to the cluster before, its address will be published in the cluster state + /// so that all other nodes can reach it. + /// + /// # Errors + /// if the api call fails e.g. because the node is already a learner + pub async fn add_learner(&self, node_id: &NodeId) -> ClientResult>> { + self.api_req( + &format!("cluster/learners/{node_id}"), + Method::PUT, + None::<&()>, + ) + .await + } + + /// Add a node as learner. + /// + /// The node to add has to exist, i.e., being added with `write(ExampleRequest::AddNode{})` + /// + /// # Errors + /// if the api call fails + pub async fn remove_learner(&self, id: &NodeId) -> ClientResult<()> { + self.api_req::<(), ()>(&format!("cluster/learners/{id}"), Method::DELETE, None) + .await + } + + /// Promote node with `node_id` from learner to voter. + /// + /// The node with `node_id` has to be already added as learner with [`add_learner`], + /// or an error [`LearnerNotFound`] will be returned. + /// + /// # Errors + /// if the api call fails + pub async fn promote_voter(&self, id: &NodeId) -> ClientResult> { + self.api_req::<(), Option>(&format!("cluster/voters/{id}"), Method::PUT, None) + .await + } + + /// Demote node with `node_id` from voter back to learner. + /// + /// All nodes in `req` have to be already added as learner with [`add_learner`], + /// or an error [`LearnerNotFound`] will be returned. + /// + /// # Errors + /// if the api call fails + pub async fn demote_voter(&self, id: &NodeId) -> ClientResult> { + self.api_req::<(), Option>(&format!("cluster/voters/{id}"), Method::DELETE, None) + .await + } + + /// Get the metrics about the cluster. + /// + /// Metrics contains various information about the cluster, such as current leader, + /// membership config, replication status etc. + /// See [`RaftMetrics`]. + /// + /// # Errors + /// if the api call fails + pub async fn metrics(&self) -> ClientResult> { + self.api_req::<(), RaftMetrics>("cluster/metrics", Method::GET, None) + .await + } +} + +fn maybe_id(id: Option) -> String { + match id { + Some(id) => id.to_string(), + None => "-".to_string(), + } +} +pub fn print_metrics(metrics: &RaftMetrics) { + println!( + r#"Node: + Id: {} + State: {:?} + +Cluster: + Term: {} + Last log: {} + Last applied: {} + Leader: {}"#, + metrics.id, + metrics.state, + metrics.current_term, + maybe_id(metrics.last_log_index), + maybe_id(metrics.last_applied), + maybe_id(metrics.current_leader), + ); + let membership = &metrics.membership_config; + let log_id = membership.log_id().unwrap_or_default(); + if let Some(config) = membership.membership().get_joint_config().last() { + println!( + r#" +Membership: + Log: {log_id} + Nodes:"# + ); + for id in config { + println!(" - {id}"); + } + } + + println!( + r#" +Snapshot: + {:?} +"#, + metrics.snapshot, + ); +} + +#[derive(Debug)] +pub enum Error { + HTTP(reqwest::Error), + Other(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::HTTP(e) => e.fmt(f), + Self::Other(e) => e.fmt(f), + } + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(e: reqwest::Error) -> Self { + Self::HTTP(e) + } +} + +impl From for Error { + fn from(e: crate::Error) -> Self { + Self::Other(e.to_string()) + } +} +impl<'s> From<&'s str> for Error { + fn from(e: &'s str) -> Self { + Self::Other(e.into()) + } +} + +impl Error { + #[must_use] + pub fn is_not_found(&self) -> bool { + match self { + Self::HTTP(e) => e.status() == Some(reqwest::StatusCode::NOT_FOUND), + Self::Other(_) => false, + } + } +} diff --git a/src/raft/api/cluster.rs b/src/raft/api/cluster.rs new file mode 100644 index 0000000000..cab2c7c89e --- /dev/null +++ b/src/raft/api/cluster.rs @@ -0,0 +1,244 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{APIRequest, API_WORKER_TIMEOUT}; +use crate::raft::{ + api::{APIError, APIResult, ToAPIResult}, + node::Addr, + store::{NodesRequest, TremorRequest}, + NodeId, +}; +use axum::{ + extract::{self, Json}, + routing::{delete, get, put}, + Router, +}; +use http::StatusCode; +use openraft::{ChangeMembers, LogId, RaftMetrics}; +use std::collections::{BTreeSet, HashMap}; +use tokio::time::timeout; + +pub(crate) fn endpoints() -> Router { + Router::::new() + .route("/nodes", get(get_nodes).post(add_node)) + .route("/nodes/:node_id", delete(remove_node)) + .route( + "/learners/:node_id", + put(add_learner).patch(add_learner).delete(remove_learner), + ) + .route( + "/voters/:node_id", + put(promote_voter).patch(promote_voter).delete(demote_voter), + ) + .route("/metrics", get(metrics)) +} + +/// Get a list of all currently known nodes (be it learner, leader, voter etc.) +async fn get_nodes( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, +) -> APIResult>> { + state.ensure_leader(Some(uri)).await?; + + let nodes = timeout(API_WORKER_TIMEOUT, state.raft_manager.get_nodes()).await??; + Ok(Json(nodes)) +} + +/// Make a node known to cluster by putting it onto the cluster state +/// +/// This is a precondition for the node being added as learner and promoted to voter +async fn add_node( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + Json(addr): Json, +) -> APIResult> { + // 1. ensure we are on the leader, as we need to read some state-machine state + // in order to give a good answer here + state.ensure_leader(Some(uri.clone())).await?; + + // 2. ensure we don't add the node twice if it is already there + // we need to make sure we don't hold on to the state machine lock any further here + + let maybe_existing_node_id = timeout( + API_WORKER_TIMEOUT, + state.raft_manager.get_node_id(addr.clone()), + ) + .await??; + if let Some(existing_node_id) = maybe_existing_node_id { + Ok(Json(existing_node_id)) + } else { + // 2a. add the node with its metadata to the state machine, so the network impl can reach it + // this will fail, when we are not on the leader + // when this succeeds the local state machine should have the node addr stored, so the network can access it + // in order to establish a network connection + debug!("node {addr} not yet known to cluster"); + let response = state + .raft + .client_write(TremorRequest::Nodes(NodesRequest::AddNode { + addr: addr.clone(), + })) + .await + .to_api_result(&uri, &state) + .await?; + let node_id: NodeId = NodeId::try_from(response.data)?; + debug!("node {addr} added to the cluster as node {node_id}"); + Ok(Json(node_id)) + } +} + +/// Remove the node from the cluster +/// +/// # Errors +/// if the API call fails +async fn remove_node( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path(node_id): extract::Path, +) -> APIResult> { + // make sure the node is not a learner or a voter + + let membership = + timeout(API_WORKER_TIMEOUT, state.raft_manager.get_last_membership()).await??; + if membership.contains(&node_id) { + return Err(APIError::HTTP { + status: StatusCode::CONFLICT, + message: format!("Node {node_id} cannot be removed as it is still a voter."), + }); + } + // TODO: how to check if the node is a learner? + // remove the node metadata from the state machine + state + .raft + .client_write(TremorRequest::Nodes(NodesRequest::RemoveNode { node_id })) + .await + .to_api_result(&uri, &state) + .await?; + Ok(Json(())) +} + +/// Add a node as **Learner**. +/// +/// A Learner receives log replication from the leader but does not vote. +/// This should be done before adding a node as a member into the cluster +async fn add_learner( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path(node_id): extract::Path, +) -> APIResult>> { + // 1. ensure we are on the leader, as we need to read some state-machine state + // in order to give a good answer here + state.ensure_leader(Some(uri.clone())).await?; + + // 2. check that the node has already been added + // we need to make sure we don't hold on to the state machine lock any further here + + let node_addr = timeout(API_WORKER_TIMEOUT, state.raft_manager.get_node(node_id)) + .await?? + .ok_or(APIError::HTTP { + status: StatusCode::NOT_FOUND, + message: format!("Node {node_id} is not known to the cluster yet."), + })?; + + // add the node as learner + debug!("Adding node {node_id} as learner..."); + state + .raft + .add_learner(node_id, node_addr, true) + .await + .to_api_result(&uri, &state) + .await + .map(|d| Json(d.log_id)) +} + +/// Removes a node from **Learners** only +/// +async fn remove_learner( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path(node_id): extract::Path, +) -> APIResult> { + debug!("[API] Removing learner {node_id}",); + // remove the node as learner + // let result = state.raft.remove_learner(node_id).await; + let mut nodes = BTreeSet::new(); + nodes.insert(node_id); + + let _result = state + .raft + .change_membership(ChangeMembers::RemoveNodes(nodes), true) + .await + .to_api_result(&uri, &state) + .await?; + Ok(Json(())) +} + +/// Changes specified learners to members, or remove members. +async fn promote_voter( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path(node_id): extract::Path, +) -> APIResult>> { + // we introduce a new scope here to release the lock on the state machine + // not releasing it can lead to dead-locks, if executed on the leader (as the store is shared between the API and the raft engine) + + let mut membership = + timeout(API_WORKER_TIMEOUT, state.raft_manager.get_last_membership()).await??; + + let value = if membership.insert(node_id) { + // only update state if not already in the membership config + // this call always returns TremorResponse { value: None } // see store.rs + state + .raft + .change_membership(membership, true) + .await + .to_api_result(&uri, &state) + .await?; + Some(node_id) + } else { + None + }; + Ok(Json(value)) +} + +/// Changes specified learners to members, or remove members. +async fn demote_voter( + extract::State(state): extract::State, + extract::OriginalUri(uri): extract::OriginalUri, + extract::Path(node_id): extract::Path, +) -> APIResult>> { + // scoping here to not hold the state machine locked for too long + + let mut membership = + timeout(API_WORKER_TIMEOUT, state.raft_manager.get_last_membership()).await??; + let value = if membership.remove(&node_id) { + state + .raft + .change_membership(membership, true) + .await + .to_api_result(&uri, &state) + .await?; + Some(node_id) + } else { + None + }; + Ok(Json(value)) +} + +/// Get the latest metrics of the cluster (from the viewpoint of the targeted node) +#[allow(clippy::unused_async)] +async fn metrics( + extract::State(state): extract::State, +) -> APIResult>> { + Ok(Json(state.raft.metrics().borrow().clone())) +} diff --git a/src/raft/api/kv.rs b/src/raft/api/kv.rs new file mode 100644 index 0000000000..a4a97e3a0d --- /dev/null +++ b/src/raft/api/kv.rs @@ -0,0 +1,79 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::raft::api::{APIError, APIRequest, APIResult}; +use axum::{extract, routing::post, Json, Router}; +use http::StatusCode; +use simd_json::OwnedValue; +use tokio::time::timeout; + +use super::API_WORKER_TIMEOUT; + +pub(crate) fn endpoints() -> Router { + Router::::new() + .route("/write", post(write)) + .route("/read", post(read)) + .route("/consistent_read", post(consistent_read)) +} + +/// KV Write request +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct KVSet { + /// they key + pub key: String, + /// the value + pub value: OwnedValue, +} + +async fn write( + extract::State(state): extract::State, + extract::Json(body): extract::Json, +) -> APIResult> { + Ok(state + .raft_manager + .kv_set(body.key, simd_json::to_vec(&body.value)?) + .await?) +} + +/// read a value from the current node, not necessarily the leader, thus this value can be stale +async fn read( + extract::State(state): extract::State, + extract::Json(key): extract::Json, +) -> APIResult> { + let value = timeout(API_WORKER_TIMEOUT, state.raft_manager.kv_get_local(key)).await??; + if let Some(value) = value { + Ok(Json(value)) + } else { + Err(APIError::HTTP { + status: StatusCode::NOT_FOUND, + message: "Key not found".to_string(), + }) + } +} + +/// read a value from the leader. If this request is received by another node, it will return a redirect +async fn consistent_read( + extract::State(state): extract::State, + extract::Json(key): extract::Json, +) -> APIResult> { + let value = timeout(API_WORKER_TIMEOUT, state.raft_manager.kv_get(key)).await??; + if let Some(value) = value { + Ok(Json(value)) + } else { + Err(APIError::HTTP { + status: StatusCode::NOT_FOUND, + message: "Key not found".to_string(), + }) + } +} diff --git a/src/raft/api/worker.rs b/src/raft/api/worker.rs new file mode 100644 index 0000000000..1b18e637a5 --- /dev/null +++ b/src/raft/api/worker.rs @@ -0,0 +1,75 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::apps::AppState; +use crate::channel::{OneShotSender, Receiver}; +use crate::raft::{api::APIStoreReq, store::Store}; +use std::collections::HashMap; +use tremor_common::alias; + +fn send(tx: OneShotSender, t: T) { + if tx.send(t).is_err() { + error!("Error sending response to API"); + } +} + +pub(super) async fn api_worker(store: Store, mut store_rx: Receiver) { + while let Some(msg) = store_rx.recv().await { + match msg { + APIStoreReq::GetApp(app_id, tx) => { + let sm = store.state_machine.read().await; + send(tx, sm.apps.get_app(&app_id).cloned()); + } + APIStoreReq::GetApps(tx) => { + let sm = store.state_machine.read().await; + send( + tx, + sm.apps + .list() + .map(|(k, v)| (k.clone(), AppState::from(v))) + .collect::>(), + ); + } + APIStoreReq::KVGet(k, tx) => { + let sm = store.state_machine.read().await; + let v = sm.kv.get(k.as_str()).ok().flatten(); // return errors as not-found here, this might be bad + send(tx, v); + } + APIStoreReq::GetNode(node_id, tx) => { + let sm = store.state_machine.read().await; + let node = sm.nodes.get_node(node_id).cloned(); + send(tx, node); + } + APIStoreReq::GetNodes(tx) => { + let sm = store.state_machine.read().await; + let nodes = sm.nodes.get_nodes().clone(); + send(tx, nodes); + } + APIStoreReq::GetNodeId(addr, tx) => { + let sm = store.state_machine.read().await; + let node_id = sm.nodes.find_node_id(&addr).copied(); + send(tx, node_id); + } + APIStoreReq::GetLastMembership(tx) => { + let sm = store.state_machine.read().await; + let membership = sm.get_last_membership().ok().flatten(); // return errors as option here, this might be bad + let last_membership = membership + .and_then(|m| m.membership().get_joint_config().last().cloned()) + .unwrap_or_default(); + send(tx, last_membership); + } + } + } + info!("API Worker done."); +} diff --git a/src/raft/archive.rs b/src/raft/archive.rs new file mode 100644 index 0000000000..b7956204f7 --- /dev/null +++ b/src/raft/archive.rs @@ -0,0 +1,289 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::errors::Result; +use sha2::{Digest, Sha256}; +use simd_json::OwnedValue; +use std::{ + collections::{BTreeSet, HashMap}, + io::Read, + path::PathBuf, +}; +use tar::{Archive, Header}; +use tokio::io::AsyncWriteExt; +use tremor_common::{alias, asy::file, base64}; +use tremor_script::{ + arena::{self, Arena}, + ast::{ + warning::{Class, Warning}, + DeployStmt, Helper, NodeId, + }, + deploy::Deploy, + highlighter::{self, Highlighter}, + module::{Manager, PreCachedNodes, MODULES}, + prelude::Ranged, + NodeMeta, FN_REGISTRY, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AppFlow { + /// arguments with possible default values + /// arguments without default values are required + pub args: HashMap>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct TremorAppDef { + pub name: alias::App, + /// hash of all the included files + /// starting with the main.troy and then all `use`d files in order + pub sha256: String, + pub flows: HashMap, +} + +impl TremorAppDef { + #[must_use] + pub fn name(&self) -> &alias::App { + &self.name + } +} + +/// Packages a tremor application into a tarball, entry point is the `main.troy` file, target the tar.gz file +/// # Errors +/// if the tarball cannot be created +pub async fn package(target: &str, entrypoint: &str, name: Option) -> Result<()> { + let mut output = file::create(target).await?; + let file = PathBuf::from(entrypoint); + let dir = file.parent().ok_or("Failed to get parent dir")?; + let path = dir.to_string_lossy(); + info!("Adding {path} to path"); + Manager::add_path(&path)?; + let name = name + .or_else(|| { + file.file_stem() + .and_then(std::ffi::OsStr::to_str) + .map(ToString::to_string) + }) + .ok_or("Failed to get name")?; + info!("Building archive for {name}"); + output + .write_all(&build_archive(&name, entrypoint).await?) + .await?; + Ok(()) +} + +pub(crate) async fn build_archive(name: &str, entrypoint: &str) -> Result> { + let src = file::read_to_string(entrypoint).await?; + build_archive_from_source(name, src.as_str()) +} + +#[allow(clippy::too_many_lines)] +pub(crate) fn build_archive_from_source(name: &str, src: &str) -> Result> { + use tar::Builder; + let mut hasher = Sha256::new(); + + let aggr_reg = tremor_script::registry::aggr(); + let mut hl = highlighter::Term::stderr(); + + let mut deploy = match Deploy::parse(&src, &*FN_REGISTRY.read()?, &aggr_reg) { + Ok(deploy) => deploy, + Err(e) => { + hl.format_error(&e)?; + return Err(e.into()); + } + }; + let mut other_warnings = BTreeSet::new(); + let reg = &*FN_REGISTRY.read()?; + let helper = Helper::new(reg, &aggr_reg); + + for stmt in &deploy.deploy.stmts { + match stmt { + DeployStmt::FlowDefinition(_) + | DeployStmt::PipelineDefinition(_) + | DeployStmt::ConnectorDefinition(_) => (), + DeployStmt::DeployFlowStmt(f) => { + let warning = Warning { + class: Class::Behaviour, + outer: f.extent(), + inner: f.extent(), + msg: "Deploying flows in applications is not supported. This statement will be ignored".to_string(), + }; + other_warnings.insert(warning); + } + } + } + let flows: HashMap<_, _> = deploy + .deploy + .scope + .content + .flows + .iter() + .map(|(k, v)| { + debug!("Flow {k} added to tremor archive."); + Ok(( + alias::Instance(k.to_string()), + AppFlow { + args: v + .clone() + .params + .args + .0 + .iter() + .map(|(k, v)| { + Ok(( + k.to_string(), + v.clone() + .map(|v| v.try_into_value(&helper)) + .transpose()? + .map(OwnedValue::from), + )) + }) + .collect::>()?, + }, + )) + }) + .collect::>()?; + if !flows.contains_key(&alias::Instance("main".to_string())) { + let w = Warning { + class: Class::Behaviour, + outer: deploy.extent(), + inner: deploy.extent(), + msg: "No main flow found".to_string(), + }; + other_warnings.insert(w); + } + + deploy.warnings.extend(other_warnings); + deploy.format_warnings_with(&mut hl)?; + + // first hash main.troy + hasher.update(src.as_bytes()); + // then hash all the modules + for aid in MODULES.read()?.modules().iter().map(|m| m.arena_idx) { + if let Some(src) = Arena::get(aid)? { + hasher.update(src.as_bytes()); + } + } + let hash = base64::encode(hasher.finalize().as_slice()); + info!("App {name} Package hash: {hash}"); + + let app = TremorAppDef { + name: alias::App(name.to_string()), + sha256: hash, + flows, + }; + + let mut ar = Builder::new(Vec::new()); + let app = simd_json::to_vec(&app)?; + let mut header = Header::new_gnu(); + header.set_size(app.len() as u64); + header.set_cksum(); + ar.append_data(&mut header, "app.json", app.as_slice())?; + + let mut header = Header::new_gnu(); + header.set_size(src.as_bytes().len() as u64); + header.set_cksum(); + ar.append_data(&mut header, "main.troy", src.as_bytes())?; + + for (id, paths) in MODULES + .read()? + .modules() + .iter() + .map(|m| (m.arena_idx, m.paths())) + { + if let Some(src) = Arena::get(id)? { + for p in paths { + let mut file: PathBuf = p.module().iter().collect(); + file.push(p.id()); + let mut header = Header::new_gnu(); + header.set_size(src.as_bytes().len() as u64); + header.set_cksum(); + debug!("Adding module {paths:?} with id {id} as file {file:?} to archive"); + ar.append_data(&mut header, file, src.as_bytes())?; + } + } else { + error!("Module {paths:?} not found"); + } + } + Ok(ar.into_inner()?) +} + +/// gets the app name from an archive +/// # Errors +/// if the archive is invalid +pub fn get_app(src: &[u8]) -> Result { + let mut ar = Archive::new(src); + + let mut entries = ar.entries()?; + let mut app = entries.next().ok_or("Invalid archive: empty")??; + + if app.path()?.to_string_lossy() != "app.json" { + return Err("Invalid archive: app.json missing".into()); + } + let mut content = String::new(); + app.read_to_string(&mut content)?; + + let app: TremorAppDef = serde_json::from_str(&content)?; + Ok(app) +} + +/// Extract app deploy an all used arena indices +/// # Errors +/// if the archive is invalid +pub fn extract(src: &[u8]) -> Result<(TremorAppDef, Deploy, Vec)> { + let mut ar = Archive::new(src); + + let mut entries = ar.entries()?; + let mut app = entries.next().ok_or("Invalid archive: empty")??; + + if app.path()?.to_string_lossy() != "app.json" { + return Err("Invalid archive: app.json missing".into()); + } + let mut content = String::new(); + app.read_to_string(&mut content)?; + + let app: TremorAppDef = serde_json::from_str(&content)?; + + let mut main = entries + .next() + .ok_or("Invalid archive: main.troy missing")??; + + content.clear(); + main.read_to_string(&mut content)?; + let main = content; + + let mut modules = PreCachedNodes::new(); + + for e in entries { + let mut entry = e?; + let path = entry.path()?; + let mut module: Vec<_> = path + .iter() + .map(|p| p.to_string_lossy().to_string()) + .collect(); + let id = module.pop().ok_or("No module name")?; + let module = NodeId::new(id.clone(), module.clone(), NodeMeta::dummy()); + + info!("included library: {}", entry.path()?.to_string_lossy()); + let mut contents = String::new(); + entry.read_to_string(&mut contents)?; + let (aid, _) = Arena::insert(&contents)?; + modules.insert(module, aid); + } + let aggr_reg = tremor_script::registry::aggr(); + let deploy = Deploy::parse_with_cache(&main, &*FN_REGISTRY.read()?, &aggr_reg, &modules)?; + let mut aids = modules.values().collect::>(); + aids.push(deploy.aid); + Ok((app, deploy, aids)) +} diff --git a/src/raft/manager.rs b/src/raft/manager.rs new file mode 100644 index 0000000000..6bd95217ae --- /dev/null +++ b/src/raft/manager.rs @@ -0,0 +1,199 @@ +// Copyright 2021, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{BTreeSet, HashMap}; + +use super::{ + node::Addr, + store::{StateApp, TremorRequest, TremorSet}, + TremorRaftConfig, TremorRaftImpl, +}; +use crate::channel::{oneshot, Sender}; +use crate::raft::api::APIStoreReq; +use crate::Result; +use crate::{ + errors::{Error, ErrorKind}, + raft::NodeId, +}; +use openraft::{ + error::{CheckIsLeaderError, ForwardToLeader, RaftError}, + raft::ClientWriteResponse, +}; +use simd_json::OwnedValue; +use tremor_common::alias; + +#[derive(Clone, Default)] +pub(crate) struct Cluster { + node_id: NodeId, + raft: Option, + sender: Option>, +} + +impl std::fmt::Debug for Cluster { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Manager") + .field("node_id", &self.node_id) + .field("raft", &self.raft.is_some()) + .field("sender", &self.sender.is_some()) + .finish() + } +} + +impl Cluster { + pub(crate) fn id(&self) -> NodeId { + self.node_id + } + async fn send(&self, command: APIStoreReq) -> Result<()> { + self.sender + .as_ref() + .ok_or(crate::errors::ErrorKind::RaftNotRunning)? + .send(command) + .await?; + Ok(()) + } + + fn raft(&self) -> Result<&TremorRaftImpl> { + Ok(self + .raft + .as_ref() + .ok_or(crate::errors::ErrorKind::RaftNotRunning)?) + } + + async fn client_write(&self, command: T) -> Result> + where + T: Into + Send + 'static, + { + Ok(self.raft()?.client_write(command.into()).await?) + } + + pub(crate) async fn is_leader(&self) -> Result<()> { + Ok(self.raft()?.is_leader().await?) + } + + pub(crate) fn new(node_id: NodeId, sender: Sender, raft: TremorRaftImpl) -> Self { + Self { + node_id, + raft: Some(raft), + sender: Some(sender), + } + } + + // cluster + pub(crate) async fn get_node(&self, node_id: u64) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::GetNode(node_id, tx); + self.send(command).await?; + Ok(rx.await?) + } + pub(crate) async fn get_nodes(&self) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::GetNodes(tx); + self.send(command).await?; + Ok(rx.await?) + } + pub(crate) async fn get_node_id(&self, addr: Addr) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::GetNodeId(addr, tx); + self.send(command).await?; + Ok(rx.await?) + } + pub(crate) async fn get_last_membership(&self) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::GetLastMembership(tx); + self.send(command).await?; + Ok(rx.await?) + } + + // apps + pub(crate) async fn get_app_local(&self, app_id: alias::App) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::GetApp(app_id, tx); + self.send(command).await?; + Ok(rx.await?) + } + + pub(crate) async fn get_apps_local( + &self, + ) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::GetApps(tx); + self.send(command).await?; + Ok(rx.await?) + } + + // kv + pub(crate) async fn kv_set(&self, key: String, mut value: Vec) -> Result> { + match self.is_leader().await { + Ok(()) => self.kv_set_local(key, value).await, + Err(Error( + ErrorKind::CheckIsLeaderError(RaftError::APIError( + CheckIsLeaderError::ForwardToLeader(ForwardToLeader { + leader_node: Some(n), + .. + }), + )), + _, + )) => { + let client = crate::raft::api::client::Tremor::new(n.api())?; + // TODO: there should be a better way to forward then the client + Ok(simd_json::to_vec( + &client + .write(&crate::raft::api::kv::KVSet { + key, + value: simd_json::from_slice(&mut value)?, + }) + .await?, + )?) + } + Err(e) => Err(e), + } + } + pub(crate) async fn kv_set_local(&self, key: String, value: Vec) -> Result> { + let tremor_res = self.client_write(TremorSet { key, value }).await?; + tremor_res.data.into_kv_value() + } + + pub(crate) async fn kv_get(&self, key: String) -> Result> { + match self.is_leader().await { + Ok(()) => self.kv_get_local(key).await, + Err(Error( + ErrorKind::CheckIsLeaderError(RaftError::APIError( + CheckIsLeaderError::ForwardToLeader(ForwardToLeader { + leader_node: Some(n), + .. + }), + )), + _, + )) => { + let client = crate::raft::api::client::Tremor::new(n.api())?; + let res = client.read(&key).await; + match res { + Ok(v) => Ok(Some(v)), + Err(e) if e.is_not_found() => Ok(None), + Err(e) => Err(e.into()), + } + } + Err(e) => Err(e), + } + } + pub(crate) async fn kv_get_local(&self, key: String) -> Result> { + let (tx, rx) = oneshot(); + let command = APIStoreReq::KVGet(key, tx); + self.send(command).await?; + Ok(rx + .await? + .map(|mut v| simd_json::from_slice(&mut v)) + .transpose()?) + } +} diff --git a/src/raft/network.rs b/src/raft/network.rs new file mode 100644 index 0000000000..160fbd3c4d --- /dev/null +++ b/src/raft/network.rs @@ -0,0 +1,37 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::TremorRaftConfig; +use openraft::raft::{ + AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, InstallSnapshotResponse, + VoteRequest, VoteResponse, +}; +use openraft::AnyError; +use tarpc; + +pub mod raft; +pub mod raft_network_impl; + +#[tarpc::service] +pub(crate) trait Raft { + async fn vote( + vote: VoteRequest, + ) -> Result, AnyError>; + async fn append( + req: AppendEntriesRequest, + ) -> Result, AnyError>; + async fn snapshot( + req: InstallSnapshotRequest, + ) -> Result, AnyError>; +} diff --git a/src/raft/network/raft.rs b/src/raft/network/raft.rs new file mode 100644 index 0000000000..5f810089df --- /dev/null +++ b/src/raft/network/raft.rs @@ -0,0 +1,69 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use openraft::{ + raft::{ + AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, + InstallSnapshotResponse, VoteRequest, VoteResponse, + }, + AnyError, +}; +use tarpc::context; + +use crate::raft::{TremorRaftConfig, TremorRaftImpl}; + +// --- Raft communication server side +#[derive(Clone)] +pub struct Server { + raft: Arc, +} + +impl Server { + pub(crate) fn new(raft: Arc) -> Self { + Self { raft } + } +} + +#[tarpc::server] +impl super::Raft for Server { + async fn vote( + self, + _: context::Context, + vote: VoteRequest, + ) -> Result, AnyError> { + self.raft.vote(vote).await.map_err(|e| AnyError::new(&e)) + } + async fn append( + self, + _: context::Context, + req: AppendEntriesRequest, + ) -> Result, AnyError> { + self.raft + .append_entries(req) + .await + .map_err(|e| AnyError::new(&e)) + } + async fn snapshot( + self, + _: context::Context, + req: InstallSnapshotRequest, + ) -> Result, AnyError> { + self.raft + .install_snapshot(req) + .await + .map_err(|e| AnyError::new(&e)) + } +} diff --git a/src/raft/network/raft_network_impl.rs b/src/raft/network/raft_network_impl.rs new file mode 100644 index 0000000000..6134130e64 --- /dev/null +++ b/src/raft/network/raft_network_impl.rs @@ -0,0 +1,190 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::raft::{network::RaftClient, node::Addr, NodeId, TremorRaftConfig}; +use async_trait::async_trait; +use openraft::{ + error::{InstallSnapshotError, NetworkError, RaftError}, + raft::{ + AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, + InstallSnapshotResponse, VoteRequest, VoteResponse, + }, + AnyError, RaftNetwork, RaftNetworkFactory, +}; +use std::cmp::min; +use tarpc::{client::RpcError, context}; +use tokio::time::sleep; + +#[derive(Clone, Debug)] +pub struct Network {} + +impl Network { + pub(crate) fn new() -> Self { + Self {} + } +} + +// NOTE: This could be implemented also on `Arc`, but since it's empty, implemented +// directly. +#[async_trait] +impl RaftNetworkFactory for Network { + type Network = NetworkConnection; + + async fn new_client(&mut self, target: NodeId, node: &Addr) -> Self::Network { + NetworkConnection { + client: None, + target, + addr: node.clone(), + error_count: 0, + last_reconnect: std::time::Instant::now(), + } + } +} + +const RECONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); + +pub struct NetworkConnection { + client: Option, + target: NodeId, + addr: Addr, + error_count: u32, + last_reconnect: std::time::Instant, +} + +// We need this for the types :sob: +#[derive(Debug)] +struct SendError(String); + +impl std::fmt::Display for SendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Send error: {}", self.0) + } +} + +impl std::error::Error for SendError {} + +impl NetworkConnection { + /// Ensure we have a client to the node identified by `target` + /// + /// Pick it from the pool first, if that fails, create a new one + /// /// Ensure we have a client to the node identified by `target` + /// + /// Pick it from the pool first, if that fails, create a new one + async fn ensure_client(&mut self) -> Result> { + // TODO: get along without the cloning + + match &self.client { + Some(client) => Ok(client.clone()), + None => { + sleep(RECONNECT_TIMEOUT * min(self.error_count, 10)).await; + let client = self.new_client().await?; + self.client = Some(client.clone()); + Ok(client) + } + } + } + + /// Create a new TCP client for the given `target` node + /// + /// This requires the `target` to be known to the cluster state. + async fn new_client(&mut self) -> Result> { + let transport = tarpc::serde_transport::tcp::connect( + self.addr.rpc(), + tarpc::tokio_serde::formats::Bincode::default, + ) + .await + .map_err(|e| RPCError::Network(NetworkError::new(&e))); + if transport.is_err() { + self.error_count += 1; + } + let client = RaftClient::new(tarpc::client::Config::default(), transport?).spawn(); + self.last_reconnect = std::time::Instant::now(); + self.error_count = 0; + Ok(client) + } + fn handle_response( + &mut self, + target: NodeId, + response: Result, RpcError>, + ) -> Result> { + match response { + Ok(res) => res.map_err(|e| RPCError::Network(NetworkError::new(&e))), + Err(e @ RpcError::Shutdown) => { + self.client = None; + self.error_count += 1; + error!("Client disconnected from node {target}"); + Err(RPCError::Network(NetworkError::new(&e))) + } + Err(e @ RpcError::DeadlineExceeded) => { + // no need to drop the client here + // TODO: implement some form of backoff + error!("Request against node {target} timed out"); + Err(RPCError::Network(NetworkError::new(&e))) + } + Err(RpcError::Server(e)) => { + self.client = None; // TODO necessary here? + self.error_count += 1; + error!("Server error from node {target}: {e}"); + Err(RPCError::Network(NetworkError::new(&e))) + } + Err(RpcError::Send(e)) => { + self.client = None; // TODO necessary here? + self.error_count += 1; + error!("Server error from node {target}: {e}"); + Err(RPCError::Network(NetworkError::new(&SendError(format!( + "{e}" + ))))) + } + Err(RpcError::Receive(e)) => { + self.client = None; // TODO necessary here? + self.error_count += 1; + error!("Server error from node {target}: {e}"); + Err(RPCError::Network(NetworkError::new(&e))) + } + } + } +} +type RPCError = openraft::error::RPCError; +#[async_trait] +impl RaftNetwork for NetworkConnection { + // the raft engine will retry upon network failures + // so all we need to do is to drop the client if need be to trigger a reconnect + // it would be nice to have some kind of backoff, but this might halt the raft engine + // and might lead to some nasty timeouts + async fn send_append_entries( + &mut self, + rpc: AppendEntriesRequest, + ) -> Result, RPCError>> { + let client = self.ensure_client().await?; + self.handle_response(self.target, client.append(context::current(), rpc).await) + } + + async fn send_install_snapshot( + &mut self, + rpc: InstallSnapshotRequest, + ) -> Result, RPCError>> + { + let client = self.ensure_client().await?; + let r = client.snapshot(context::current(), rpc).await; + self.handle_response(self.target, r) + } + + async fn send_vote( + &mut self, + rpc: VoteRequest, + ) -> Result, RPCError>> { + let client = self.ensure_client().await?; + self.handle_response(self.target, client.vote(context::current(), rpc).await) + } +} diff --git a/src/raft/node.rs b/src/raft/node.rs new file mode 100644 index 0000000000..cb8e4c4203 --- /dev/null +++ b/src/raft/node.rs @@ -0,0 +1,476 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The entirety of a cluster node as a struct +use crate::{ + channel::{bounded, Sender}, + errors::Result, + qsize, + raft::{ + api::{self, ServerState}, + network::{raft, Raft as TarPCRaftService}, + store::{NodesRequest, Store, TremorRequest}, + Cluster, ClusterError, ClusterResult, Network, NodeId, + }, + system::{Runtime, ShutdownMode, WorldConfig}, +}; +use futures::{future, prelude::*}; +use openraft::{storage::Adaptor, Config, Raft}; +use std::{ + collections::BTreeMap, + net::ToSocketAddrs, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; +use tarpc::{ + server::{self, Channel}, + tokio_serde::formats::Bincode, +}; + +use tokio::task::{self, JoinHandle}; + +use super::TremorRaftImpl; + +#[derive(Clone, Debug)] +pub struct ClusterNodeKillSwitch { + sender: Sender, +} + +impl ClusterNodeKillSwitch { + /// Stop the running node with the given `mode` + /// # Errors + /// if the node is already stopped or failed to be stopped + pub fn stop(&self, mode: ShutdownMode) -> ClusterResult<()> { + self.sender + .try_send(mode) + .map_err(|_| ClusterError::from("Error stopping cluster node")) + } +} + +pub struct Running { + node: Node, + server_state: Arc, + kill_switch_tx: Sender, + run_handle: JoinHandle>, +} + +impl Running { + #[must_use] + pub fn node_data(&self) -> (crate::raft::NodeId, Addr) { + (self.server_state.id(), self.server_state.addr().clone()) + } + + #[must_use] + pub fn node(&self) -> &Node { + &self.node + } + + async fn start( + node: Node, + raft: TremorRaftImpl, + api_worker_handle: JoinHandle<()>, + server_state: Arc, + runtime: Runtime, + runtime_handle: JoinHandle>, + ) -> ClusterResult { + let node_id = server_state.id(); + let (kill_switch_tx, mut kill_switch_rx) = bounded(1); + + let tcp_server_state = Arc::new(raft.clone()); + let mut listener = + tarpc::serde_transport::tcp::listen(&server_state.addr().rpc(), Bincode::default) + .await?; + listener.config_mut().max_frame_length(usize::MAX); + + let http_api_addr = server_state.addr().api().to_string(); + let app = api::endpoints().with_state(server_state.clone()); + let http_api_server = + axum::Server::bind(&http_api_addr.to_socket_addrs()?.next().ok_or("badaddr")?) + .serve(app.into_make_service()); + + let run_handle = task::spawn(async move { + let mut tcp_future = Box::pin( + listener + // Ignore accept errors. + .filter_map(|r| future::ready(r.ok())) + .map(server::BaseChannel::with_defaults) + // Limit channels to 1 per IP. + // TODO .max_channels_per_key(1, |t| t.transport().peer_addr().unwrap().ip()) + // serve is generated by the service attribute. It takes as input any type implementing + // the generated World trait. + .map(|channel| { + let server = raft::Server::new(tcp_server_state.clone()); + channel.execute(server.serve()) + }) + // Max 10 channels. + .buffer_unordered(10) + .for_each(|()| async {}) + .fuse(), + ); + let mut http_future = Box::pin(http_api_server.fuse()); + let mut runtime_future = Box::pin(runtime_handle.fuse()); + let mut kill_switch_future = Box::pin(kill_switch_rx.recv().fuse()); + futures::select! { + () = tcp_future => { + warn!("[Node {node_id}] TCP cluster API shutdown."); + // Important: this will free and drop the store and thus the rocksdb + api_worker_handle.abort(); + raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + runtime.stop(ShutdownMode::Graceful).await?; + runtime_future.await??; + } + http_res = http_future => { + if let Err(e) = http_res { + error!("[Node {node_id}] HTTP cluster API failed: {e}"); + } + // Important: this will free and drop the store and thus the rocksdb + api_worker_handle.abort(); + raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + runtime.stop(ShutdownMode::Graceful).await?; + runtime_future.await??; + + } + runtime_res = runtime_future => { + if let Err(e) = runtime_res { + error!("[Node {node_id}] Local runtime failed: {e}"); + } + // Important: this will free and drop the store and thus the rocksdb + api_worker_handle.abort(); + // runtime is already down, we only need to stop local raft + raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + } + shutdown_mode = kill_switch_future => { + let shutdown_mode = shutdown_mode.unwrap_or(ShutdownMode::Forceful); + info!("[Node {node_id}] Node stopping in {shutdown_mode:?} mode"); + // Important: this will free and drop the store and thus the rocksdb + api_worker_handle.abort(); + // tcp and http api stopped listening as we don't poll them no more + raft.shutdown().await.map_err(|_| ClusterError::from("Error shutting down local raft node"))?; + info!("[Node {node_id}] Raft engine did stop."); + info!("[Node {node_id}] Stopping the Tremor runtime..."); + runtime.stop(shutdown_mode).await?; + runtime_future.await??; + info!("[Node {node_id}] Tremor runtime stopped."); + } + } + info!("[Node {node_id}] Tremor cluster node stopped"); + + Ok::<(), ClusterError>(()) + }); + + Ok(Self { + node, + server_state, + kill_switch_tx, + run_handle, + }) + } + + /// get a kill-switch + #[must_use] + pub fn kill_switch(&self) -> ClusterNodeKillSwitch { + ClusterNodeKillSwitch { + sender: self.kill_switch_tx.clone(), + } + } + + /// block until the cluster node is done noodling + /// + /// # Errors + /// if the node failed to run + pub async fn join(self) -> ClusterResult<()> { + self.run_handle.await? + } +} + +/// internal struct carrying all data to start a cluster node +/// and keeps all the state for an ordered clean shutdown +#[derive(Clone, Debug)] +pub struct Node { + db_dir: PathBuf, + raft_config: Arc, +} + +#[derive(Clone, Debug, Serialize, Deserialize, Hash, Eq, PartialEq)] +pub struct Addr { + /// Address for API access (from outside of the cluster) + api: String, + /// Address for RPC access (inter-node) + rpc: String, +} + +impl Default for Addr { + fn default() -> Self { + Self { + api: String::from("127.0.0.1:8888"), + rpc: String::from("127.0.0.1:9999"), + } + } +} + +impl std::fmt::Display for Addr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Addr") + .field("api", &self.api) + .field("rpc", &self.rpc) + .finish() + } +} + +impl Addr { + /// constructor + pub fn new(api: impl Into, rpc: impl Into) -> Self { + Self { + api: api.into(), + rpc: rpc.into(), + } + } + + /// get the api addr + #[must_use] + pub fn api(&self) -> &str { + &self.api + } + + /// get the rpc addr + #[must_use] + pub fn rpc(&self) -> &str { + &self.rpc + } +} + +impl Node { + pub fn new(db_dir: impl AsRef, raft_config: Config) -> Self { + Self { + db_dir: PathBuf::from(db_dir.as_ref()), + raft_config: Arc::new(raft_config), + } + } + /// Load the latest state from `db_dir` + /// and start the cluster with it + /// + /// # Errors + /// if the store does not exist, is not properly initialized + pub async fn load_from_store( + db_dir: impl AsRef, + raft_config: Config, + ) -> ClusterResult { + let db = Store::init_db(&db_dir)?; + // ensure we have working node data + let (node_id, addr) = Store::get_self(&db)?; + + let world_config = WorldConfig::default(); // TODO: make configurable + let (runtime, runtime_handle) = Runtime::start(world_config).await?; + let (store_tx, store_rx) = bounded(qsize()); + + let store: Store = Store::load(Arc::new(db), runtime.clone()).await?; + let node = Self::new(db_dir, raft_config.clone()); + + let network = Network::new(); + let (log_store, state_machine) = Adaptor::new(store.clone()); + let raft = Raft::new( + node_id, + node.raft_config.clone(), + network, + log_store, + state_machine, + ) + .await?; + let manager = Cluster::new(node_id, store_tx.clone(), raft.clone()); + *(runtime + .cluster_manager + .write() + .map_err(|_| "Failed to set world manager")?) = Some(manager); + let (api_worker_handle, server_state) = api::initialize( + node_id, + addr, + raft.clone(), + store.clone(), + store_tx, + store_rx, + ); + Running::start( + node, + raft, + api_worker_handle, + server_state, + runtime, + runtime_handle, + ) + .await + } + + /// Just start the cluster node and let it do whatever it does + /// # Errors + /// when the node can't be started + pub async fn try_join( + &mut self, + addr: Addr, + endpoints: Vec, + promote_to_voter: bool, + ) -> ClusterResult { + if endpoints.is_empty() { + return Err(ClusterError::Other( + "No join endpoints provided".to_string(), + )); + } + + // for now we infinitely try to join until it succeeds + let mut join_wait = Duration::from_secs(2); + let (client, node_id) = 'outer: loop { + for endpoint in &endpoints { + info!("Trying to join existing cluster via {endpoint}..."); + let client = api::client::Tremor::new(endpoint)?; + // TODO: leader will start replication stream to this node and fail, until we start our machinery + let node_id = match client.add_node(&addr).await { + Ok(node_id) => node_id, + Err(e) => { + // TODO: ensure we don't error here, when we are already learner + warn!("Error connecting to {endpoint}: {e}"); + continue; + } + }; + break 'outer (client, node_id); + } + // exponential backoff + join_wait *= 2; + info!( + "Waiting for {}s before retrying to join...", + join_wait.as_secs() + ); + tokio::time::sleep(join_wait).await; + }; + + let world_config = WorldConfig::default(); // TODO: make configurable + let (runtime, runtime_handle) = Runtime::start(world_config).await?; + let (store_tx, store_rx) = bounded(qsize()); + let store = Store::bootstrap(node_id, &addr, &self.db_dir, runtime.clone()).await?; + let network = Network::new(); + let (log_store, state_machine) = Adaptor::new(store.clone()); + let raft = Raft::new( + node_id, + self.raft_config.clone(), + network, + log_store, + state_machine, + ) + .await?; + let manager = Cluster::new(node_id, store_tx.clone(), raft.clone()); + *(runtime + .cluster_manager + .write() + .map_err(|_| "Failed to set world manager")?) = Some(manager); + let (api_worker_handle, server_state) = + api::initialize(node_id, addr, raft.clone(), store, store_tx, store_rx); + let running = Running::start( + self.clone(), + raft.clone(), + api_worker_handle, + server_state, + runtime, + runtime_handle, + ) + .await?; + + // only when the node is started (listens for HTTP, TCP etc) + // we can add it as learner and optionally promote it to a voter + + info!("Adding Node {node_id} as Learner..."); + let res = client.add_learner(&node_id).await?; + info!("Node {node_id} successully added as learner"); + if let Some(log_id) = res { + info!("Learner {node_id} has applied the log up to {log_id}."); + } + + if promote_to_voter { + info!("Promoting Node {node_id} to Voter..."); + client.promote_voter(&node_id).await?; + raft.wait(None) + .state( + openraft::ServerState::Follower, + "waiting for node to be Follower", + ) + .await?; + info!("Node {node_id} became Voter."); + } + Ok(running) + } + + /// Bootstrap and start this cluster node as a single node cluster + /// of which it immediately becomes the leader. + /// # Errors + /// if bootstrapping a a leader fails + pub async fn bootstrap_as_single_node_cluster(&mut self, addr: Addr) -> ClusterResult { + let node_id = crate::raft::NodeId::default(); + let world_config = WorldConfig::default(); // TODO: make configurable + let (runtime, runtime_handle) = Runtime::start(world_config).await?; + let (store_tx, store_rx) = bounded(qsize()); + + let store = Store::bootstrap(node_id, &addr, &self.db_dir, runtime.clone()).await?; + let network = Network::new(); + let (log_store, state_machine) = Adaptor::new(store.clone()); + let raft = Raft::new( + node_id, + self.raft_config.clone(), + network, + log_store, + state_machine, + ) + .await?; + let manager = Cluster::new(node_id, store_tx.clone(), raft.clone()); + *(runtime + .cluster_manager + .write() + .map_err(|_| "Failed to set world manager")?) = Some(manager); + let mut nodes = BTreeMap::new(); + nodes.insert(node_id, addr.clone()); + // this is the crucial bootstrapping step + raft.initialize(nodes).await?; + raft.wait(None) + .state( + openraft::ServerState::Leader, + "waiting for bootstrap node to become leader", + ) + .await?; + // lets make ourselves known to the cluster state as first operation, so new joiners will see us as well + // this is critical + match raft + .client_write(TremorRequest::Nodes(NodesRequest::AddNode { + addr: addr.clone(), + })) + .await + { + Ok(r) => { + let assigned_node_id = NodeId::try_from(r.data)?; + debug_assert_eq!(node_id, assigned_node_id, "Adding initial leader resulted in a differing node_id: {assigned_node_id}, expected: {node_id}"); + let (worker_handle, server_state) = + api::initialize(node_id, addr, raft.clone(), store, store_tx, store_rx); + + Running::start( + self.clone(), + raft, + worker_handle, + server_state, + runtime, + runtime_handle, + ) + .await + } + Err(e) => Err(ClusterError::Other(format!( + "Error adding myself to the bootstrapped cluster: {e}" + ))), + } + } +} diff --git a/src/raft/store.rs b/src/raft/store.rs new file mode 100644 index 0000000000..1715b4bf90 --- /dev/null +++ b/src/raft/store.rs @@ -0,0 +1,904 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod statemachine; + +pub(crate) use self::statemachine::apps::{FlowInstance, Instances, StateApp}; +use crate::{ + errors::Error as RuntimeError, + instance::IntendedState, + raft::{ + archive::TremorAppDef, + node::Addr, + store::statemachine::{SerializableTremorStateMachine, TremorStateMachine}, + ClusterError, NodeId, TremorRaftConfig, + }, + system::{flow::DeploymentType, Runtime}, +}; +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; +use openraft::{ + async_trait::async_trait, storage::Snapshot, AnyError, Entry, EntryPayload, ErrorSubject, + ErrorVerb, LogId, LogState, RaftLogReader, RaftSnapshotBuilder, RaftStorage, RaftTypeConfig, + SnapshotMeta, StorageError, StorageIOError, StoredMembership, Vote, +}; +use rocksdb::{ColumnFamily, Direction, FlushOptions, Options, DB}; +use serde::{Deserialize, Serialize}; +use simd_json::OwnedValue; +use std::{ + error::Error as StdError, + fmt::{Debug, Display, Formatter}, + io::Cursor, + ops::RangeBounds, + path::Path, + string::FromUtf8Error, + sync::{Arc, Mutex}, +}; +use tokio::sync::RwLock; +use tremor_common::alias; + +/// Kv Operation +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum KvRequest { + /// Set a key to the provided value in the cluster state + Set { key: String, value: Vec }, +} + +/// Operations on the nodes known to the cluster +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum NodesRequest { + /// Add the given node to the cluster and store its metadata (only addr for now) + /// This command should be committed before a learner is added to the cluster, so the leader can contact it via its `addr`. + AddNode { addr: Addr }, + /// Remove Node with the given `node_id` + /// This command should be committed after removing a learner from the cluster. + RemoveNode { node_id: NodeId }, +} + +/// Operations on apps and their instances +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum AppsRequest { + /// extract archive, parse sources, save sources in arena, put app into state machine + InstallApp { app: TremorAppDef, file: Vec }, + /// delete from statemachine, delete sources from arena + UninstallApp { + app: alias::App, + /// if `true`, stop all instances of this app + force: bool, + }, + /// Deploy and Start a flow of an installed app + Deploy { + app: alias::App, + flow: alias::FlowDefinition, + instance: alias::Flow, + config: std::collections::HashMap, + state: IntendedState, + deployment_type: DeploymentType, + }, + + /// Stopps and Undeploys an instance of a app + Undeploy(alias::Flow), + + /// Requests a instance state change + InstanceStateChange { + instance: alias::Flow, + state: IntendedState, + }, +} + +/** + * Here you will set the types of request that will interact with the raft nodes. + * For example the `Set` will be used to write data (key and value) to the raft database. + * The `AddNode` will append a new node to the current existing shared list of nodes. + * You will want to add any request that can write data in all nodes here. + */ +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TremorRequest { + /// KV operation + Kv(KvRequest), + Nodes(NodesRequest), + Apps(AppsRequest), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct TremorStart { + pub(crate) instance: alias::Flow, + pub(crate) config: std::collections::HashMap, + pub(crate) running: bool, + pub(crate) single_node: bool, +} +impl TremorStart { + pub(crate) fn state(&self) -> IntendedState { + if self.running { + IntendedState::Running + } else { + IntendedState::Paused + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Copy)] +pub enum TremorInstanceState { + /// Pauses a running instance + Pause, + /// Resumes a paused instance + Resume, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TremorSet { + pub key: String, + pub value: Vec, +} + +impl From for TremorRequest { + fn from(set: TremorSet) -> Self { + TremorRequest::Kv(KvRequest::Set { + key: set.key, + value: set.value, + }) + } +} + +/** + * Here you will defined what type of answer you expect from reading the data of a node. + * In this example it will return a optional value from a given key in + * the `ExampleRequest.Set`. + * + * TODO: `Should` we explain how to create multiple `AppDataResponse`? + * + */ +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub enum TremorResponse { + #[default] + None, + KvValue(Vec), + AppId(alias::App), + NodeId(NodeId), + AppFlowInstanceId(alias::Flow), +} + +impl TremorResponse { + pub(crate) fn into_kv_value(self) -> crate::Result> { + match self { + TremorResponse::KvValue(v) => Ok(v), + _ => Err(RuntimeError::from("Not a kv value")), + } + } +} + +impl TryFrom for alias::App { + type Error = RuntimeError; + fn try_from(response: TremorResponse) -> crate::Result { + match response { + TremorResponse::AppId(id) => Ok(id), + _ => Err(RuntimeError::from("Not an app id")), + } + } +} + +impl TryFrom for NodeId { + type Error = RuntimeError; + fn try_from(response: TremorResponse) -> crate::Result { + match response { + TremorResponse::NodeId(id) => Ok(id), + _ => Err(RuntimeError::from("Not a node id")), + } + } +} + +impl TryFrom for alias::Flow { + type Error = RuntimeError; + fn try_from(response: TremorResponse) -> crate::Result { + match response { + TremorResponse::AppFlowInstanceId(id) => Ok(id), + _ => Err(RuntimeError::from("Not an app flow instance id")), + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TremorSnapshot { + pub meta: SnapshotMeta, + + /// The data of the state machine at the time of this snapshot. + pub data: Vec, +} + +#[derive(Debug, Clone)] +pub struct Store { + /// The Raft state machine. + pub(crate) state_machine: Arc>, + // the database + db: Arc, +} + +type StorageResult = Result>; + +/// converts an id to a byte vector for storing in the database. +/// Note that we're using big endian encoding to ensure correct sorting of keys +fn id_to_bin(id: u64) -> Result, Error> { + let mut buf = Vec::with_capacity(8); + buf.write_u64::(id)?; + Ok(buf) +} + +fn bin_to_id(buf: &[u8]) -> Result { + Ok(buf + .get(0..8) + .ok_or_else(|| Error::Other(format!("Invalid buffer length: {}", buf.len()).into()))? + .read_u64::()?) +} + +#[derive(Debug)] +pub enum Error { + MissingCf(&'static str), + Utf8(FromUtf8Error), + StrUtf8(std::str::Utf8Error), + MsgPackEncode(rmp_serde::encode::Error), + MsgPackDecode(rmp_serde::decode::Error), + RocksDB(rocksdb::Error), + Io(std::io::Error), + Storage(openraft::StorageError), + // TODO: this is horrid, aaaaaahhhhh! + Tremor(Mutex), + TremorScript(Mutex), + MissingApp(alias::App), + MissingFlow(alias::App, alias::FlowDefinition), + MissingInstance(alias::Flow), + RunningInstances(alias::App), + NodeAlreadyAdded(crate::raft::NodeId), + Other(Box), +} + +impl From> for Error { + fn from(e: std::sync::PoisonError) -> Self { + Self::Other(Box::new(e)) + } +} + +impl From for Error { + fn from(e: FromUtf8Error) -> Self { + Error::Utf8(e) + } +} + +impl From for Error { + fn from(e: std::str::Utf8Error) -> Self { + Error::StrUtf8(e) + } +} + +impl From for Error { + fn from(e: rmp_serde::encode::Error) -> Self { + Error::MsgPackEncode(e) + } +} + +impl From for Error { + fn from(e: rmp_serde::decode::Error) -> Self { + Error::MsgPackDecode(e) + } +} +impl From for Error { + fn from(e: rocksdb::Error) -> Self { + Error::RocksDB(e) + } +} + +impl From for Error { + fn from(e: std::io::Error) -> Self { + Error::Io(e) + } +} + +impl From for Error { + fn from(e: crate::errors::Error) -> Self { + Error::Tremor(Mutex::new(e)) + } +} +impl From for Error { + fn from(e: tremor_script::errors::Error) -> Self { + Error::TremorScript(Mutex::new(e)) + } +} + +impl From> for Error { + fn from(e: StorageError) -> Self { + Error::Storage(e) + } +} + +impl StdError for Error {} +impl Display for Error { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Error::MissingCf(cf) => write!(f, "missing column family: `{cf}`"), + Error::Utf8(e) => write!(f, "invalid utf8: {e}"), + Error::StrUtf8(e) => write!(f, "invalid utf8: {e}"), + Error::MsgPackEncode(e) => write!(f, "invalid msgpack: {e}"), + Error::MsgPackDecode(e) => write!(f, "invalid msgpack: {e}"), + Error::RocksDB(e) => write!(f, "rocksdb error: {e}"), + Error::Io(e) => write!(f, "io error: {e}"), + Error::Tremor(e) => write!(f, "tremor error: {:?}", e.lock()), + Error::TremorScript(e) => write!(f, "tremor script error: {:?}", e.lock()), + Error::Other(e) => write!(f, "other error: {e}"), + Error::MissingApp(app) => write!(f, "missing app: {app}"), + Error::MissingFlow(app, flow) => write!(f, "missing flow: {app}::{flow}"), + Error::MissingInstance(instance) => { + write!(f, "missing instance: {instance}") + } + Error::Storage(e) => write!(f, "Storage: {e}"), + Error::NodeAlreadyAdded(node_id) => write!(f, "Node {node_id} already added"), + Error::RunningInstances(app_id) => { + write!(f, "App {app_id} still has running instances") + } + } + } +} + +fn store_w_err(e: impl StdError + 'static) -> StorageError { + StorageIOError::new(ErrorSubject::Store, ErrorVerb::Write, AnyError::new(&e)).into() +} +fn store_r_err(e: impl StdError + 'static) -> StorageError { + StorageIOError::new(ErrorSubject::Store, ErrorVerb::Read, AnyError::new(&e)).into() +} +fn logs_r_err(e: impl StdError + 'static) -> StorageError { + StorageIOError::new(ErrorSubject::Logs, ErrorVerb::Read, AnyError::new(&e)).into() +} +fn logs_w_err(e: impl StdError + 'static) -> StorageError { + StorageIOError::new(ErrorSubject::Logs, ErrorVerb::Read, AnyError::new(&e)).into() +} +fn snap_w_err( + meta: &SnapshotMeta, + e: impl StdError + 'static, +) -> StorageError { + StorageIOError::new( + ErrorSubject::Snapshot(Some(meta.signature())), + ErrorVerb::Write, + AnyError::new(&e), + ) + .into() +} + +impl From for StorageError { + fn from(e: Error) -> StorageError { + StorageIOError::new(ErrorSubject::Store, ErrorVerb::Read, AnyError::new(&e)).into() + } +} + +pub(crate) trait GetCfHandle { + fn cf(&self, cf: &'static str) -> Result<&ColumnFamily, Error>; + + /// store column family + fn cf_store(&self) -> Result<&ColumnFamily, Error> { + self.cf(Store::STORE) + } + + fn cf_self(&self) -> Result<&ColumnFamily, Error> { + self.cf(Store::SELF) + } + + /// logs columns family + fn cf_logs(&self) -> Result<&ColumnFamily, Error> { + self.cf(Store::LOGS) + } +} +impl GetCfHandle for rocksdb::DB { + fn cf(&self, cf: &'static str) -> Result<&ColumnFamily, Error> { + self.cf_handle(cf).ok_or(Error::MissingCf(cf)) + } +} + +impl Store { + pub(crate) fn flush(&self) -> Result<(), Error> { + self.db.flush_wal(true)?; + let mut opts = FlushOptions::default(); + opts.set_wait(true); + self.db.flush_opt(&opts)?; + Ok(()) + } + + fn put(&self, cf: &ColumnFamily, key: K, value: V) -> Result<(), Error> + where + K: AsRef<[u8]>, + V: AsRef<[u8]>, + { + self.db.put_cf(cf, key, value)?; + self.flush()?; + Ok(()) + } + + fn get_vote_(&self) -> StorageResult>> { + Ok(self + .db + .get_cf(self.db.cf_store()?, Self::VOTE) + .map_err(store_r_err)? + .and_then(|v| rmp_serde::from_slice(&v).ok())) + } + + fn set_vote_(&self, hard_state: &Vote) -> StorageResult<()> { + self.put( + self.db.cf_store()?, + Self::VOTE, + rmp_serde::to_vec(hard_state) + .map_err(Error::MsgPackEncode)? + .as_slice(), + ) + .map_err(store_w_err) + } + + fn get_last_purged_(&self) -> StorageResult>> { + Ok(self + .db + .get_cf(self.db.cf_store()?, Store::LAST_PURGED_LOG_ID) + .map_err(store_r_err)? + .and_then(|v| rmp_serde::from_slice(&v).ok())) + } + + fn set_last_purged_(&self, log_id: &LogId) -> StorageResult<()> { + self.put( + self.db.cf_store()?, + Self::LAST_PURGED_LOG_ID, + rmp_serde::to_vec(log_id) + .map_err(Error::MsgPackEncode)? + .as_slice(), + ) + .map_err(store_w_err) + } + + fn get_snapshot_index_(&self) -> StorageResult { + Ok(self + .db + .get_cf(self.db.cf_store()?, Self::SNAPSHOT_INDEX) + .map_err(store_r_err)? + .map(|v| rmp_serde::from_slice(&v).map_err(Error::MsgPackDecode)) + .transpose()? + .unwrap_or_default()) + } + + fn set_snapshot_index_(&self, snapshot_index: u64) -> StorageResult<()> { + self.put( + self.db.cf_store()?, + Self::SNAPSHOT_INDEX, + rmp_serde::to_vec(&snapshot_index) + .map_err(store_w_err)? + .as_slice(), + ) + .map_err(store_w_err)?; + Ok(()) + } + + fn get_current_snapshot_(&self) -> StorageResult> { + Ok(self + .db + .get_cf(self.db.cf_store()?, Self::SNAPSHOT) + .map_err(store_r_err)? + .and_then(|v| rmp_serde::from_slice(&v).ok())) + } + + fn set_current_snapshot_(&self, snap: &TremorSnapshot) -> StorageResult<()> { + self.put( + self.db.cf_store()?, + Self::SNAPSHOT, + rmp_serde::to_vec(&snap).map_err(|e| snap_w_err(&snap.meta, e))?, + ) + .map_err(|e| snap_w_err(&snap.meta, e))?; + Ok(()) + } +} + +#[async_trait] + +impl RaftLogReader for Store { + async fn get_log_state(&mut self) -> StorageResult> { + let last = self + .db + .iterator_cf(self.db.cf_logs()?, rocksdb::IteratorMode::End) + .next() + .and_then(|d| { + let (_, ent) = d.ok()?; + Some( + rmp_serde::from_slice::>(&ent) + .ok()? + .log_id, + ) + }); + + let last_purged_log_id = self.get_last_purged_()?; + + let last_log_id = match last { + None => last_purged_log_id, + Some(x) => Some(x), + }; + Ok(LogState { + last_purged_log_id, + last_log_id, + }) + } + + async fn try_get_log_entries + Clone + Debug + Send + Sync>( + &mut self, + range: RB, + ) -> StorageResult>> { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => id_to_bin(*x), + std::ops::Bound::Excluded(x) => id_to_bin(*x + 1), + std::ops::Bound::Unbounded => id_to_bin(0), + }?; + self.db + .iterator_cf( + self.db.cf_logs()?, + rocksdb::IteratorMode::From(&start, Direction::Forward), + ) + .map(|d| -> StorageResult<_> { + let (id, val) = d.map_err(store_r_err)?; + let entry: Entry<_> = rmp_serde::from_slice(&val).map_err(logs_r_err)?; + debug_assert_eq!(bin_to_id(&id)?, entry.log_id.index); + Ok(entry) + }) + .take_while(|r| { + r.as_ref() + .map(|e| range.contains(&e.log_id.index)) + .unwrap_or(false) + }) + .collect::>() + } +} +#[async_trait] + +impl RaftSnapshotBuilder for Store { + async fn build_snapshot(&mut self) -> StorageResult> { + let data; + let last_applied_log; + let last_membership; + + { + // Serialize the data of the state machine. + let state_machine = + SerializableTremorStateMachine::try_from(&*self.state_machine.read().await)?; + data = state_machine.to_vec()?; + + last_applied_log = state_machine.last_applied_log; + last_membership = state_machine.last_membership; + } + + // TODO: we probably want thius to be atomic. + let snapshot_idx: u64 = self.get_snapshot_index_()? + 1; + self.set_snapshot_index_(snapshot_idx)?; + + let snapshot_id = format!( + "{}-{}-{}", + last_applied_log.map(|x| x.leader_id).unwrap_or_default(), + last_applied_log.map_or(0, |l| l.index), + snapshot_idx + ); + + let meta = SnapshotMeta { + last_log_id: last_applied_log, + last_membership: last_membership.unwrap_or_default(), + snapshot_id, + }; + + let snapshot = TremorSnapshot { + meta: meta.clone(), + data: data.clone(), + }; + + self.set_current_snapshot_(&snapshot)?; + + Ok(Snapshot { + meta, + snapshot: Box::new(Cursor::new(data)), + }) + } +} +#[async_trait] +impl RaftStorage for Store { + type LogReader = Self; + type SnapshotBuilder = Self; + + async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder { + self.clone() + } + + async fn get_log_reader(&mut self) -> Self::LogReader { + self.clone() + } + + async fn save_vote( + &mut self, + vote: &Vote, + ) -> Result<(), StorageError> { + self.set_vote_(vote) + } + + async fn read_vote( + &mut self, + ) -> Result>, StorageError> { + self.get_vote_() + } + + // #[tracing::instrument(level = "trace", skip(self, entries))] + async fn append_to_log(&mut self, entries: I) -> StorageResult<()> + where + I: IntoIterator> + Send, + { + for entry in entries { + let id = id_to_bin(entry.log_id.index)?; + assert_eq!(bin_to_id(&id)?, entry.log_id.index); + self.put( + self.db.cf_logs()?, + &id, + rmp_serde::to_vec(&entry).map_err(logs_w_err)?, + ) + .map_err(logs_w_err)?; + } + Ok(()) + } + + // #[tracing::instrument(level = "debug", skip(self))] + async fn delete_conflict_logs_since( + &mut self, + log_id: LogId, + ) -> StorageResult<()> { + debug!("delete_conflict_logs_since: [{log_id}, +oo)"); + + let from = id_to_bin(log_id.index)?; + let to = id_to_bin(0xff_ff_ff_ff_ff_ff_ff_ff)?; + self.db + .delete_range_cf(self.db.cf_logs()?, &from, &to) + .map_err(logs_w_err) + } + + // #[tracing::instrument(level = "debug", skip(self))] + async fn purge_logs_upto(&mut self, log_id: LogId) -> StorageResult<()> { + debug!("purge_logs_upto: [0, {log_id}]"); + + self.set_last_purged_(&log_id)?; + let from = id_to_bin(0)?; + let to = id_to_bin(log_id.index + 1)?; + self.db + .delete_range_cf(self.db.cf_logs()?, &from, &to) + .map_err(logs_w_err) + } + + async fn last_applied_state( + &mut self, + ) -> StorageResult<( + std::option::Option>, + openraft::StoredMembership, + )> { + let state_machine = self.state_machine.read().await; + Ok(( + state_machine.get_last_applied_log()?, + state_machine.get_last_membership()?.unwrap_or_default(), + )) + } + + //#[tracing::instrument(level = "trace", skip(self, entries))] + /// apply committed entries to the state machine, start the operation encoded in the `TremorRequest` + async fn apply_to_state_machine( + &mut self, + entries: &[Entry], + ) -> StorageResult> { + //debug!("apply_to_state_machine {entries:?}"); + let mut result = Vec::with_capacity(entries.len()); + + let mut sm = self.state_machine.write().await; + + for entry in entries { + debug!("[{}] replicate to sm", entry.log_id); + + sm.set_last_applied_log(entry.log_id)?; + + match entry.payload { + EntryPayload::Blank => result.push(TremorResponse::None), + EntryPayload::Normal(ref request) => { + result.push(sm.handle_request(entry.log_id, request).await?); + } + EntryPayload::Membership(ref mem) => { + debug!("[{}] replicate membership to sm", entry.log_id); + sm.set_last_membership(&StoredMembership::new( + Some(entry.log_id), + mem.clone(), + ))?; + result.push(TremorResponse::None); + } + }; + } + self.db.flush_wal(true).map_err(logs_w_err)?; + Ok(result) + } + + // #[tracing::instrument(level = "trace", skip(self))] + async fn begin_receiving_snapshot( + &mut self, + ) -> StorageResult::SnapshotData>> { + Ok(Box::new(Cursor::new(Vec::new()))) + } + + // #[tracing::instrument(level = "trace", skip(self, snapshot))] + /// installs snapshot and applies all the deltas to statemachine and runtime + async fn install_snapshot( + &mut self, + meta: &SnapshotMeta, + snapshot: Box<::SnapshotData>, + ) -> StorageResult<()> { + info!( + "decoding snapshot for installation size: {} ", + snapshot.get_ref().len() + ); + + let new_snapshot = TremorSnapshot { + meta: meta.clone(), + data: snapshot.into_inner(), + }; + + // Update the state machine. + { + let updated_state_machine: SerializableTremorStateMachine = + rmp_serde::from_slice(&new_snapshot.data).map_err(|e| { + StorageIOError::new( + ErrorSubject::Snapshot(Some(new_snapshot.meta.signature())), + ErrorVerb::Read, + AnyError::new(&e), + ) + })?; + let mut state_machine = self.state_machine.write().await; + state_machine + .apply_diff_from_snapshot(updated_state_machine) + .await?; + self.set_current_snapshot_(&new_snapshot)?; + } + + Ok(()) + } + + // #[tracing::instrument(level = "trace", skip(self))] + async fn get_current_snapshot(&mut self) -> StorageResult>> { + match Store::get_current_snapshot_(self)? { + Some(snapshot) => { + let data = snapshot.data.clone(); + Ok(Some(Snapshot { + meta: snapshot.meta, + snapshot: Box::new(Cursor::new(data)), + })) + } + None => Ok(None), + } + } +} + +impl Store { + // db column families + + /// storing `node_id` and addr of the current node + const SELF: &'static str = "self"; + /// storing raft logs + const LOGS: &'static str = "logs"; + /// storing `RaftStorage` related stuff + const STORE: &'static str = "store"; + + const COLUMN_FAMILIES: [&'static str; 3] = [Self::SELF, Self::LOGS, Self::STORE]; + + // keys + const LAST_PURGED_LOG_ID: &'static str = "last_purged_log_id"; + const SNAPSHOT_INDEX: &'static str = "snapshot_index"; + const SNAPSHOT: &'static str = "snapshot"; + const VOTE: &'static str = "vote"; + /// for storing the own `node_id` + const NODE_ID: &'static str = "node_id"; + const NODE_ADDR: &'static str = "node_addr"; + + /// bootstrapping constructor - storing the given node data in the db + pub(crate) async fn bootstrap>( + node_id: crate::raft::NodeId, + addr: &Addr, + db_path: P, + world: Runtime, + ) -> Result { + let db = Self::init_db(db_path)?; + Self::set_self(&db, node_id, addr)?; + + let db = Arc::new(db); + let state_machine = Arc::new(RwLock::new( + TremorStateMachine::new(db.clone(), world) + .await + .map_err(Error::from)?, + )); + Ok(Self { state_machine, db }) + } + + /// Initialize the database + /// + /// This function is safe and never cleans up or resets the current state, + /// but creates a new db if there is none. + pub(crate) fn init_db>(db_path: P) -> Result { + let mut db_opts = Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + + let cfs = TremorStateMachine::column_families().chain(Self::COLUMN_FAMILIES); + let db = DB::open_cf(&db_opts, db_path, cfs).map_err(ClusterError::Rocks)?; + Ok(db) + } + + /// loading constructor - loading the given database + pub(crate) async fn load(db: Arc, world: Runtime) -> Result { + let state_machine = Arc::new(RwLock::new( + TremorStateMachine::new(db.clone(), world.clone()) + .await + .map_err(Error::from)?, + )); + let this = Self { state_machine, db }; + Ok(this) + } + + /// Store the information about the current node itself in the `db` + fn set_self(db: &DB, node_id: crate::raft::NodeId, addr: &Addr) -> Result<(), ClusterError> { + let node_id_bytes = id_to_bin(node_id)?; + let addr_bytes = rmp_serde::to_vec(addr)?; + let cf = db.cf_self()?; + db.put_cf(cf, Store::NODE_ID, node_id_bytes)?; + db.put_cf(cf, Store::NODE_ADDR, addr_bytes)?; + Ok(()) + } + + pub(crate) fn get_self(db: &DB) -> Result<(crate::raft::NodeId, Addr), ClusterError> { + let id = Self::get_self_node_id(db)?.ok_or("invalid cluster store, node_id missing")?; + let addr = Self::get_self_addr(db)?.ok_or("invalid cluster store, node_addr missing")?; + + Ok((id, addr)) + } + + /// # Errors + /// if the store fails to read the RPC address + pub fn get_self_addr(db: &DB) -> Result, Error> { + Ok(db + .get_cf(db.cf_self()?, Store::NODE_ADDR)? + .map(|v| rmp_serde::from_slice(&v)) + .transpose()?) + } + + /// # Errors + /// if the store fails to read the node id + pub fn get_self_node_id(db: &DB) -> Result, Error> { + db.get_cf(db.cf_self()?, Store::NODE_ID)? + .map(|v| bin_to_id(&v)) + .transpose() + } +} + +#[cfg(test)] +mod tests { + use crate::raft::ClusterResult; + + use super::*; + + #[test] + fn init_db_is_idempotent() -> ClusterResult<()> { + let dir = tempfile::tempdir()?; + let db = Store::init_db(dir.path())?; + let handle = db.cf_handle(Store::STORE).ok_or("no data")?; + let data = vec![1_u8, 2_u8, 3_u8]; + db.put_cf(handle, "node_id", data.clone())?; + drop(db); + + let db2 = Store::init_db(dir.path())?; + let handle2 = db2.cf_handle(Store::STORE).ok_or("no data")?; + let res2 = db2.get_cf(handle2, "node_id")?; + assert_eq!(Some(data), res2); + Ok(()) + } +} diff --git a/src/raft/store/statemachine.rs b/src/raft/store/statemachine.rs new file mode 100644 index 0000000000..3be7fb2f3a --- /dev/null +++ b/src/raft/store/statemachine.rs @@ -0,0 +1,268 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + raft::store::{ + self, statemachine::nodes::NodesStateMachine, StorageResult, TremorRequest, TremorResponse, + }, + system::Runtime, +}; +use openraft::{ + AnyError, ErrorSubject, ErrorVerb, LogId, StorageError, StorageIOError, StoredMembership, +}; +use rocksdb::ColumnFamily; +use serde::{Deserialize, Serialize}; +use std::{error::Error, fmt::Debug, sync::Arc}; + +// apps related state machine +pub(crate) mod apps; +// nodes related state machine +mod nodes; +// kv state machine +mod kv; + +fn sm_r_err(e: E) -> StorageError { + StorageIOError::new( + ErrorSubject::StateMachine, + ErrorVerb::Read, + AnyError::new(&e), + ) + .into() +} +fn sm_w_err(e: E) -> StorageError { + StorageIOError::new( + ErrorSubject::StateMachine, + ErrorVerb::Write, + AnyError::new(&e), + ) + .into() +} +fn sm_d_err(e: E) -> StorageError { + StorageIOError::new( + ErrorSubject::StateMachine, + ErrorVerb::Delete, + AnyError::new(&e), + ) + .into() +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct SerializableTremorStateMachine { + pub last_applied_log: Option>, + + pub last_membership: Option>, + + /// Application data, for the k/v store + pub(crate) kv: kv::KvSnapshot, + + pub(crate) apps: apps::AppsSnapshot, + + /// nodes known to the cluster (learners and voters) + /// necessary for establishing a network connection to them + pub(crate) nodes: nodes::NodesSnapshot, +} + +impl SerializableTremorStateMachine { + pub(crate) fn to_vec(&self) -> StorageResult> { + // FIXME: serialize better? + rmp_serde::to_vec(&self).map_err(sm_r_err) + } +} + +impl TryFrom<&TremorStateMachine> for SerializableTremorStateMachine { + type Error = StorageError; + + fn try_from(state: &TremorStateMachine) -> Result { + let nodes = state.nodes.as_snapshot()?; + let kv = state.kv.as_snapshot()?; + let apps = state.apps.as_snapshot()?; + let last_membership = state.get_last_membership()?; + let last_applied_log = state.get_last_applied_log()?; + Ok(Self { + last_applied_log, + last_membership, + kv, + apps, + nodes, + }) + } +} + +/// abstract raft statemachine for implementing sub-statemachines +#[async_trait::async_trait] +trait RaftStateMachine, Cmd> { + async fn load(db: &Arc, world: &Runtime) -> Result + where + Self: std::marker::Sized; + async fn apply_diff_from_snapshot(&mut self, snapshot: &Ser) -> StorageResult<()>; + fn as_snapshot(&self) -> StorageResult; + async fn transition(&mut self, cmd: &Cmd) -> StorageResult; + fn column_families() -> &'static [&'static str]; +} + +#[derive(Debug, Clone)] +pub(crate) struct TremorStateMachine { + /// sub-statemachine for nodes known to the cluster + pub(crate) nodes: nodes::NodesStateMachine, + pub(crate) kv: kv::KvStateMachine, + pub(crate) apps: apps::AppsStateMachine, + pub db: Arc, +} + +/// DB Helpers +impl TremorStateMachine { + /// storing state machine related stuff + const CF: &'static str = "state_machine"; + + const LAST_MEMBERSHIP: &'static str = "last_membership"; + const LAST_APPLIED_LOG: &'static str = "last_applied_log"; + + /// state machine column family + fn cf_state_machine(&self) -> StorageResult<&ColumnFamily> { + self.db + .cf_handle(Self::CF) + .ok_or(store::Error::MissingCf(Self::CF)) + .map_err(sm_w_err) + } + + pub(super) fn column_families() -> impl Iterator { + let iter = nodes::NodesStateMachine::column_families() + .iter() + .copied() + .chain( + kv::KvStateMachine::column_families().iter().copied().chain( + apps::AppsStateMachine::column_families() + .iter() + .copied() + .chain([Self::CF]), + ), + ); + iter + } +} + +/// Core impl +impl TremorStateMachine { + pub(crate) async fn new( + db: Arc, + world: Runtime, + ) -> Result { + let nodes = NodesStateMachine::load(&db, &world).await?; + let kv = kv::KvStateMachine::load(&db, &world).await?; + let apps = apps::AppsStateMachine::load(&db, &world).await?; + Ok(Self { + db: db.clone(), + nodes, + kv, + apps, + }) + } + + pub(crate) fn get_last_membership( + &self, + ) -> StorageResult>> { + self.db + .get_cf(self.cf_state_machine()?, Self::LAST_MEMBERSHIP) + .map_err(sm_r_err) + .and_then(|value| { + value + .map(|v| rmp_serde::from_slice(&v).map_err(sm_r_err)) + .transpose() + }) + } + + pub(crate) fn set_last_membership( + &self, + membership: &StoredMembership, + ) -> StorageResult<()> { + self.db + .put_cf( + self.cf_state_machine()?, + Self::LAST_MEMBERSHIP, + rmp_serde::to_vec(&membership).map_err(sm_w_err)?, + ) + .map_err(sm_w_err) + } + + pub(crate) fn get_last_applied_log(&self) -> StorageResult>> { + self.db + .get_cf(self.cf_state_machine()?, Self::LAST_APPLIED_LOG) + .map_err(sm_r_err) + .and_then(|value| { + value + .map(|v| rmp_serde::from_slice(&v).map_err(sm_r_err)) + .transpose() + }) + } + + pub(crate) fn set_last_applied_log( + &self, + log_id: LogId, + ) -> StorageResult<()> { + self.db + .put_cf( + self.cf_state_machine()?, + Self::LAST_APPLIED_LOG, + rmp_serde::to_vec(&log_id).map_err(sm_w_err)?, + ) + .map_err(sm_w_err) + } + + fn delete_last_applied_log(&self) -> StorageResult<()> { + self.db + .delete_cf(self.cf_state_machine()?, Self::LAST_APPLIED_LOG) + .map_err(sm_d_err) + } + + // FIXME: reason about error handling and avoid leaving the state machine in an inconsistent state + pub(crate) async fn apply_diff_from_snapshot( + &mut self, + snapshot: SerializableTremorStateMachine, + ) -> StorageResult<()> { + // apply snapshot on the nodes state machine + self.nodes.apply_diff_from_snapshot(&snapshot.nodes).await?; + self.kv.apply_diff_from_snapshot(&snapshot.kv).await?; + self.apps.apply_diff_from_snapshot(&snapshot.apps).await?; + + // TODO: delete every key that is not in the snapshot - not necessarily needed today as we don't have a DELETE op on our k/v store + + if let Some(log_id) = snapshot.last_applied_log { + self.set_last_applied_log(log_id)?; + } else { + self.delete_last_applied_log()?; + } + + if let Some(last_membership) = &snapshot.last_membership { + self.set_last_membership(last_membership)?; + } + + Ok(()) + } + + pub(crate) async fn handle_request( + &mut self, + log_id: LogId, + req: &TremorRequest, + ) -> StorageResult { + match req { + TremorRequest::Kv(kv) => { + debug!("[{log_id}] set"); + self.kv.transition(kv).await + } + // forward to nodes statemachine + TremorRequest::Nodes(nodes_req) => self.nodes.transition(nodes_req).await, + TremorRequest::Apps(apps_req) => self.apps.transition(apps_req).await, + } + } +} diff --git a/src/raft/store/statemachine/apps.rs b/src/raft/store/statemachine/apps.rs new file mode 100644 index 0000000000..f77c5184af --- /dev/null +++ b/src/raft/store/statemachine/apps.rs @@ -0,0 +1,506 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + instance::IntendedState, + raft::{ + archive::{extract, get_app, TremorAppDef}, + store::{ + self, + statemachine::{sm_d_err, sm_r_err, sm_w_err, RaftStateMachine}, + store_w_err, AppsRequest, StorageResult, TremorResponse, + }, + }, + system::{flow::DeploymentType, Runtime}, +}; +use rocksdb::ColumnFamily; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tremor_common::alias; +use tremor_script::{ + arena::{self, Arena}, + ast::{ + optimizer::Optimizer, visitors::ArgsRewriter, walkers::DeployWalker, CreationalWith, + DeployFlow, FlowDefinition, Helper, Ident, ImutExpr, WithExprs, + }, + deploy::Deploy, + module::GetMod, + prelude::BaseExpr, + AggrRegistry, FN_REGISTRY, +}; + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct FlowInstance { + /// Identifier of the instance + pub id: alias::Flow, + /// the id of the flow definition this instance is based upon + pub definition: alias::FlowDefinition, + pub config: HashMap, + pub state: IntendedState, + pub deployment_type: DeploymentType, +} +pub type Instances = HashMap; + +#[derive(Debug, Clone)] +pub struct StateApp { + pub app: TremorAppDef, + pub instances: Instances, + /// we keep the arena indices around, so we can safely delete its contents + arena_indices: Vec, + main: Deploy, +} + +#[derive(Clone, Debug)] +pub(crate) struct AppsStateMachine { + db: Arc, + apps: HashMap, + world: Runtime, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct AppsSnapshot { + /// App definition archives + archives: Vec>, + + /// Instances and their desired state + instances: HashMap, +} + +#[async_trait::async_trait] +impl RaftStateMachine for AppsStateMachine { + async fn load(db: &Arc, world: &Runtime) -> Result + where + Self: std::marker::Sized, + { + let mut me = Self { + db: db.clone(), + apps: HashMap::new(), + world: world.clone(), + }; + // load apps + for kv in db.iterator_cf(Self::cf_apps(db)?, rocksdb::IteratorMode::Start) { + let (_, archive) = kv?; + me.load_archive(&archive) + .map_err(|e| store::Error::Other(Box::new(e)))?; + } + + // load instances + let instances = db + .iterator_cf(Self::cf_instances(db)?, rocksdb::IteratorMode::Start) + .map(|kv| { + let (app_id, instances) = kv?; + let app_id = String::from_utf8(app_id.to_vec())?; + let instances: Instances = rmp_serde::from_slice(&instances)?; + Ok((alias::App(app_id), instances)) + }) + .collect::, store::Error>>()?; + + // start instances and put them into state machine state + for (app_id, app_instances) in instances { + for ( + _, + FlowInstance { + id, + definition, + config, + state, + deployment_type, + }, + ) in app_instances + { + me.deploy_flow(&app_id, definition, id, config, state, deployment_type) + .await + .map_err(store::Error::Storage)?; + } + } + Ok(me) + } + + async fn apply_diff_from_snapshot(&mut self, snapshot: &AppsSnapshot) -> StorageResult<()> { + // load archives + let mut snapshot_apps = HashSet::with_capacity(snapshot.archives.len()); + for archive in &snapshot.archives { + let app_def = get_app(archive).map_err(sm_r_err)?; + snapshot_apps.insert(app_def.name().clone()); + if let Some(existing_app) = self.apps.get(&app_def.name) { + // this is by no means secure or anything (archive contents can be forged), just a cheap way to compare for bytewise identity + if app_def.sha256 != existing_app.app.sha256 { + info!( + "App definition '{}' changed. We need to restart all instances.", + &app_def.name + ); + // this will stop all instances and then delete the app + self.uninstall_app(app_def.name(), true).await?; + info!("Reloading changed app '{}'", &app_def.name); + self.load_archive(archive)?; + } + } else { + info!("Loading app '{}'", &app_def.name); + self.load_archive(archive)?; + } + } + // load instances, app by app + let app_ids = self.apps.keys().cloned().collect::>(); + for app_id in &app_ids { + if let Some(snapshot_instances) = snapshot.instances.get(app_id) { + let instances = self + .apps + .get(app_id) + .map(|app| app.instances.clone()) + .expect("Dang, we just put this in the map, where is it gone?"); + for (instance_id, flow) in &instances { + if let Some(s_flow) = snapshot_instances.get(instance_id) { + // redeploy existing instance with different config + if s_flow.config != flow.config { + info!("Flow instance {app_id}/{instance_id} with parameters differ, redeploying..."); + self.stop_and_remove_flow(&s_flow.id).await?; + self.deploy_flow( + app_id, + s_flow.definition.clone(), + s_flow.id.clone(), + s_flow.config.clone(), // important: this is the new config + s_flow.state, + s_flow.deployment_type, + ) + .await?; + } else if s_flow.state != flow.state { + // same flow, same config, different state - just change state + + self.change_flow_state( + &alias::Flow::new(app_id.clone(), instance_id), + s_flow.state, + ) + .await + .map_err(sm_w_err)?; + } + } else { + // stop and remove instances that are not in the snapshot + self.stop_and_remove_flow(&alias::Flow::new(app_id.clone(), instance_id)) + .await?; + } + } + // deploy instances that are not in self + for (s_instance_id, s_flow) in snapshot_instances { + let flow_id = alias::Flow::new(app_id.clone(), s_instance_id); + if !instances.contains_key(flow_id.instance_id()) { + self.deploy_flow( + app_id, + s_flow.definition.clone(), + flow_id, + s_flow.config.clone(), + s_flow.state, + s_flow.deployment_type, + ) + .await?; + } + } + } else { + // uninstall apps that are not in the snapshot + self.uninstall_app(app_id, true).await?; + } + } + Ok(()) + } + + fn as_snapshot(&self) -> StorageResult { + let archives = self + .db + .iterator_cf(Self::cf_apps(&self.db)?, rocksdb::IteratorMode::Start) + .map(|kv| kv.map(|(_, tar)| tar.to_vec())) + .collect::>, _>>() + .map_err(sm_r_err)?; + let instances = self + .apps + .iter() + .map(|(k, v)| (k.clone(), v.instances.clone())) + .collect(); + Ok(AppsSnapshot { + archives, + instances, + }) + } + + async fn transition(&mut self, cmd: &AppsRequest) -> StorageResult { + match cmd { + AppsRequest::InstallApp { app, file } => { + self.load_archive(file)?; + Ok(TremorResponse::AppId(app.name().clone())) + } + AppsRequest::UninstallApp { app, force } => { + self.uninstall_app(app, *force).await?; + Ok(TremorResponse::AppId(app.clone())) + } + AppsRequest::Deploy { + app, + flow, + instance, + config, + state, + deployment_type, + } => { + self.deploy_flow( + app, + flow.clone(), + instance.clone(), + config.clone(), + *state, + *deployment_type, + ) + .await?; + Ok(TremorResponse::AppFlowInstanceId(instance.clone())) + } + AppsRequest::Undeploy(instance) => { + self.stop_and_remove_flow(instance).await?; + Ok(TremorResponse::AppFlowInstanceId(instance.clone())) + } + AppsRequest::InstanceStateChange { instance, state } => { + self.change_flow_state(instance, *state).await?; + Ok(TremorResponse::AppFlowInstanceId(instance.clone())) + } + } + } + + fn column_families() -> &'static [&'static str] { + &Self::COLUMN_FAMILIES + } +} + +impl AppsStateMachine { + const CF_APPS: &str = "apps"; + const CF_INSTANCES: &str = "instances"; + + const COLUMN_FAMILIES: [&'static str; 2] = [Self::CF_APPS, Self::CF_INSTANCES]; + + fn cf_apps(db: &Arc) -> Result<&ColumnFamily, store::Error> { + db.cf_handle(Self::CF_APPS) + .ok_or(store::Error::MissingCf(Self::CF_APPS)) + } + fn cf_instances(db: &Arc) -> Result<&ColumnFamily, store::Error> { + db.cf_handle(Self::CF_INSTANCES) + .ok_or(store::Error::MissingCf(Self::CF_INSTANCES)) + } + + /// Load app definition and `Deploy` AST from the tar-archive in `archive` + /// and store the definition in the db and state machine + fn load_archive(&mut self, archive: &[u8]) -> StorageResult<()> { + let (app, main, arena_indices) = extract(archive).map_err(store_w_err)?; + + info!("Loading Archive for app: {}", app.name()); + + self.db + .put_cf( + Self::cf_apps(&self.db).map_err(sm_r_err)?, + app.name().0.as_bytes(), + archive, + ) + .map_err(store_w_err)?; + + let app = StateApp { + app, + main, + arena_indices, + instances: HashMap::new(), + }; + self.apps.insert(app.app.name().clone(), app); + + Ok(()) + } + + /// Deploy flow instance and transition it to the state we want + /// Also store the instance into the state machine + async fn deploy_flow( + &mut self, + app_id: &alias::App, + flow: alias::FlowDefinition, + instance: alias::Flow, + config: HashMap, + intended_state: IntendedState, + deployment_type: DeploymentType, + ) -> StorageResult<()> { + info!("Deploying flow instance {app_id}/{flow}/{instance}"); + let app = self + .apps + .get_mut(app_id) + .ok_or_else(|| store::Error::MissingApp(app_id.clone()))?; + + let mut defn: FlowDefinition = app + .main + .deploy + .scope + .content + .get(&flow.0) + .ok_or_else(|| store::Error::MissingFlow(app_id.clone(), flow.clone()))?; + let mid = defn.meta().clone(); + + defn.params + .ingest_creational_with(&CreationalWith { + with: WithExprs( + config + .iter() + .map(|(k, v)| { + ( + Ident::new(k.to_string().into(), Box::new(mid.clone())), + ImutExpr::literal(Box::new(mid.clone()), v.clone().into()), + ) + }) + .collect(), + ), + mid: Box::new(mid), + }) + .map_err(store::Error::from)?; + + let fake_aggr_reg = AggrRegistry::default(); + { + let reg = &*FN_REGISTRY.read().map_err(store_w_err)?; + let mut helper = Helper::new(reg, &fake_aggr_reg); + Optimizer::new(&helper) + .walk_flow_definition(&mut defn) + .map_err(store::Error::from)?; + + let inner_args = defn.params.render().map_err(store::Error::from)?; + + ArgsRewriter::new(inner_args, &mut helper, defn.params.meta()) + .walk_flow_definition(&mut defn) + .map_err(store::Error::from)?; + Optimizer::new(&helper) + .walk_flow_definition(&mut defn) + .map_err(store::Error::from)?; + } + + let mid = Box::new(defn.meta().clone()); + let deploy = DeployFlow { + mid: mid.clone(), + from_target: tremor_script::ast::NodeId::new( + flow.0.clone(), + vec![app_id.0.clone()], + mid, + ), + instance_alias: instance.instance_id().to_string(), + defn, + docs: None, + }; + app.instances.insert( + instance.instance_id().clone(), + FlowInstance { + id: instance.clone(), + definition: flow, + config, + state: intended_state, // we are about to apply this state further below + deployment_type, + }, + ); + let instances = rmp_serde::to_vec(&app.instances).map_err(sm_w_err)?; + + self.db + .put_cf( + Self::cf_instances(&self.db)?, + app_id.0.as_bytes(), + &instances, + ) + .map_err(store_w_err)?; + + // deploy the flow but don't start it yet + // ensure the cluster is running + self.world.wait_for_cluster().await; + self.world + .deploy_flow(app_id.clone(), &deploy, deployment_type) + .await + .map_err(sm_w_err)?; + // change the flow state to the intended state + self.world + .change_flow_state(instance, intended_state) + .await + .map_err(sm_w_err)?; + Ok(()) + } + + async fn stop_and_remove_flow(&mut self, instance_id: &alias::Flow) -> StorageResult<()> { + info!("Stop and remove flow {instance_id}"); + if let Some(app) = self.apps.get_mut(instance_id.app_id()) { + if app.instances.get(instance_id.instance_id()).is_some() { + self.world + .stop_flow(instance_id.clone()) + .await + .map_err(sm_d_err)?; + app.instances.remove(instance_id.instance_id()); + } + } + Ok(()) + } + + async fn uninstall_app(&mut self, app_id: &alias::App, force: bool) -> StorageResult<()> { + info!("Uninstall app: {app_id}"); + if let Some(app) = self.apps.remove(app_id) { + if !app.instances.is_empty() && !force { + // error out, we have running instances, which need to be stopped first + return Err(sm_d_err(store::Error::RunningInstances(app_id.clone()))); + } + // stop instances then delete the app + for (instance_id, _instance) in app.instances { + let flow_instance_id = alias::Flow::new(app.app.name().clone(), instance_id); + self.world + .stop_flow(flow_instance_id) + .await + .map_err(sm_d_err)?; + } + self.db + .delete_cf(Self::cf_apps(&self.db)?, app.app.name().0.as_bytes()) + .map_err(store_w_err)?; + // delete from arena + for aid in app.arena_indices { + // ALLOW: we have stopped all instances, so nothing referencing those arena contents should be alive anymore (fingers crossed) + unsafe { + Arena::delete_index_this_is_really_unsafe_dont_use_it(aid).map_err(sm_d_err)?; + } + } + } + Ok(()) + } + + async fn change_flow_state( + &mut self, + instance_id: &alias::Flow, + intended_state: IntendedState, + ) -> StorageResult<()> { + info!("Change flow state {instance_id} to {intended_state}"); + let app = self + .apps + .get_mut(instance_id.app_id()) + .ok_or_else(|| store::Error::MissingApp(instance_id.app_id().clone()))?; + let instance = app + .instances + .get_mut(instance_id.instance_id()) + .ok_or_else(|| store::Error::MissingInstance(instance_id.clone()))?; + // set the intended state in our state machine + instance.state = intended_state; + // ... and attempt to bring the flow instance in the runtime in the desired state + self.world + .change_flow_state(instance_id.clone(), intended_state) + .await + .map_err(sm_w_err)?; + Ok(()) + } +} + +impl AppsStateMachine { + pub(crate) fn get_app(&self, app_id: &alias::App) -> Option<&StateApp> { + self.apps.get(app_id) + } + + pub(crate) fn list(&self) -> impl Iterator { + self.apps.iter() + } +} diff --git a/src/raft/store/statemachine/kv.rs b/src/raft/store/statemachine/kv.rs new file mode 100644 index 0000000000..6043d519d8 --- /dev/null +++ b/src/raft/store/statemachine/kv.rs @@ -0,0 +1,118 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! KV raft sub-statemachine +//! storing key value stores in rocksdb only + +use crate::{ + raft::{ + store::{ + self, + statemachine::{sm_r_err, sm_w_err, RaftStateMachine}, + store_r_err, store_w_err, Error as StoreError, KvRequest, StorageResult, + TremorResponse, + }, + NodeId, + }, + system::Runtime, +}; +use openraft::StorageError; +use rocksdb::ColumnFamily; +use std::{collections::BTreeMap, marker::Sized, sync::Arc}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct KvSnapshot(BTreeMap); + +#[derive(Clone, Debug)] +pub(crate) struct KvStateMachine { + db: Arc, +} + +impl KvStateMachine { + const CF: &str = "data"; + const COLUMN_FAMILIES: [&'static str; 1] = [Self::CF]; + + fn cf(db: &Arc) -> Result<&ColumnFamily, StoreError> { + db.cf_handle(Self::CF) + .ok_or(StoreError::MissingCf(Self::CF)) + } + + /// Store `value` at `key` in the distributed KV store + fn insert(&self, key: &str, value: &[u8]) -> StorageResult<()> { + self.db + .put_cf(Self::cf(&self.db)?, key.as_bytes(), value) + .map_err(store_w_err) + } + + /// try to obtain the value at the given `key`. + /// Returns `Ok(None)` if there is no value for that key. + pub(crate) fn get(&self, key: &str) -> StorageResult>> { + let key = key.as_bytes(); + self.db + .get_cf(Self::cf(&self.db)?, key) + .map_err(store_r_err) + } +} + +#[async_trait::async_trait] +impl RaftStateMachine for KvStateMachine { + async fn load(db: &Arc, _world: &Runtime) -> Result + where + Self: Sized, + { + Ok(Self { db: db.clone() }) + } + + async fn apply_diff_from_snapshot(&mut self, snapshot: &KvSnapshot) -> StorageResult<()> { + for (key, value) in &snapshot.0 { + self.db + .put_cf(Self::cf(&self.db)?, key.as_bytes(), value.as_bytes()) + .map_err(sm_w_err)?; + } + Ok(()) + } + + fn as_snapshot(&self) -> StorageResult { + let data = self + .db + .iterator_cf( + self.db + .cf_handle(Self::CF) + .ok_or(store::Error::MissingCf(Self::CF))?, + rocksdb::IteratorMode::Start, + ) + .map(|kv| { + let (key, value) = kv.map_err(sm_r_err)?; + Ok(( + String::from_utf8(key.to_vec()).map_err(sm_r_err)?, + String::from_utf8(value.to_vec()).map_err(sm_r_err)?, + )) + }) + .collect::, StorageError>>()?; + Ok(KvSnapshot(data)) + } + + async fn transition(&mut self, cmd: &KvRequest) -> StorageResult { + match cmd { + KvRequest::Set { key, value } => { + self.insert(key, value)?; + Ok(TremorResponse::KvValue(value.clone())) + } + } + } + + fn column_families() -> &'static [&'static str] { + &Self::COLUMN_FAMILIES + } +} diff --git a/src/raft/store/statemachine/nodes.rs b/src/raft/store/statemachine/nodes.rs new file mode 100644 index 0000000000..7467102f3b --- /dev/null +++ b/src/raft/store/statemachine/nodes.rs @@ -0,0 +1,207 @@ +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! nodes raft sub-statemachine +//! handling all the nodes that are known to the cluster and is responsible for assigning node ids + +use crate::{ + raft::{ + node::Addr, + store::{ + bin_to_id, id_to_bin, + statemachine::{sm_d_err, sm_w_err, RaftStateMachine}, + store_w_err, Error as StoreError, NodesRequest, StorageResult, TremorResponse, + }, + }, + system::Runtime, +}; +use rocksdb::ColumnFamily; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; + +#[derive(Debug, Clone)] +pub(crate) struct NodesStateMachine { + db: Arc, + next_node_id: Arc, + known_nodes: HashMap, +} + +impl NodesStateMachine { + const CF: &str = "nodes"; + const NEXT_NODE_ID: &str = "next_node_id"; + const COLUMN_FAMILIES: [&'static str; 1] = [Self::CF]; + + fn cf(db: &Arc) -> Result<&ColumnFamily, StoreError> { + db.cf_handle(Self::CF) + .ok_or(StoreError::MissingCf(Self::CF)) + } +} + +#[async_trait::async_trait] +impl RaftStateMachine for NodesStateMachine { + async fn load(db: &Arc, _world: &Runtime) -> Result + where + Self: std::marker::Sized, + { + // load known nodes + let next_node_id = + if let Some(next_node_id) = db.get_cf(Self::cf(db)?, Self::NEXT_NODE_ID)? { + bin_to_id(next_node_id.as_slice())? + } else { + debug!("No next_node_id stored in db, starting from 0"); + 0 + }; + let known_nodes = db + .iterator_cf(Self::cf(db)?, rocksdb::IteratorMode::Start) + .filter(|x| { + // filter out NEXT_NODE_ID + if let Ok((key_raw, _)) = x { + key_raw.as_ref() != Self::NEXT_NODE_ID.as_bytes() + } else { + true + } + }) + .map(|x| { + let (key_raw, value_raw) = x?; + let node_id = bin_to_id(&key_raw)?; + let addr: Addr = rmp_serde::from_slice(&value_raw)?; + Ok((node_id, addr)) + }) + .collect::, StoreError>>()?; + + Ok(Self { + db: db.clone(), + next_node_id: Arc::new(AtomicU64::new(next_node_id)), + known_nodes, + }) + } + + async fn apply_diff_from_snapshot(&mut self, snapshot: &NodesSnapshot) -> StorageResult<()> { + // overwrite the current next_node_id + self.set_next_node_id(snapshot.next_node_id)?; + + // overwrite the nodes in the db and the local map + // TODO: should we remove all nodes not in the snapshot? + for (node_id, addr) in &snapshot.nodes { + self.store_node(*node_id, addr)?; + } + Ok(()) + } + + fn as_snapshot(&self) -> StorageResult { + Ok(NodesSnapshot { + next_node_id: self.next_node_id.load(Ordering::SeqCst), + nodes: self.known_nodes.clone(), + }) + } + + async fn transition(&mut self, cmd: &NodesRequest) -> StorageResult { + let node_id = match cmd { + NodesRequest::AddNode { addr } => self.add_node(addr)?, + NodesRequest::RemoveNode { node_id } => { + self.remove_node(*node_id)?; + *node_id + } + }; + + Ok(TremorResponse::NodeId(node_id)) + } + fn column_families() -> &'static [&'static str] { + &Self::COLUMN_FAMILIES + } +} + +impl NodesStateMachine { + fn next_node_id(&self) -> StorageResult { + let s = self.next_node_id.fetch_add(1, Ordering::SeqCst); + let s_bytes = id_to_bin(s)?; + self.db + .put_cf(Self::cf(&self.db)?, Self::NEXT_NODE_ID, s_bytes) + .map_err(sm_w_err)?; + Ok(s) + } + + fn set_next_node_id(&mut self, next_node_id: u64) -> StorageResult { + let bytes = id_to_bin(next_node_id)?; + self.db + .put_cf(Self::cf(&self.db)?, Self::NEXT_NODE_ID, bytes) + .map_err(sm_w_err)?; + let res = self.next_node_id.swap(next_node_id, Ordering::SeqCst); + Ok(res) + } + + /// find a `NodeId` for the given `addr` if it is already stored + pub(crate) fn find_node_id(&self, addr: &Addr) -> Option<&crate::raft::NodeId> { + self.known_nodes + .iter() + .find(|(_node_id, existing_addr)| *existing_addr == addr) + .map(|(node_id, _)| node_id) + } + + /// get the `Addr` of the node identified by `node_id` if it is stored + pub(crate) fn get_node(&self, node_id: crate::raft::NodeId) -> Option<&Addr> { + self.known_nodes.get(&node_id) + } + + /// get all known nodes with their `NodeId` and `Addr` + pub(crate) fn get_nodes(&self) -> &HashMap { + &self.known_nodes + } + + /// add the node identified by `addr` if not already there and assign and return a new `node_id` + fn add_node(&mut self, addr: &Addr) -> StorageResult { + if let Some(node_id) = self.find_node_id(addr) { + Err(store_w_err(StoreError::NodeAlreadyAdded(*node_id))) + } else { + let node_id = self.next_node_id()?; + self.store_node(node_id, addr)?; + debug!(target: "TremorStateMachine::add_node", "Node {addr} added as node {node_id}"); + Ok(node_id) + } + } + + /// store the given addr under the given `node_id`, without creating a new one + fn store_node(&mut self, node_id: crate::raft::NodeId, addr: &Addr) -> StorageResult<()> { + let node_id_bytes = id_to_bin(node_id)?; + self.db + .put_cf( + Self::cf(&self.db)?, + node_id_bytes, + rmp_serde::to_vec(addr).map_err(sm_w_err)?, + ) + .map_err(sm_w_err)?; + self.known_nodes.insert(node_id, addr.clone()); + Ok(()) + } + + fn remove_node(&mut self, node_id: crate::raft::NodeId) -> StorageResult<()> { + self.known_nodes.remove(&node_id); + let node_id_bytes = id_to_bin(node_id)?; + self.db + .delete_cf(Self::cf(&self.db)?, node_id_bytes) + .map_err(sm_d_err)?; + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub(crate) struct NodesSnapshot { + next_node_id: u64, + nodes: HashMap, +} diff --git a/src/raft/test.rs b/src/raft/test.rs new file mode 100644 index 0000000000..68adaa6d9a --- /dev/null +++ b/src/raft/test.rs @@ -0,0 +1,221 @@ +use matches::assert_matches; + +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use crate::raft::{ + api::client::Tremor as ClusterClient, + config as raft_config, + node::{Addr, Node, Running}, + ClusterResult, +}; +use crate::system::ShutdownMode; +use crate::{connectors::tests::free_port::find_free_tcp_port, errors::Result}; +use std::path::Path; +use std::time::Duration; + +mod learner; +mod prelude; + +async fn free_node_addr() -> Result { + let api_port = find_free_tcp_port().await?; + let rpc_port = find_free_tcp_port().await?; + Ok(Addr::new( + format!("127.0.0.1:{api_port}"), + format!("127.0.0.1:{rpc_port}"), + )) +} + +struct TestNode { + client: ClusterClient, + addr: Addr, + running: Running, +} + +impl TestNode { + async fn bootstrap(path: impl AsRef) -> ClusterResult { + let addr = free_node_addr().await?; + let mut node = Node::new(path, raft_config()?); + + let running = node.bootstrap_as_single_node_cluster(addr.clone()).await?; + let client = ClusterClient::new(&addr.api())?; + Ok(Self { + client, + addr, + running, + }) + } + + async fn start_and_join(path: impl AsRef, join_addr: &Addr) -> ClusterResult { + let addr = free_node_addr().await?; + let mut node = Node::new(path, raft_config()?); + let running = node + .try_join(addr.clone(), vec![join_addr.api().to_string()], true) + .await?; + let client = ClusterClient::new(&addr.api())?; + Ok(Self { + client, + addr, + running, + }) + } + + #[allow(dead_code)] + async fn just_start(path: impl AsRef) -> ClusterResult { + let addr = free_node_addr().await?; + let running = Node::load_from_store(path, raft_config()?).await?; + let client = ClusterClient::new(&addr.api())?; + Ok(Self { + client, + addr, + running, + }) + } + + #[allow(dead_code)] + async fn join_as_learner(path: impl AsRef, join_addr: &Addr) -> ClusterResult { + let addr = free_node_addr().await?; + let mut node = Node::new(path, raft_config()?); + let running = node + .try_join(addr.clone(), vec![join_addr.api().to_string()], false) + .await?; + let client = ClusterClient::new(&addr.api())?; + Ok(Self { + client, + addr, + running, + }) + } + + async fn stop(self) -> ClusterResult<()> { + let Self { running, .. } = self; + let kill_switch = running.kill_switch(); + kill_switch.stop(ShutdownMode::Graceful)?; + running.join().await + + // only ever destroy the db when the raft node is known to have stopped + // rocksdb::DB::destroy(&rocksdb::Options::default(), &path).map_err(ClusterError::Rocks) + } + + fn client(&self) -> &ClusterClient { + &self.client + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn cluster_join_test() -> ClusterResult<()> { + let _: std::result::Result<_, _> = env_logger::try_init(); + let dir0 = tempfile::tempdir()?; + let dir1 = tempfile::tempdir()?; + let dir2 = tempfile::tempdir()?; + let node0 = TestNode::bootstrap(dir0.path()).await?; + let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + + // all see the same leader + let client0 = node0.client(); + let client1 = node1.client(); + let client2 = node2.client(); + let metrics = client0.metrics().await?; + let node0_leader = metrics.current_leader.expect("expect a leader from node 0"); + let node1_leader = client1 + .metrics() + .await? + .current_leader + .expect("expect a leader from node1"); + let node2_leader = client2 + .metrics() + .await? + .current_leader + .expect("expect a leader from node2"); + let node0_id = node0.running.node_data().0; + assert_eq!(node0_id, node0_leader); + assert_eq!(node0_id, node1_leader); + assert_eq!(node0_id, node2_leader); + + // all are voters in the cluster + let members = metrics + .membership_config + .membership() + .get_joint_config() + .last() + .expect("No nodes in membership config"); + assert_eq!(3, members.len()); + + node2.stop().await?; + node1.stop().await?; + node0.stop().await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn kill_and_restart_voter() -> ClusterResult<()> { + let _: std::result::Result<_, _> = env_logger::try_init(); + let dir0 = tempfile::tempdir()?; + let dir1 = tempfile::tempdir()?; + let dir2 = tempfile::tempdir()?; + + let node0 = TestNode::bootstrap(dir0.path()).await?; + let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + + let client0 = node0.client(); + let metrics = client0.metrics().await?; + let members = metrics + .membership_config + .membership() + .get_joint_config() + .last() + .expect("No nodes in membership config"); + assert_eq!(3, members.len()); + + node1.stop().await?; + // wait until we hit some timeouts + tokio::time::sleep(Duration::from_millis(500)).await; + + // restart the node + let node1 = TestNode::just_start(dir1.path()).await?; + + // check that the leader is available + // TODO: solidify to guard against timing issues + let client1 = node0.client(); + let k = client1.consistent_read("snot").await; + // Snot was never set so it should be a 404 + assert_matches!(k, Err(e) if e.is_not_found()); + + node1.stop().await?; + node2.stop().await?; + node0.stop().await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn db_fun() { + // See: https://github.com/rust-rocksdb/rust-rocksdb/issues/720 + let path = Path::new("/tmp/node01"); + if path.exists() { + std::fs::remove_dir_all(path).expect("remove to succeed"); + } + let mut db_opts = rocksdb::Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + + let cf = "foo"; + let db = rocksdb::DB::open_cf(&db_opts, path, [cf]).expect("open to succeed"); + tokio::task::spawn(async move { + let res = db + .get_cf(&db.cf_handle(cf).expect("cf to be there"), "key") + .expect("ok"); + assert!(res.is_none()); + }); +} diff --git a/src/raft/test/learner.rs b/src/raft/test/learner.rs new file mode 100644 index 0000000000..6ed7bc24b1 --- /dev/null +++ b/src/raft/test/learner.rs @@ -0,0 +1,168 @@ +use tremor_common::alias; + +// Copyright 2022, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use super::prelude::*; +use crate::raft::archive::build_archive_from_source; +use std::collections::HashMap; +use std::time::Duration; + +#[tokio::test(flavor = "multi_thread")] +async fn add_learner_test() -> ClusterResult<()> { + let _: std::result::Result<_, _> = env_logger::try_init(); + let dir0 = tempfile::tempdir()?; + let dir1 = tempfile::tempdir()?; + let dir2 = tempfile::tempdir()?; + let dir3 = tempfile::tempdir()?; + let node0 = TestNode::bootstrap(dir0.path()).await?; + let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + let client0 = node0.client(); + let metrics = client0.metrics().await?; + let members = metrics + .membership_config + .membership() + .get_joint_config() + .last() + .expect("No nodes in membership config"); + assert_eq!(3, members.len()); + + let learner_node = TestNode::join_as_learner(dir3.path(), &node0.addr).await?; + let (learner_node_id, learner_addr) = learner_node.running.node_data(); + // learner is known to the cluster + let nodemap = client0.get_nodes().await?; + assert_eq!(Some(&learner_addr), nodemap.get(&learner_node_id)); + // but is not a voter + let metrics = client0.metrics().await?; + let members = metrics + .membership_config + .membership() + .get_joint_config() + .last() + .expect("No nodes in membership config"); + assert!( + !members.contains(&learner_node_id), + "learner not to be part of cluster voters" + ); + // remove the learner again + dbg!("remove learner", &learner_node_id); + client0.remove_learner(&learner_node_id).await?; + dbg!("stop learner"); + learner_node.stop().await?; + + dbg!("remove learner node"); + client0.remove_node(&learner_node_id).await?; + dbg!("removed"); + + // TODO: deploy an app and see if the learner also runs it + // TODO: verify the whole lifecycle shenanigans of app instances with and without learner + // TODO: verify kv stuff + + node2.stop().await?; + node1.stop().await?; + node0.stop().await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn learner_runs_app() -> ClusterResult<()> { + let _: std::result::Result<_, _> = env_logger::try_init(); + let dir0 = tempfile::tempdir()?; + let dir1 = tempfile::tempdir()?; + let dir2 = tempfile::tempdir()?; + let dir3 = tempfile::tempdir()?; + let node0 = TestNode::bootstrap(dir0.path()).await?; + let node1 = TestNode::start_and_join(dir1.path(), &node0.addr).await?; + let node2 = TestNode::start_and_join(dir2.path(), &node1.addr).await?; + let client0 = node0.client(); + let metrics = client0.metrics().await?; + let members = metrics + .membership_config + .membership() + .get_joint_config() + .last() + .expect("No nodes in membership config"); + assert_eq!(3, members.len()); + + let learner_node = TestNode::join_as_learner(dir3.path(), &node0.addr).await?; + let (_learner_node_id, _learner_addr) = learner_node.running.node_data(); + let tmpfile = tempfile::NamedTempFile::new()?; + let out_path = tmpfile.into_temp_path(); + let app_entrypoint = format!( + r#" +define flow main +flow + define pipeline pt + pipeline + select event from in into out; + end; + create pipeline pt; + + define connector output from file + with + codec = "string", + config = {{ + "mode": "append", + "path": "{}" + }} + end; + create connector output; + + define connector input from oneshot + with + config = {{ + "value": 1 + }} + end; + create connector input; + + connect /connector/input to /pipeline/pt; + connect /pipeline/pt to /connector/output; +end; + "#, + out_path.display() + ); + let archive = build_archive_from_source("main", app_entrypoint.as_str())?; + let app_id = client0.install(&archive).await?; + + let flow_id = alias::FlowDefinition("main".to_string()); + let instance = alias::Flow::new(app_id, "01".to_string()); + let config = HashMap::new(); + let instance_id = client0 + .start(&flow_id, &instance, config, true, false) + .await?; + + // wait for the app to be actually started + // wait for the file to exist + while !out_path.exists() { + tokio::time::sleep(Duration::from_millis(500)).await; + } + // wait another short while for all nodes to finish writing + tokio::time::sleep(Duration::from_millis(500)).await; + // stop the flow instance + dbg!("Stopping instance"); + client0.stop_instance(&instance_id).await?; + dbg!("Instance stopped"); + // shut the nodes down + learner_node.stop().await?; + node2.stop().await?; + node1.stop().await?; + node0.stop().await?; + + // verify that each node had a flow instance running and did write the even to the file + let out_bytes = tokio::fs::read(&out_path).await?; + assert_eq!("1111", &String::from_utf8_lossy(&out_bytes)); + out_path.close()?; + Ok(()) +} diff --git a/tremor-api/src/api/prelude.rs b/src/raft/test/prelude.rs similarity index 79% rename from tremor-api/src/api/prelude.rs rename to src/raft/test/prelude.rs index f9dea47fe5..5a0c6e617f 100644 --- a/tremor-api/src/api/prelude.rs +++ b/src/raft/test/prelude.rs @@ -1,4 +1,4 @@ -// Copyright 2020-2021, The Tremor Team +// Copyright 2022, The Tremor Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use crate::api::*; -pub(crate) use http_types::StatusCode; -pub(crate) use tide::Response; +pub(super) use crate::raft::test::TestNode; +pub(super) use crate::raft::ClusterResult; diff --git a/src/system.rs b/src/system.rs index 6ec12629f3..8b7ee454fd 100644 --- a/src/system.rs +++ b/src/system.rs @@ -17,15 +17,27 @@ pub mod flow; /// contains the runtime actor starting and maintaining flows pub mod flow_supervisor; -use self::flow::Flow; +use std::{ + sync::{Arc, RwLock}, + time::Duration, +}; + +use self::flow::{DeploymentType, Flow}; use crate::{ - channel::Sender, + channel::{oneshot, Sender}, connectors, errors::{Error, Kind as ErrorKind, Result}, + instance::IntendedState as IntendedInstanceState, + log_error, raft, }; -use std::time::Duration; use tokio::{sync::oneshot, task::JoinHandle, time::timeout}; -use tremor_script::{ast, highlighter::Highlighter}; +use tremor_common::alias; +use tremor_script::{ + ast, + deploy::Deploy, + highlighter::{self, Highlighter}, + FN_REGISTRY, +}; /// Configuration for the runtime #[derive(Default)] @@ -48,6 +60,19 @@ pub enum ShutdownMode { Forceful, } +impl std::fmt::Display for ShutdownMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Graceful => "graceful", + Self::Forceful => "forceful", + } + ) + } +} + /// for draining and stopping #[derive(Debug, Clone)] pub struct KillSwitch(Sender); @@ -72,7 +97,7 @@ impl KillSwitch { ); } } - let res = self.0.send(flow_supervisor::Msg::Stop).await; + let res = self.0.send(flow_supervisor::Msg::Terminate).await; if let Err(e) = &res { error!("Error stopping all Flows: {e}"); } @@ -92,47 +117,174 @@ impl KillSwitch { /// Tremor runtime #[derive(Clone, Debug)] -pub struct World { - pub(crate) system: flow_supervisor::Channel, +pub struct Runtime { + pub(crate) flows: flow_supervisor::Channel, pub(crate) kill_switch: KillSwitch, + pub(crate) cluster_manager: Arc>>, } -impl World { - /// Instantiate a flow from +impl Runtime { + // pub(crate) fn get_manager(&self) -> Result { + // self.cluster_manager + // .read()? + // .as_ref() + // .cloned() + // .ok_or_else(|| ErrorKind::RaftNotRunning.into()) + // } + pub(crate) fn maybe_get_manager(&self) -> Result> { + Ok(self.cluster_manager.read()?.as_ref().cloned()) + } + + pub async fn wait_for_cluster(&self) { + while self + .cluster_manager + .read() + .ok() + .map_or(true, |v| v.is_none()) + { + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + /// Loads a Troy src and starts all deployed flows. + /// Returns the number of deployed and started flows + /// /// # Errors - /// If the flow can't be started - pub async fn start_flow(&self, flow: &ast::DeployFlow<'static>) -> Result<()> { + /// Fails if the source can not be loaded + pub async fn load_troy(&self, name: &str, src: &str) -> Result { + info!("Loading troy src {}", name); + + let aggr_reg = tremor_script::registry::aggr(); + + let deployable = Deploy::parse(&src, &*FN_REGISTRY.read()?, &aggr_reg); + let mut h = highlighter::Term::stderr(); + let deployable = match deployable { + Ok(deployable) => { + deployable.format_warnings_with(&mut h)?; + deployable + } + Err(e) => { + log_error!(h.format_error(&e), "Error: {e}"); + + return Err(format!("failed to load troy file: {src}").into()); + } + }; + + let mut count = 0; + // first deploy them + for flow in deployable.iter_flows() { + self.deploy_flow(alias::App::default(), flow, DeploymentType::AllNodes) + .await?; + } + // start flows in a second step + for flow in deployable.iter_flows() { + self.start_flow(alias::Flow::new( + alias::App::default(), + &flow.instance_alias, + )) + .await?; + count += 1; + } + Ok(count) + } + + /// Deploy a flow - create an instance of it + /// + /// This flow instance is not started yet. + /// # Errors + /// If the flow can't be deployed + pub async fn deploy_flow( + &self, + app_id: alias::App, + flow: &ast::DeployFlow<'static>, + deployment_type: DeploymentType, + ) -> Result { let (tx, rx) = oneshot::channel(); - self.system - .send(flow_supervisor::Msg::StartDeploy { + self.flows + .send(flow_supervisor::Msg::DeployFlow { + app: app_id, flow: Box::new(flow.clone()), sender: tx, + raft: self.maybe_get_manager()?.unwrap_or_default(), + deployment_type, }) .await?; - if let Err(e) = rx.await? { - let err_str = match e { - Error( - ErrorKind::Script(e) - | ErrorKind::Pipeline(tremor_pipeline::errors::ErrorKind::Script(e)), - _, - ) => { - let mut h = crate::ToStringHighlighter::new(); - h.format_error(&tremor_script::errors::Error::from(e))?; - h.finalize()?; - h.to_string() - } - err => err.to_string(), - }; - error!( - "Error starting deployment of flow {}: {}", - flow.instance_alias, &err_str - ); - Err(ErrorKind::DeployFlowError(flow.instance_alias.clone(), err_str).into()) - } else { - Ok(()) + match rx.await? { + Err(e) => { + let err_str = match e { + Error( + ErrorKind::Script(e) + | ErrorKind::Pipeline(tremor_pipeline::errors::ErrorKind::Script(e)), + _, + ) => { + let mut h = crate::ToStringHighlighter::new(); + h.format_error(&tremor_script::errors::Error::from(e))?; + h.finalize()?; + h.to_string() + } + err => err.to_string(), + }; + error!( + "Error starting deployment of flow {}: {err_str}", + flow.instance_alias + ); + Err(ErrorKind::DeployFlowError(flow.instance_alias.clone(), err_str).into()) + } + Ok(flow_id) => Ok(flow_id), } } + /// # Errors + /// if the flow state change fails + pub async fn change_flow_state( + &self, + id: alias::Flow, + intended_state: IntendedInstanceState, + ) -> Result<()> { + let (reply_tx, reply_rx) = oneshot(); + self.flows + .send(flow_supervisor::Msg::ChangeInstanceState { + id, + intended_state, + reply_tx, + }) + .await?; + reply_rx.await? + } + + /// start a flow and wait for the result + /// + /// # Errors + /// if the flow can't be started + pub async fn start_flow(&self, id: alias::Flow) -> Result<()> { + self.change_flow_state(id, IntendedInstanceState::Running) + .await + } + + /// stops a flow and waits for the result + /// + /// # Errors + /// if the flow can't be stopped + pub async fn stop_flow(&self, id: alias::Flow) -> Result<()> { + self.change_flow_state(id, IntendedInstanceState::Stopped) + .await + } + + /// pauses a flow and waits for the result + /// + /// # Errors + /// if the flow can't be paused + pub async fn pause_flow(&self, id: alias::Flow) -> Result<()> { + self.change_flow_state(id, IntendedInstanceState::Paused) + .await + } + /// resumes a flow + /// + /// # Errors + /// if the flow can't be resumed + pub async fn resume_flow(&self, id: alias::Flow) -> Result<()> { + self.start_flow(id).await // equivalent + } + /// Registers the given connector type with `type_name` and the corresponding `builder` /// /// # Errors @@ -141,7 +293,7 @@ impl World { &self, builder: Box, ) -> Result<()> { - self.system + self.flows .send(flow_supervisor::Msg::RegisterConnectorType { connector_type: builder.connector_type(), builder, @@ -156,11 +308,10 @@ impl World { /// /// # Errors /// * if we fail to send the request or fail to receive it - pub async fn get_flow(&self, flow_id: String) -> Result { + pub async fn get_flow(&self, flow_id: alias::Flow) -> Result { let (flow_tx, flow_rx) = oneshot::channel(); - let flow_id = tremor_common::alias::Flow::new(flow_id); - self.system - .send(flow_supervisor::Msg::GetFlow(flow_id.clone(), flow_tx)) + self.flows + .send(flow_supervisor::Msg::GetFlow(flow_id, flow_tx)) .await?; flow_rx.await? } @@ -171,7 +322,7 @@ impl World { /// * if we fail to send the request or fail to receive it pub async fn get_flows(&self) -> Result> { let (reply_tx, reply_rx) = oneshot::channel(); - self.system + self.flows .send(flow_supervisor::Msg::GetFlows(reply_tx)) .await?; reply_rx.await? @@ -182,11 +333,13 @@ impl World { /// # Errors /// * if the world manager can't be started pub async fn start(config: WorldConfig) -> Result<(Self, JoinHandle>)> { + let cluster_manager = Arc::new(RwLock::new(None)); let (system_h, system, kill_switch) = flow_supervisor::FlowSupervisor::new().start(); let world = Self { - system, + flows: system, kill_switch, + cluster_manager, }; connectors::register_builtin_connector_types(&world, config.debug_connectors).await?; diff --git a/src/system/flow.rs b/src/system/flow.rs index d3dc047960..e280c4c21e 100644 --- a/src/system/flow.rs +++ b/src/system/flow.rs @@ -13,29 +13,32 @@ // limitations under the License. use crate::{ - channel::{bounded, Sender}, + channel::{bounded, oneshot, OneShotSender, Sender}, errors::empty_error, - qsize, + raft::{self, NodeId}, }; use crate::{ connectors::{self, ConnectorResult, Known}, errors::{Error, Kind as ErrorKind, Result}, - instance::State, + instance::{IntendedState, State}, log_error, pipeline::{self, InputTarget}, primerge::PriorityMerge, - system::KillSwitch, + system::{KillSwitch, ShutdownMode}, }; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use hashbrown::HashMap; -use std::collections::HashSet; -use std::time::Duration; -use tokio::{task, time::timeout}; +use std::{collections::HashSet, ops::ControlFlow, pin::Pin, time::Duration}; +use tokio::{ + task, + time::{self, timeout}, +}; use tokio_stream::wrappers::ReceiverStream; use tremor_common::{ alias, - ids::{ConnectorIdGen, OperatorIdGen}, + uids::{ConnectorUIdGen, OperatorUIdGen}, }; +use tremor_pipeline::MetricsChannel; use tremor_script::{ ast::{self, ConnectStmt, Helper}, errors::{error_generic, not_defined_err}, @@ -44,24 +47,27 @@ use tremor_script::{ #[derive(Debug)] /// Control Plane message accepted by each binding control plane handler pub(crate) enum Msg { - /// start all contained instances - Start, - /// pause all contained instances - Pause, - /// resume all contained instance from pause - Resume, - /// stop all contained instances, and get notified via the given sender when the stop process is done - Stop(Sender>), - /// drain all contained instances, and get notified via the given sender when the drain process is done - Drain(Sender>), + /// Change the state of this flow to `intended_state` + ChangeState { + // the state this flow should be changed to + intended_state: IntendedState, + // this is where we send the result + reply_tx: OneShotSender>, + }, /// Request a `StatusReport` from this instance. /// /// The sender expects a Result, which makes it easier to signal errors on the message handling path to the sender - Report(Sender>), + Report(OneShotSender>), /// Get the addr for a single connector - GetConnector(alias::Connector, Sender>), + GetConnector( + alias::Flow, + alias::Connector, + OneShotSender>, + ), /// Get the addresses for all connectors of this flow - GetConnectors(Sender>>), + GetConnectors(OneShotSender>>), + /// Periodic tick + Tick, } type Addr = Sender; @@ -79,19 +85,52 @@ pub struct StatusReport { pub alias: alias::Flow, /// the current state pub status: State, - /// the crated connectors + /// the created connectors pub connectors: Vec, } +#[derive(Debug, Clone, Default)] +pub(crate) struct AppContext { + pub(crate) id: alias::Flow, + pub(crate) raft: raft::Cluster, + pub(crate) metrics: MetricsChannel, +} + +impl AppContext { + pub fn id(&self) -> &alias::App { + self.id.app_id() + } + pub fn instance(&self) -> &alias::Instance { + self.id.instance_id() + } + pub fn node_id(&self) -> NodeId { + self.raft.id() + } +} + +impl std::fmt::Display for AppContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[Node::{}][{}]", self.raft.id(), self.id) + } +} + impl Flow { pub(crate) fn id(&self) -> &alias::Flow { &self.alias } - pub(crate) async fn stop(&self, tx: Sender>) -> Result<()> { - self.addr.send(Msg::Stop(tx)).await.map_err(Error::from) - } - pub(crate) async fn drain(&self, tx: Sender>) -> Result<()> { - self.addr.send(Msg::Drain(tx)).await.map_err(Error::from) + + pub(crate) async fn change_state( + &self, + new_state: IntendedState, + tx: OneShotSender>, + ) -> Result<()> { + self.addr + .send(Msg::ChangeState { + intended_state: new_state, + reply_tx: tx, + }) + .await + .map_err(Error::from) } /// request a `StatusReport` from this `Flow` @@ -99,9 +138,9 @@ impl Flow { /// # Errors /// if the flow is not running anymore and can't be reached pub async fn report_status(&self) -> Result { - let (tx, mut rx) = bounded(1); + let (tx, rx) = oneshot(); self.addr.send(Msg::Report(tx)).await?; - rx.recv().await.ok_or_else(empty_error)? + rx.await? } /// get the Address used to send messages of a connector within this flow, identified by `connector_id` @@ -109,12 +148,12 @@ impl Flow { /// # Errors /// if the flow is not running anymore and can't be reached or if the connector is not part of the flow pub async fn get_connector(&self, connector_alias: String) -> Result { - let connector_alias = alias::Connector::new(self.id().clone(), connector_alias); - let (tx, mut rx) = bounded(1); + let connector_alias = alias::Connector::new(connector_alias); + let (tx, rx) = oneshot(); self.addr - .send(Msg::GetConnector(connector_alias, tx)) + .send(Msg::GetConnector(self.id().clone(), connector_alias, tx)) .await?; - rx.recv().await.ok_or_else(empty_error)? + rx.await? } /// Get the Addresses of all connectors of this flow @@ -122,9 +161,17 @@ impl Flow { /// # Errors /// if the flow is not running anymore and can't be reached pub async fn get_connectors(&self) -> Result> { - let (tx, mut rx) = bounded(1); + let (tx, rx) = oneshot(); self.addr.send(Msg::GetConnectors(tx)).await?; - rx.recv().await.ok_or_else(empty_error)? + rx.await? + } + + /// Start this flow and all connectors in it. + /// + /// # Errors + /// if the connector is not running anymore and can't be reached + pub async fn start(&self, tx: OneShotSender>) -> Result<()> { + self.change_state(IntendedState::Running, tx).await } /// Pause this flow and all connectors in it. @@ -132,8 +179,8 @@ impl Flow { /// # Errors /// if the connector is not running anymore and can't be reached /// or if the connector is in a state where it can't be paused (e.g. failed) - pub async fn pause(&self) -> Result<()> { - self.addr.send(Msg::Pause).await.map_err(Error::from) + pub async fn pause(&self, tx: OneShotSender>) -> Result<()> { + self.change_state(IntendedState::Paused, tx).await } /// Resume this flow and all connectors in it. @@ -141,20 +188,32 @@ impl Flow { /// # Errors /// if the connector is not running anymore and can't be reached /// or if the connector is in a state where it can't be resumed (e.g. failed) - pub async fn resume(&self) -> Result<()> { - self.addr.send(Msg::Resume).await.map_err(Error::from) + pub async fn resume(&self, tx: OneShotSender>) -> Result<()> { + self.change_state(IntendedState::Running, tx).await } - pub(crate) async fn start( + pub(crate) async fn stop(&self, tx: OneShotSender>) -> Result<()> { + self.change_state(IntendedState::Stopped, tx).await + } + + /// Deploy the given flow instance and all its pipelines and connectors + /// but doesn't start them yet + /// + /// # Errors + /// If any of the operations of spawning connectors, linking pipelines and connectors or spawning the flow instance + /// fails. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn deploy( + ctx: AppContext, flow: ast::DeployFlow<'static>, - operator_id_gen: &mut OperatorIdGen, - connector_id_gen: &mut ConnectorIdGen, + operator_id_gen: &mut OperatorUIdGen, + connector_id_gen: &mut ConnectorUIdGen, known_connectors: &Known, kill_switch: &KillSwitch, + deployment_type: DeploymentType, ) -> Result { let mut pipelines = HashMap::new(); let mut connectors = HashMap::new(); - let flow_alias = alias::Flow::from(&flow); for create in &flow.defn.creates { let alias: &str = &create.instance_alias; @@ -162,7 +221,7 @@ impl Flow { ast::CreateTargetDefinition::Connector(defn) => { let mut defn = defn.clone(); defn.params.ingest_creational_with(&create.with)?; - let connector_alias = alias::Connector::new(flow_alias.clone(), alias); + let connector_alias = alias::Connector::new(alias); let config = crate::Connector::from_defn(&connector_alias, &defn)?; let builder = known_connectors @@ -178,6 +237,7 @@ impl Flow { builder.as_ref(), config, kill_switch, + ctx.clone(), ) .await?, ); @@ -190,11 +250,12 @@ impl Flow { defn.to_query(&create.with, &mut helper)? }; - let pipeline_alias = alias::Pipeline::new(flow_alias.clone(), alias); + let pipeline_alias = alias::Pipeline::new(alias); let pipeline = tremor_pipeline::query::Query( tremor_script::query::Query::from_query(query), ); - let addr = pipeline::spawn(pipeline_alias, &pipeline, operator_id_gen)?; + let addr = + pipeline::spawn(ctx.clone(), pipeline_alias, &pipeline, operator_id_gen)?; pipelines.insert(alias.to_string(), addr); } } @@ -206,16 +267,15 @@ impl Flow { } let addr = spawn_task( - flow_alias.clone(), - &pipelines, - connectors, + ctx.clone(), + pipelines, + &connectors, &flow.defn.connections, + deployment_type, ); - addr.send(Msg::Start).await?; - let this = Flow { - alias: flow_alias.clone(), + alias: ctx.id, addr, }; @@ -352,337 +412,655 @@ async fn link( } Ok(()) } +#[derive(Debug)] +/// wrapper for all possible messages handled by the flow task +enum MsgWrapper { + Msg(Msg), + StartResult(ConnectorResult<()>), + DrainResult(ConnectorResult<()>), + StopResult(ConnectorResult<()>), +} -/// task handling flow instance control plane -#[allow(clippy::too_many_lines)] -fn spawn_task( - id: alias::Flow, - pipelines: &HashMap, - connectors: HashMap, - links: &[ConnectStmt], -) -> Addr { - #[derive(Debug)] - /// wrapper for all possible messages handled by the flow task - enum MsgWrapper { - Msg(Msg), - StartResult(ConnectorResult<()>), - DrainResult(ConnectorResult<()>), - StopResult(ConnectorResult<()>), - } - - let (msg_tx, msg_rx) = bounded(qsize()); - let (drain_tx, drain_rx) = bounded(qsize()); - let (stop_tx, stop_rx) = bounded(qsize()); - let (start_tx, start_rx) = bounded(qsize()); - - let mut input_channel = PriorityMerge::new( - ReceiverStream::new(msg_rx).map(MsgWrapper::Msg), - PriorityMerge::new( - ReceiverStream::new(drain_rx).map(MsgWrapper::DrainResult), - PriorityMerge::new( - ReceiverStream::new(stop_rx).map(MsgWrapper::StopResult), - ReceiverStream::new(start_rx).map(MsgWrapper::StartResult), - ), - ), - ); - let addr = msg_tx; - let mut state = State::Initializing; - // let registries = self.reg.clone(); - - // extracting connectors and pipes from the links - let sink_connectors: HashSet = links - .iter() - .filter_map(|c| { - if let ConnectStmt::PipelineToConnector { to, .. } = c { - Some(to.alias().to_string()) - } else { - None - } - }) - .collect(); - let source_connectors: HashSet = links - .iter() - .filter_map(|c| { - if let ConnectStmt::ConnectorToPipeline { from, .. } = c { - Some(from.alias().to_string()) - } else { - None - } - }) - .collect(); - - let pipelines: Vec<_> = pipelines.values().cloned().collect(); - - let start_points: Vec<_> = source_connectors - .difference(&sink_connectors) - .filter_map(|p| connectors.get(p)) - .cloned() - .collect(); - let mixed_pickles: Vec<_> = sink_connectors - .intersection(&source_connectors) - .filter_map(|p| connectors.get(p)) - .cloned() - .collect(); - let end_points: Vec<_> = sink_connectors - .difference(&source_connectors) - .filter_map(|p| connectors.get(p)) - .cloned() - .collect(); - - // for receiving drain/stop completion notifications from connectors - let mut expected_drains: usize = 0; - let mut expected_stops: usize = 0; - - // for storing senders that have been sent to us - let mut drain_senders = Vec::with_capacity(1); - let mut stop_senders = Vec::with_capacity(1); - - let prefix = format!("[Flow::{id}]"); - - task::spawn(async move { - let mut wait_for_start_responses: usize = 0; - while let Some(wrapped) = input_channel.next().await { - match wrapped { - MsgWrapper::Msg(Msg::Start) if state == State::Initializing => { - info!("{prefix} Starting..."); - // start all pipelines first - order doesnt matter as connectors aren't started yet - for pipe in &pipelines { - pipe.start().await?; - } +/// How the depoloyment is distributed on a cluster +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize)] +pub enum DeploymentType { + #[default] + /// Let the pipeline run on all nodes in the cluster + AllNodes, + /// Run the pipeline on a single node on the cluster + OneNode, +} - if connectors.is_empty() { - state = State::Running; - info!("{prefix} Started."); - } else { - // start sink connectors first then source/sink connectors then source only connectors - for conn in end_points.iter().chain(&mixed_pickles).chain(&start_points) { - conn.start(start_tx.clone()).await?; - wait_for_start_responses += 1; - } +struct RunningFlow { + app_ctx: AppContext, + state: State, + expected_starts: usize, + expected_drains: usize, + expected_stops: usize, + pipelines: HashMap, + source_only_connectors: Vec, + source_and_sink_connectors: Vec, + sink_only_connectors: Vec, + state_change_senders: HashMap>>>, + input_channel: Pin + Send + Sync>>, + msg_tx: Sender, + drain_tx: Sender>, + stop_tx: Sender>, + start_tx: Sender>, + deployment_type: DeploymentType, +} - debug!( - "{prefix} Waiting for {wait_for_start_responses} connectors to start." - ); - } - } - MsgWrapper::Msg(Msg::Start) => { - info!("{prefix} Ignoring Start message. Current state: {state}"); - } - MsgWrapper::Msg(Msg::Pause) if state == State::Running => { - info!("{prefix} Pausing..."); - for source in start_points.iter().chain(&mixed_pickles).chain(&end_points) { - source.pause().await?; - } - for pipeline in &pipelines { - pipeline.pause().await?; - } - state = State::Paused; - info!("{prefix} Paused."); +impl RunningFlow { + fn new( + app_ctx: AppContext, + pipelines: HashMap, + connectors: &HashMap, + deployment_type: DeploymentType, + links: &[ConnectStmt], + ) -> Self { + let (msg_tx, msg_rx) = bounded(crate::qsize()); + let (drain_tx, drain_rx) = bounded(crate::qsize()); + let (stop_tx, stop_rx) = bounded(crate::qsize()); + let (start_tx, start_rx) = bounded(crate::qsize()); + let input_channel = Box::pin(PriorityMerge::new( + ReceiverStream::new(msg_rx).map(MsgWrapper::Msg), + PriorityMerge::new( + ReceiverStream::new(drain_rx).map(MsgWrapper::DrainResult), + PriorityMerge::new( + ReceiverStream::new(stop_rx).map(MsgWrapper::StopResult), + ReceiverStream::new(start_rx).map(MsgWrapper::StartResult), + ), + ), + )); + // extracting connectors and pipes from the links + let sink_connectors: HashSet = links + .iter() + .filter_map(|c| { + if let ConnectStmt::PipelineToConnector { to, .. } = c { + Some(to.alias().to_string()) + } else { + None } - MsgWrapper::Msg(Msg::Pause) => { - info!("{prefix} Ignoring Pause message. Current state: {state}",); + }) + .collect(); + let source_connectors: HashSet = links + .iter() + .filter_map(|c| { + if let ConnectStmt::ConnectorToPipeline { from, .. } = c { + Some(from.alias().to_string()) + } else { + None } - MsgWrapper::Msg(Msg::Resume) if state == State::Paused => { - info!("{prefix} Resuming..."); + }) + .collect(); + + let source_only_connectors: Vec = source_connectors + .difference(&sink_connectors) + .filter_map(|p| connectors.get(p)) + .cloned() + .collect(); + let source_and_sink_connectors: Vec = sink_connectors + .intersection(&source_connectors) + .filter_map(|p| connectors.get(p)) + .cloned() + .collect(); + let sink_only_connectors: Vec = sink_connectors + .difference(&source_connectors) + .filter_map(|p| connectors.get(p)) + .cloned() + .collect(); + Self { + app_ctx, + state: State::Initializing, + expected_starts: 0, + expected_drains: 0, + expected_stops: 0, + pipelines, + source_only_connectors, + source_and_sink_connectors, + sink_only_connectors, + state_change_senders: HashMap::new(), + input_channel, + msg_tx, + drain_tx, + stop_tx, + start_tx, + deployment_type, + } + } - for pipeline in &pipelines { - pipeline.resume().await?; - } - for sink in end_points.iter().chain(&mixed_pickles).chain(&start_points) { - sink.resume().await?; - } - state = State::Running; - info!("{prefix} Resumed."); - } - MsgWrapper::Msg(Msg::Resume) => { - info!("{prefix} Ignoring Resume message. Current state: {state}",); - } - MsgWrapper::Msg(Msg::Drain(_sender)) if state == State::Draining => { - info!("{prefix} Ignoring Drain message. Current state: {state}",); + fn addr(&self) -> Addr { + self.msg_tx.clone() + } + + fn has_connectors(&self) -> bool { + !(self.source_only_connectors.is_empty() + && self.source_and_sink_connectors.is_empty() + && self.sink_only_connectors.is_empty()) + } + + fn connectors_start_to_end(&self) -> impl Iterator { + self.source_only_connectors + .iter() + .chain(&self.source_and_sink_connectors) + .chain(&self.sink_only_connectors) + } + + fn connectors_end_to_start(&self) -> impl Iterator { + self.sink_only_connectors + .iter() + .chain(&self.source_and_sink_connectors) + .chain(&self.source_only_connectors) + } + fn insert_state_change_sender( + &mut self, + intended_state: IntendedState, + sender: OneShotSender>, + ) { + self.state_change_senders + .entry(State::from(intended_state)) + .or_insert_with(Vec::new) + .push(sender); + } + + #[allow(clippy::too_many_lines, clippy::cast_possible_truncation)] + async fn run(mut self) -> Result<()> { + let prefix = self.app_ctx.to_string(); + let hash_key = self.app_ctx.id.to_string(); + let node_id = self.app_ctx.node_id(); + + let mut current_nodes: Vec = vec![]; + let mut slot: usize = 0; + let mut intended_active_state = IntendedState::Paused; + + let jh = + jumphash::JumpHasher::new_with_keys(8_390_880_576_440_238_080, 128_034_676_764_530); + + // let activation_tx = self.msg_tx.clone(); + // We only need ticks for a OneNode deployment + if self.deployment_type == DeploymentType::OneNode { + let tick_tx = self.msg_tx.clone(); + task::spawn(async move { + time::sleep(Duration::from_secs(20)).await; + while tick_tx.send(Msg::Tick).await.is_ok() { + time::sleep(Duration::from_secs(20)).await; } - MsgWrapper::Msg(Msg::Drain(sender)) => { - info!("{prefix} Draining..."); - - state = State::Draining; - - // handling the weird case of no connectors - if connectors.is_empty() { - info!("{prefix} Nothing to drain."); - log_error!( - sender.send(Ok(())).await, - "{prefix} Error sending drain result: {e}" - ); - } else { - drain_senders.push(sender); - - // QUIESCENCE - // - send drain msg to all connectors - // - wait until - // a) all connectors are drained (means all pipelines in between are also drained) or - // b) we timed out - - // source only connectors - for addr in start_points.iter().chain(&mixed_pickles).chain(&end_points) { - if !log_error!( - addr.drain(drain_tx.clone()).await, - "{prefix} Error starting Draining Connector {addr:?}: {e}" - ) { - expected_drains += 1; + }); + } + + while let Some(wrapped) = self.input_channel.next().await { + match wrapped { + MsgWrapper::Msg(Msg::Tick) => { + if let Ok(Ok(members)) = timeout( + Duration::from_millis(100), + self.app_ctx.raft.get_last_membership(), + ) + .await + { + current_nodes = members.into_iter().collect(); + slot = jh.slot(&hash_key, current_nodes.len() as u32) as usize; + + if is_active_node(¤t_nodes, slot, node_id) + && intended_active_state == IntendedState::Running + { + // dbg!( + // "active", + // &hash_key, + // slot, + // node_id, + // intended_active_state, + // self.state + // ); + + match self.state { + State::Paused => { + if let Err(e) = self.handle_resume(&prefix).await { + error!("{prefix} Error during resuming: {e}"); + self.change_state(State::Failed); + } + } + State::Initializing => { + if let Err(e) = self.handle_start(&prefix).await { + error!("{prefix} Error starting: {e}"); + self.change_state(State::Failed); + }; + } + state => { + debug!("not changing from state: {state}"); + } + } + } else { + // dbg!( + // "passive", + // &hash_key, + // slot, + // node_id, + // intended_active_state, + // self.state + // ); + if self.state == State::Running { + self.handle_pause(&prefix).await?; + intended_active_state = IntendedState::Running; } } } } - MsgWrapper::Msg(Msg::Stop(sender)) => { - info!("{prefix} Stopping..."); - if connectors.is_empty() { - log_error!( - sender.send(Ok(())).await, - "{prefix} Error sending Stop result: {e}" - ); - } else { - stop_senders.push(sender); - for connector in - end_points.iter().chain(&start_points).chain(&mixed_pickles) - { - if !log_error!( - connector.stop(stop_tx.clone()).await, - "{prefix} Error stopping connector {connector}: {e}" - ) { - expected_stops += 1; + MsgWrapper::Msg(Msg::ChangeState { + intended_state, + reply_tx, + }) => { + // We are always active on a all node deployment + let is_active = self.deployment_type == DeploymentType::AllNodes + || is_active_node(¤t_nodes, slot, node_id); + + intended_active_state = intended_state; + match (self.state, intended_state) { + (State::Initializing, IntendedState::Running) if is_active => { + self.insert_state_change_sender(intended_state, reply_tx); + if let Err(e) = self.handle_start(&prefix).await { + error!("{prefix} Error starting: {e}"); + self.change_state(State::Failed); } } - } - - for pipeline in &pipelines { - if let Err(e) = pipeline.stop().await { - error!("{prefix} Error stopping pipeline {pipeline:?}: {e}"); + (State::Initializing, IntendedState::Running) => { + if reply_tx.send(Ok(())).is_err() { + error!("{prefix} Error sending StateChange response failed"); + }; + trace!("{prefix} Ignoring start request, not active node"); + } + (State::Initializing, IntendedState::Paused) => { + // we ignore this + // we can only go from Initializing to Paused state by starting and then pausing right away + // this might lead to unwanted traffic along the way, so we keep it as it is + // Initializing is a good enough Pause for now + // We didn't connect yet, though, so we do not realize if config is broken just yet + log_error!( + reply_tx.send(Ok(())), + "{prefix} Error sending StateChange response: {e:?}" + ); + info!("{prefix} Paused."); + } + (state, IntendedState::Stopped) => { + let mode = match state { + State::Initializing | State::Failed | State::Draining => { + // no need to drain here + ShutdownMode::Forceful + } + State::Running | State::Paused => ShutdownMode::Graceful, // always drain + State::Stopped => { + log_error!( + reply_tx.send(Ok(())), + "{prefix} Error sending StateChagnge response: {e:?}" + ); + info!("{prefix} Already in state {state}"); + continue; + } + }; + self.insert_state_change_sender(intended_state, reply_tx); + match self.handle_stop(mode, &prefix).await { + Ok(ControlFlow::Continue(())) => {} + Ok(ControlFlow::Break(())) => { + break; + } + Err(e) => { + error!("{prefix} Error stopping: {e}"); + // we don't care if we failed here, we terminate anyways + self.change_state(State::Stopped); + break; + } + } + } + (state @ State::Running, IntendedState::Running) + | (state @ State::Paused, IntendedState::Paused) => { + log_error!( + reply_tx.send(Ok(())), + "{prefix} Error sending StateChagnge response: {e:?}" + ); + info!("{prefix} Already in state {state}"); + } + (State::Running, IntendedState::Paused) => { + self.insert_state_change_sender(intended_state, reply_tx); + if let Err(e) = self.handle_pause(&prefix).await { + error!("{prefix} Error during pausing: {e}"); + self.change_state(State::Failed); + } + } + (State::Paused, IntendedState::Running) if is_active => { + self.insert_state_change_sender(intended_state, reply_tx); + if let Err(e) = self.handle_resume(&prefix).await { + error!("{prefix} Error during resuming: {e}"); + self.change_state(State::Failed); + } + } + (State::Paused, IntendedState::Running) => { + if reply_tx.send(Ok(())).is_err() { + error!("{prefix} Error sending StateChange response failed"); + }; + trace!("{prefix} Ignoring start request, not active node"); + } + (State::Draining, IntendedState::Running | IntendedState::Paused) + | (State::Stopped, _) => { + log_error!( + reply_tx.send(Err("illegal state change".into())), + "{prefix} Error sending StateChagnge response: {e:?}" + ); + todo!("illegal state change") + } + (State::Failed, intended) => { + self.insert_state_change_sender(intended_state, reply_tx); + error!("{prefix} Cannot change state from failed to {intended}"); + // trigger erroring all listeners + self.change_state(State::Failed); } } - - state = State::Stopped; } MsgWrapper::Msg(Msg::Report(sender)) => { // TODO: aggregate states of all containing instances - let connectors = connectors - .keys() - .map(|c| alias::Connector::new(id.clone(), c)) + let connectors = self + .connectors_start_to_end() + .map(|c| &c.alias) + .cloned() .collect(); let report = StatusReport { - alias: id.clone(), - status: state, + alias: self.app_ctx.id.clone(), + status: self.state, connectors, }; log_error!( - sender.send(Ok(report)).await, - "{prefix} Error sending status report: {e}" + sender.send(Ok(report)), + "{prefix} Error sending status report: {e:?}" ); } - MsgWrapper::Msg(Msg::GetConnector(connector_alias, reply_tx)) => { + MsgWrapper::Msg(Msg::GetConnector(flow_id, connector_alias, reply_tx)) => { + // TODO: inefficient find, but on the other hand we don't need to store connectors in another way + let res = self + .connectors_start_to_end() + .find(|c| c.alias == connector_alias) + .cloned() + .ok_or_else(|| { + Error::from(ErrorKind::ConnectorNotFound( + flow_id.to_string(), + connector_alias.connector_alias().to_string(), + )) + }); log_error!( - reply_tx - .send( - connectors - .get(&connector_alias.connector_alias().to_string()) - .cloned() - .ok_or_else(|| { - ErrorKind::ConnectorNotFound( - connector_alias.flow_alias().to_string(), - connector_alias.connector_alias().to_string(), - ) - .into() - }) - ) - .await, - "{prefix} Error sending GetConnector response: {e}" + reply_tx.send(res), + "{prefix} Error sending GetConnector response: {e:?}" ); } MsgWrapper::Msg(Msg::GetConnectors(reply_tx)) => { - let res = connectors.values().cloned().collect::>(); + let res = self.connectors_start_to_end().cloned().collect::>(); log_error!( - reply_tx.send(Ok(res)).await, - "{prefix} Error sending GetConnectors response: {e}" + reply_tx.send(Ok(res)), + "{prefix} Error sending GetConnectors response: {e:?}" ); } + MsgWrapper::StartResult(res) => { + self.handle_start_result(res, &prefix); + } + MsgWrapper::DrainResult(res) => { + if let Err(e) = self.handle_drain_result(res, &prefix).await { + error!("{prefix} Error during draining: {e}"); + self.change_state(State::Failed); + } + } + MsgWrapper::StopResult(res) => { + if self.handle_stop_result(res, &prefix) == ControlFlow::Break(()) { + break; + } + } + } + } + info!("{prefix} Stopped."); + Ok(()) + } - MsgWrapper::DrainResult(conn_res) => { - info!("[Flow::{}] Connector {} drained.", &id, &conn_res.alias); + async fn handle_start(&mut self, prefix: impl std::fmt::Display) -> Result<()> { + info!("{prefix} Starting..."); + // start all pipelines first - order doesnt matter as connectors aren't started yet + for pipe in self.pipelines.values() { + pipe.start().await?; + } - log_error!( - conn_res.res, - "{prefix} Error during Draining in Connector {}: {e}", - &conn_res.alias - ); + if self.has_connectors() { + // start sink connectors first then source/sink connectors then source only connectors + let mut started = 0_usize; + for conn in self.connectors_end_to_start() { + conn.start(self.start_tx.clone()).await?; + started += 1; + } + self.expected_starts = started; + + debug!( + "{prefix} Waiting for {} connectors to start.", + self.expected_starts + ); + } else { + self.change_state(State::Running); + info!("{prefix} Started."); + } + Ok(()) + } - let old = expected_drains; - expected_drains = expected_drains.saturating_sub(1); - if expected_drains == 0 && old > 0 { - info!("{prefix} All connectors are drained."); - // upon last drain - for drain_sender in drain_senders.drain(..) { - log_error!( - drain_sender.send(Ok(())).await, - "{prefix} Error sending successful Drain result: {e}" - ); - } + fn handle_start_result( + &mut self, + conn_res: ConnectorResult<()>, + prefix: impl std::fmt::Display, + ) { + if let Err(e) = conn_res.res { + error!( + "{prefix} Error starting Connector {conn}: {e}", + conn = conn_res.alias + ); + if self.state != State::Failed { + // only report failed upon the first connector failure + info!("{prefix} Failed."); + self.change_state(State::Failed); + } + } else if self.state == State::Initializing { + // report started flow if all connectors started + self.expected_starts = self.expected_starts.saturating_sub(1); + if self.expected_starts == 0 { + info!("{prefix} Started."); + self.change_state(State::Running); + } + } + } + + async fn handle_pause(&mut self, prefix: impl std::fmt::Display) -> Result<()> { + info!("{prefix} Pausing..."); + for source in self.connectors_start_to_end() { + source.pause().await?; + } + for pipeline in self.pipelines.values() { + pipeline.pause().await?; + } + self.change_state(State::Paused); + info!("{prefix} Paused."); + Ok(()) + } + + async fn handle_resume(&mut self, prefix: impl std::fmt::Display) -> Result<()> { + info!("{prefix} Resuming..."); + + for pipeline in self.pipelines.values() { + pipeline.resume().await?; + } + for sink in self.connectors_end_to_start() { + sink.resume().await?; + } + self.change_state(State::Running); + info!("{prefix} Resumed."); + Ok(()) + } + + async fn handle_stop( + &mut self, + mode: ShutdownMode, + prefix: impl std::fmt::Display, + ) -> Result> { + if self.has_connectors() { + if let ShutdownMode::Graceful = mode { + info!("{prefix} Draining..."); + + self.change_state(State::Draining); + // QUIESCENCE + // - send drain msg to all connectors + // - wait until + // a) all connectors are drained (means all pipelines in between are also drained) or + let mut drained = 0_usize; + for addr in self.connectors_start_to_end() { + if !log_error!( + addr.drain(self.drain_tx.clone()).await, + "{prefix} Error starting Draining Connector {addr:?}: {e}" + ) { + drained += 1; + } + } + self.expected_drains = drained; + } else { + info!("{prefix} Stopping..."); + let mut stopped = 0_usize; + for connector in self.connectors_end_to_start() { + if !log_error!( + connector.stop(self.stop_tx.clone()).await, + "{prefix} Error stopping connector {connector}: {e}" + ) { + stopped += 1; } } - MsgWrapper::StopResult(conn_res) => { - info!("[Flow::{}] Connector {} stopped.", &id, &conn_res.alias); + self.expected_stops = stopped; + for pipeline in self.pipelines.values() { + if let Err(e) = pipeline.stop().await { + error!("{prefix} Error stopping pipeline {pipeline:?}: {e}"); + } + } + } + // we continue the stop process in `handle_stop_result` + Ok(ControlFlow::Continue(())) + } else { + // stop the (senseless) pipelines, as there are no events flowing through them + for pipeline in self.pipelines.values() { + if let Err(e) = pipeline.stop().await { + error!("{prefix} Error stopping pipeline {pipeline:?}: {e}"); + } + } + self.change_state(State::Stopped); + // nothing to do, we can break right away + Ok(ControlFlow::Break(())) + } + } + + fn handle_stop_result( + &mut self, + conn_res: ConnectorResult<()>, + prefix: impl std::fmt::Display, + ) -> ControlFlow<()> { + info!("{prefix} Connector {} stopped.", &conn_res.alias); + + log_error!( + conn_res.res, + "{prefix} Error during Stopping in Connector {}: {e}", + &conn_res.alias + ); + + let old = self.expected_stops; + self.expected_stops = self.expected_stops.saturating_sub(1); + if self.expected_stops == 0 && old > 0 { + info!("{prefix} All connectors are stopped."); + // upon last stop we finally know we are stopped + self.change_state(State::Stopped); + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } + + async fn handle_drain_result( + &mut self, + conn_res: ConnectorResult<()>, + prefix: impl std::fmt::Display, + ) -> Result> { + info!("{prefix} Connector {} drained.", &conn_res.alias); + + log_error!( + conn_res.res, + "{prefix} Error during Draining in Connector {}: {e}", + &conn_res.alias + ); + + let old = self.expected_drains; + self.expected_drains = self.expected_drains.saturating_sub(1); + if self.expected_drains == 0 && old > 0 { + info!("{prefix} All connectors are drained."); + // upon last drain, finally do a stop + self.handle_stop(ShutdownMode::Forceful, prefix).await + } else { + Ok(ControlFlow::Continue(())) + } + } + fn change_state(&mut self, new_state: State) { + if self.state != new_state { + if let Some(senders) = self.state_change_senders.get_mut(&new_state) { + for sender in senders.drain(..) { log_error!( - conn_res.res, - "{prefix} Error during Draining in Connector {}: {e}", - &conn_res.alias + sender.send(Ok(())), + "Error notifying {new_state} state handler: {e:?}" ); - - let old = expected_stops; - expected_stops = expected_stops.saturating_sub(1); - if expected_stops == 0 && old > 0 { - info!("{prefix} All connectors are stopped."); - // upon last stop - for stop_sender in stop_senders.drain(..) { + } + } + self.state = new_state; + // upon failed state, error out all other listeners of valid non-failed states, as there is no recovery + if let State::Failed = new_state { + for state in &[State::Running, State::Paused, State::Draining] { + if let Some(senders) = self.state_change_senders.get_mut(state) { + for sender in senders.drain(..) { log_error!( - stop_sender.send(Ok(())).await, - "{prefix} Error sending successful Stop result: {e}" + sender.send(Err( + ErrorKind::FlowFailed(self.app_ctx.to_string()).into() + )), + "Error notifying {state} state handlers of failed state: {e:?}" ); } - break; - } - } - MsgWrapper::StartResult(conn_res) => { - if let Err(e) = conn_res.res { - error!( - "{prefix} Error starting Connector {conn}: {e}", - conn = conn_res.alias - ); - if state != State::Failed { - // only report failed upon the first connector failure - state = State::Failed; - info!("{prefix} Failed."); - } - } else if state == State::Initializing { - // report started flow if all connectors started - wait_for_start_responses = wait_for_start_responses.saturating_sub(1); - if wait_for_start_responses == 0 { - state = State::Running; - info!("{prefix} Started."); - } } } } } - info!("{prefix} Stopped."); - Result::Ok(()) - }); + } +} + +/// task handling flow instance control plane +#[allow(clippy::too_many_lines)] +fn spawn_task( + app_ctx: AppContext, + pipelines: HashMap, + connectors: &HashMap, + links: &[ConnectStmt], + deployment_type: DeploymentType, +) -> Addr { + let flow = RunningFlow::new(app_ctx, pipelines, connectors, deployment_type, links); + let addr = flow.addr(); + task::spawn(flow.run()); addr } +fn is_active_node(current_nodes: &[NodeId], slot: usize, node_id: NodeId) -> bool { + match current_nodes.get(slot) { + Some(selected) if *selected == node_id => true, + Some(_selected) => false, + None => { + error!(" Slot {slot} is out of bounds for membership {current_nodes:?}"); + false + } + } +} + #[cfg(test)] mod tests { use super::*; use crate::{connectors::ConnectorBuilder, instance}; - - use tremor_common::ids::{ConnectorIdGen, OperatorIdGen}; + use tremor_common::uids::{ConnectorUIdGen, OperatorUIdGen}; use tremor_script::{ast::DeployStmt, deploy::Deploy, FN_REGISTRY}; use tremor_value::literal; @@ -788,7 +1166,6 @@ mod tests { &self, _alias: &alias::Connector, _config: &ConnectorConfig, - _kill_switch: &KillSwitch, ) -> Result> { Ok(Box::new(FakeConnector { tx: self.tx.clone(), @@ -799,8 +1176,8 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn flow_spawn() -> Result<()> { - let mut operator_id_gen = OperatorIdGen::default(); - let mut connector_id_gen = ConnectorIdGen::default(); + let mut operator_id_gen = OperatorUIdGen::default(); + let mut connector_id_gen = ConnectorUIdGen::default(); let aggr_reg = tremor_script::aggr_registry(); let src = r#" define flow test @@ -840,20 +1217,31 @@ mod tests { let (connector_tx, mut connector_rx) = crate::channel::unbounded(); let builder = connector::FakeBuilder { tx: connector_tx }; known_connectors.insert(builder.connector_type(), Box::new(builder)); - let flow = Flow::start( + let ctx = AppContext { + id: AppFlowInstanceId::new("app", "test"), + ..AppContext::default() + }; + let flow = Flow::deploy( + ctx, deploy, &mut operator_id_gen, &mut connector_id_gen, &known_connectors, &kill_switch, + DeploymentType::AllNodes, ) .await?; + + let (start_tx, start_rx) = crate::channel::oneshot(); + flow.start(start_tx).await?; + start_rx.await??; + let connector = flow.get_connector("foo".to_string()).await?; - assert_eq!(String::from("test::foo"), connector.alias.to_string()); + assert_eq!(String::from("foo"), connector.alias.to_string()); let connectors = flow.get_connectors().await?; assert_eq!(1, connectors.len()); - assert_eq!(String::from("test::foo"), connectors[0].alias.to_string()); + assert_eq!(String::from("foo"), connectors[0].alias.to_string()); // assert the flow has started and events are flowing let event = connector_rx.recv().await.ok_or("empty")?; @@ -871,23 +1259,24 @@ mod tests { assert_eq!(instance::State::Running, report.status); assert_eq!(1, report.connectors.len()); - flow.pause().await?; + let (tx, rx) = oneshot(); + flow.pause(tx).await?; + rx.await??; let report = flow.report_status().await?; assert_eq!(instance::State::Paused, report.status); assert_eq!(1, report.connectors.len()); - flow.resume().await?; + let (tx, rx) = oneshot(); + flow.resume(tx).await?; + rx.await??; let report = flow.report_status().await?; assert_eq!(instance::State::Running, report.status); assert_eq!(1, report.connectors.len()); - // drain and stop the flow - let (tx, mut rx) = bounded(128); - flow.drain(tx.clone()).await?; - rx.recv().await.ok_or("empty")??; - + // stop the flow + let (tx, rx) = oneshot(); flow.stop(tx).await?; - rx.recv().await.ok_or("empty")??; + rx.await??; Ok(()) } } diff --git a/src/system/flow_supervisor.rs b/src/system/flow_supervisor.rs index fc2feee79e..a9bad8ba27 100644 --- a/src/system/flow_supervisor.rs +++ b/src/system/flow_supervisor.rs @@ -12,22 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::flow::Flow; -use super::KillSwitch; -use crate::system::DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT; -use crate::{ - channel::{bounded, Sender}, - errors::empty_error, -}; use crate::{ + channel::{bounded, OneShotSender, Sender}, connectors::{self, ConnectorBuilder, ConnectorType}, - log_error, -}; -use crate::{ errors::{Kind as ErrorKind, Result}, - qsize, + instance::IntendedState, + log_error, qsize, raft, + system::{ + flow::{AppContext, Flow}, + KillSwitch, DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, + }, }; -use hashbrown::{hash_map::Entry, HashMap}; +use std::collections::{hash_map::Entry, HashMap}; use tokio::{ sync::oneshot, task::{self, JoinHandle}, @@ -35,21 +31,38 @@ use tokio::{ }; use tremor_common::{ alias, - ids::{ConnectorIdGen, OperatorIdGen}, + uids::{ConnectorUIdGen, OperatorUIdGen}, }; use tremor_script::ast::DeployFlow; +use super::flow::DeploymentType; + pub(crate) type Channel = Sender; /// This is control plane #[derive(Debug)] pub(crate) enum Msg { - /// deploy a Flow - StartDeploy { - /// deploy flow + /// Deploy a flow, instantiate it, but does not start it or any child instance (connector, pipeline) + DeployFlow { + /// the App this flow shall be a part of + app: alias::App, + /// the `deploy flow` AST flow: Box>, /// result sender - sender: oneshot::Sender>, + sender: OneShotSender>, + /// API request sender + raft: raft::Cluster, + /// Type of the deployment + deployment_type: DeploymentType, + }, + /// change instance state + ChangeInstanceState { + /// unique ID for the `Flow` instance to start + id: alias::Flow, + /// The state the instance should be changed to + intended_state: IntendedState, + /// result sender + reply_tx: OneShotSender>, }, RegisterConnectorType { /// the type of connector @@ -57,19 +70,19 @@ pub(crate) enum Msg { /// the builder builder: Box, }, - GetFlows(oneshot::Sender>>), + GetFlows(OneShotSender>>), GetFlow(alias::Flow, oneshot::Sender>), /// Initiate the Quiescence process - Drain(oneshot::Sender>), + Drain(OneShotSender>), /// stop this manager - Stop, + Terminate, } #[derive(Debug)] pub(crate) struct FlowSupervisor { flows: HashMap, - operator_id_gen: OperatorIdGen, - connector_id_gen: ConnectorIdGen, + operator_id_gen: OperatorUIdGen, + connector_id_gen: ConnectorUIdGen, known_connectors: connectors::Known, } @@ -78,8 +91,8 @@ impl FlowSupervisor { Self { flows: HashMap::new(), known_connectors: connectors::Known::new(), - operator_id_gen: OperatorIdGen::new(), - connector_id_gen: ConnectorIdGen::new(), + operator_id_gen: OperatorUIdGen::new(), + connector_id_gen: ConnectorUIdGen::new(), } } @@ -93,26 +106,39 @@ impl FlowSupervisor { } } - async fn handle_start_deploy( + async fn handle_deploy( &mut self, + app_id: alias::App, flow: DeployFlow<'static>, - sender: oneshot::Sender>, + sender: oneshot::Sender>, kill_switch: &KillSwitch, + raft: raft::Cluster, + deployment_type: DeploymentType, ) { - let id = alias::Flow::from(&flow); + let id = alias::Flow::new(app_id, &flow.instance_alias); let res = match self.flows.entry(id.clone()) { Entry::Occupied(_occupied) => Err(ErrorKind::DuplicateFlow(id.to_string()).into()), - Entry::Vacant(vacant) => Flow::start( - flow, - &mut self.operator_id_gen, - &mut self.connector_id_gen, - &self.known_connectors, - kill_switch, - ) - .await - .map(|deploy| { - vacant.insert(deploy); - }), + Entry::Vacant(vacant) => { + let ctx = AppContext { + id: id.clone(), + raft, + ..AppContext::default() + }; + Flow::deploy( + ctx, + flow, + &mut self.operator_id_gen, + &mut self.connector_id_gen, + &self.known_connectors, + kill_switch, + deployment_type, + ) + .await + .map(|deploy| { + vacant.insert(deploy); + id + }) + } }; log_error!( sender.send(res).map_err(|_| "send error"), @@ -140,30 +166,27 @@ impl FlowSupervisor { ); } - async fn handle_stop(&self) -> Result<()> { + async fn handle_terminate(&mut self) -> Result<()> { info!("Stopping Manager ..."); if !self.flows.is_empty() { - // send stop to each deployment - let (tx, mut rx) = bounded(self.flows.len()); - let mut expected_stops: usize = 0; - for flow in self.flows.values() { - log_error!( - flow.stop(tx.clone()).await, + // drain the flows, we are stopping anyways, this is the last interaction with them + let mut rxs = Vec::with_capacity(self.flows.len()); + for (_, flow) in self.flows.drain() { + let (tx, rx) = crate::channel::oneshot(); + if !log_error!( + flow.stop(tx).await, "Failed to stop Deployment \"{alias}\": {e}", alias = flow.id() - ); - expected_stops += 1; + ) { + rxs.push(rx); + } } timeout( DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, task::spawn(async move { - while expected_stops > 0 { - log_error!( - rx.recv().await.ok_or_else(empty_error)?, - "Error during Stopping: {e}" - ); - expected_stops = expected_stops.saturating_sub(1); + for rx in rxs.drain(..) { + log_error!(rx.await?, "Error during Stopping: {e}"); } Result::Ok(()) }), @@ -172,7 +195,7 @@ impl FlowSupervisor { } Ok(()) } - async fn handle_drain(&self, sender: oneshot::Sender>) { + async fn handle_drain(&mut self, sender: oneshot::Sender>) { if self.flows.is_empty() { log_error!( sender.send(Ok(())).map_err(|_| "send error"), @@ -181,26 +204,26 @@ impl FlowSupervisor { } else { let num_flows = self.flows.len(); info!("Draining all {num_flows} Flows ..."); - let mut alive_flows = 0_usize; - let (tx, mut rx) = bounded(num_flows); - for (_, flow) in &self.flows { + let mut rxs = Vec::with_capacity(self.flows.len()); + for (_, flow) in self.flows.drain() { + let (tx, rx) = crate::channel::oneshot(); + if !log_error!( - flow.drain(tx.clone()).await, + flow.stop(tx).await, "Failed to drain Deployment \"{alias}\": {e}", alias = flow.id() ) { - alive_flows += 1; + rxs.push(rx); } } task::spawn(async move { - while alive_flows > 0 { - match rx.recv().await { - Some(Err(e)) => { + for rx in rxs.drain(..) { + match rx.await { + Ok(Err(e)) => { error!("Error during Draining: {e}"); } - None | Some(_) => {} + Ok(_) | Err(_) => {} }; - alive_flows -= 1; } info!("Flows drained."); sender.send(Ok(())).map_err(|_| "Failed to send reply")?; @@ -209,10 +232,40 @@ impl FlowSupervisor { } } + async fn handle_change_state( + &mut self, + id: alias::Flow, + intended_state: IntendedState, + reply_tx: OneShotSender>, + ) -> Result<()> { + if let IntendedState::Stopped = intended_state { + // we remove the flow as it won't be reachable anymore, once it is stopped + // keeping it around will lead to errors upon stopping + if let Some(flow) = self.flows.remove(&id) { + flow.stop(reply_tx).await?; + Ok(()) + } else { + reply_tx + .send(Err(ErrorKind::FlowNotFound(id.to_string()).into())) + .map_err(|_| "can't reply")?; + Err(ErrorKind::FlowNotFound(id.to_string()).into()) + } + } else if let Some(flow) = self.flows.get(&id) { + flow.change_state(intended_state, reply_tx).await?; + Ok(()) + } else { + reply_tx + .send(Err(ErrorKind::FlowNotFound(id.to_string()).into())) + .map_err(|_| "can't reply")?; + Err(ErrorKind::FlowNotFound(id.to_string()).into()) + } + } + pub fn start(mut self) -> (JoinHandle>, Channel, KillSwitch) { let (tx, mut rx) = bounded(qsize()); let kill_switch = KillSwitch(tx.clone()); let task_kill_switch = kill_switch.clone(); + let system_h = task::spawn(async move { while let Some(msg) = rx.recv().await { match msg { @@ -221,17 +274,41 @@ impl FlowSupervisor { builder, .. } => self.handle_register_connector_type(connector_type, builder), - Msg::StartDeploy { flow, sender } => { - self.handle_start_deploy(*flow, sender, &task_kill_switch) - .await; + Msg::DeployFlow { + app, + flow, + sender, + raft: raft_api_tx, + deployment_type, + } => { + self.handle_deploy( + app, + *flow, + sender, + &task_kill_switch, + raft_api_tx, + deployment_type, + ) + .await; } Msg::GetFlows(reply_tx) => self.handle_get_flows(reply_tx), Msg::GetFlow(id, reply_tx) => self.handle_get_flow(&id, reply_tx), - Msg::Stop => { - self.handle_stop().await?; + Msg::Terminate => { + self.handle_terminate().await?; break; } Msg::Drain(sender) => self.handle_drain(sender).await, + Msg::ChangeInstanceState { + id, + intended_state, + reply_tx, + } => { + // if an error happened here already, the reply_tx hasn't been sent anywhere so we need to send the error here + log_error!( + self.handle_change_state(id, intended_state, reply_tx).await, + "Error sending ChangeInstanceState reply: {e}" + ); + } } } info!("Manager stopped."); diff --git a/src/utils.rs b/src/utils.rs index 55d6d547da..342fcc418a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -28,6 +28,5 @@ pub fn hostname() -> String { #[must_use] pub(crate) fn task_id() -> String { - // tokio::task::try_id().map_or_else(|| String::from(""), |i| i.to_string()) String::from("") } diff --git a/src/version.rs b/src/version.rs index c2594275b9..3ffe5d2bc6 100644 --- a/src/version.rs +++ b/src/version.rs @@ -16,12 +16,8 @@ use rdkafka::util::get_rdkafka_version; /// Version of the tremor crate; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -#[cfg(not(debug_assertions))] /// Checks if a we are in a debug build -pub const DEBUG: bool = false; -#[cfg(debug_assertions)] -/// Checks if a we are in a debug build -pub const DEBUG: bool = true; +pub const DEBUG: bool = cfg!(debug_assertions); #[must_use] /// Provides formatting for "long" version name of build @@ -42,7 +38,7 @@ pub fn long_ver() -> String { /// Prints tremor and librdkafka version. pub fn print() { - eprintln!("tremor version: {}", long_ver().as_str()); + eprintln!("tremor version: {}", long_ver()); eprintln!("tremor instance: {}", instance!()); let (version_n, version_s) = get_rdkafka_version(); eprintln!("rd_kafka version: 0x{version_n:08x}, {version_s}"); @@ -50,7 +46,7 @@ pub fn print() { /// Logs tremor and librdkafka version. pub fn log() { - info!("tremor version: {}", long_ver().as_str()); + info!("tremor version: {}", long_ver()); let (version_n, version_s) = get_rdkafka_version(); info!("rd_kafka version: 0x{version_n:08x}, {version_s}"); } diff --git a/static/openapi.yaml b/static/openapi.yaml index 81a46ceaba..e06b37cd9f 100644 --- a/static/openapi.yaml +++ b/static/openapi.yaml @@ -81,12 +81,18 @@ paths: application/yaml: schema: $ref: '#/components/schemas/flows' - /v1/flows/{flow-id}: + /v1/flows/{app-id}/{flow-id}: parameters: + - name: app-id + in: path + required: true + description: The unique id of an app inside the runtime + schema: + type: string - name: flow-id in: path required: true - description: The unique id of the flow in the runtime + description: The unique id of the flow in the the app identified by `app_id` schema: type: string get: @@ -141,8 +147,14 @@ paths: '404': description: The flow 'flow-id' wasnt found. It is thus not deployed in the runtime. - /v1/flows/{flow-id}/connectors: + /v1/flows/{app-id}/{flow-id}/connectors: parameters: + - name: app-id + in: path + required: true + description: The unique id of an app inside the runtime + schema: + type: string - name: flow-id in: path required: true @@ -167,8 +179,14 @@ paths: $ref: '#/components/schemas/connectors' '404': description: The flow 'flow-id' wasnt found. It is thus not deployed in the runtime. - /v1/flows/{flow-id}/connectors/{connector-id}: + /v1/flows/{app-id}/{flow-id}/connectors/{connector-id}: parameters: + - name: app-id + in: path + required: true + description: The unique id of an app inside the runtime + schema: + type: string - name: flow-id in: path required: true diff --git a/tests/flow_error.rs b/tests/flow_error.rs index 50f9644b47..99b913d2d2 100644 --- a/tests/flow_error.rs +++ b/tests/flow_error.rs @@ -19,7 +19,7 @@ use tremor_script::{deploy::Deploy, errors::*, highlighter::Dumb, module::Manage fn parse(deploy: &str) -> tremor_script::Result { let aggr_reg = tremor_script::aggr_registry(); let reg = tremor_script::registry::registry(); - Deploy::parse(deploy, ®, &aggr_reg) + Deploy::parse(&deploy, ®, &aggr_reg) } macro_rules! test_cases { diff --git a/tests/flows.rs b/tests/flows.rs index 2c9c4bfc4b..63661593cd 100644 --- a/tests/flows.rs +++ b/tests/flows.rs @@ -14,17 +14,17 @@ use serial_test::serial; use std::io::prelude::*; use std::time::Duration; -use tremor_common::file; +use tremor_common::{alias, file}; use tremor_runtime::{ errors::*, - system::{World, WorldConfig}, + system::{Runtime, ShutdownMode, WorldConfig}, }; use tremor_script::{deploy::Deploy, module::Manager}; fn parse(deploy: &str) -> tremor_script::Result { let aggr_reg = tremor_script::aggr_registry(); let reg = tremor_script::registry::registry(); - Deploy::parse(deploy, ®, &aggr_reg) + Deploy::parse(&deploy, ®, &aggr_reg) } macro_rules! test_cases { @@ -52,15 +52,19 @@ macro_rules! test_cases { let config = WorldConfig{ debug_connectors: true, }; - let (world, h) = World::start(config).await?; + let (runtime, h) = Runtime::start(config).await?; + let app_id = alias::App::default(); for flow in deployable.iter_flows() { - world.start_flow(flow).await?; + let flow_alias = alias::Flow::new(app_id.clone(), flow.instance_alias.clone()); + runtime.deploy_flow(app_id.clone(), flow, DeploymentType::AllNodes).await?; + runtime.start_flow(flow_alias).await?; } + runtime.stop(ShutdownMode::Forceful).await?; // this isn't good tokio::time::timeout(Duration::from_secs(10), h).await???; }, otherwise => { - println!("Expected valid deployment file, compile phase, but got an unexpected error: {:?}", otherwise); + println!("Expected valid deployment file, compile phase, but got an unexpected error: {otherwise:?}"); assert!(false); } } diff --git a/tests/query.rs b/tests/query.rs index 1ac8d06578..e825bf9ed0 100644 --- a/tests/query.rs +++ b/tests/query.rs @@ -13,10 +13,10 @@ // limitations under the License. use pretty_assertions::assert_eq; use std::io::prelude::*; -use tremor_common::{file, ids::OperatorIdGen, ports::IN}; -use tremor_pipeline::ExecutableGraph; +use tremor_common::{file, ports::IN, uids::OperatorUIdGen}; use tremor_pipeline::{query::Query, EventOriginUri}; use tremor_pipeline::{Event, EventId}; +use tremor_pipeline::{ExecutableGraph, MetricsChannel}; use tremor_script::FN_REGISTRY; use serial_test::serial; @@ -27,9 +27,9 @@ use tremor_value::utils::sorted_serialize; fn to_pipe(query: String) -> Result { let aggr_reg = tremor_script::aggr_registry(); - let mut idgen = OperatorIdGen::new(); + let mut idgen = OperatorUIdGen::new(); let q = Query::parse(&query, &*FN_REGISTRY.read()?, &aggr_reg)?; - Ok(q.to_executable_graph(&mut idgen)?) + Ok(q.to_executable_graph(&mut idgen, &MetricsChannel::new(128))?) } macro_rules! test_cases { @@ -80,7 +80,7 @@ macro_rules! test_cases { ..Event::default() }; let mut r = vec![]; - pipeline.enqueue(IN, event, &mut r)?; + pipeline.enqueue(0, IN, event, &mut r)?; results.append(&mut r); } assert_eq!(results.len(), out_json.len(), "Number of events differ error"); diff --git a/tests/query_error.rs b/tests/query_error.rs index 3018e4f40f..528edfc25c 100644 --- a/tests/query_error.rs +++ b/tests/query_error.rs @@ -16,9 +16,9 @@ use regex::Regex; use serial_test::serial; use std::io::prelude::*; use std::path::Path; -use tremor_common::{file, ids::OperatorIdGen}; -use tremor_pipeline::query::Query; +use tremor_common::{file, uids::OperatorUIdGen}; use tremor_pipeline::ExecutableGraph; +use tremor_pipeline::{query::Query, MetricsChannel}; use tremor_runtime::errors::*; use tremor_script::highlighter::Dumb; use tremor_script::module::Manager; @@ -26,9 +26,9 @@ use tremor_script::FN_REGISTRY; fn to_executable_graph(query: &str) -> Result { let aggr_reg = tremor_script::aggr_registry(); - let mut idgen = OperatorIdGen::new(); - let q = Query::parse(query, &*FN_REGISTRY.read()?, &aggr_reg)?; - Ok(q.to_executable_graph(&mut idgen)?) + let mut idgen = OperatorUIdGen::new(); + let q = Query::parse(&query, &*FN_REGISTRY.read()?, &aggr_reg)?; + Ok(q.to_executable_graph(&mut idgen, &MetricsChannel::default())?) } macro_rules! test_cases { diff --git a/tests/query_runtime_error.rs b/tests/query_runtime_error.rs index 54c70213bc..c64050f1ec 100644 --- a/tests/query_runtime_error.rs +++ b/tests/query_runtime_error.rs @@ -14,10 +14,10 @@ use pretty_assertions::assert_eq; use std::io::prelude::*; use std::path::PathBuf; -use tremor_common::{file, ids::OperatorIdGen, ports::IN}; +use tremor_common::{file, ports::IN, uids::OperatorUIdGen}; -use tremor_pipeline::query::Query; use tremor_pipeline::ExecutableGraph; +use tremor_pipeline::{query::Query, MetricsChannel}; use tremor_pipeline::{Event, EventId}; use tremor_script::FN_REGISTRY; @@ -31,9 +31,9 @@ use tremor_value::utils::sorted_serialize; fn to_pipe(query: &str) -> Result { let aggr_reg = tremor_script::aggr_registry(); - let mut idgen = OperatorIdGen::new(); - let q = Query::parse(query, &*FN_REGISTRY.read()?, &aggr_reg)?; - Ok(q.to_executable_graph(&mut idgen)?) + let mut idgen = OperatorUIdGen::new(); + let q = Query::parse(&query, &*FN_REGISTRY.read()?, &aggr_reg)?; + Ok(q.to_executable_graph(&mut idgen, &MetricsChannel::new(128))?) } const TEST_DIR: &str = "tests/query_runtime_errors"; @@ -88,7 +88,7 @@ macro_rules! test_cases { }; let mut r = vec![]; // run the pipeline, if an error occurs, dont stop but check for equivalence with `error.txt` - match pipeline.enqueue(IN, event, &mut r) { + match pipeline.enqueue(0, IN, event, &mut r) { Err(PipelineError(PipelineErrorKind::Script(e), o)) => { if let Some(err) = err.as_ref() { let e = tremor_script::errors::Error(e, o); diff --git a/tests/script.rs b/tests/script.rs index cce0347cbd..dbfbfac259 100644 --- a/tests/script.rs +++ b/tests/script.rs @@ -65,7 +65,7 @@ macro_rules! test_cases { port: Some(23), scheme: "snot".into(), }; - let context = EventContext::new(id as u64, Some(&uri)); + let context = EventContext::new( id as u64, Some(&uri), 0); let mut meta = Value::from(Object::default()); match script.run(&context, AggrType::Tick, &mut json, &mut state, &mut meta) { Err(e) => { @@ -180,6 +180,7 @@ test_cases!( fold_array_initial_imut, fold_array_imut, record_add, + complex_for, nested_use_with_path, multi_use, for_comprehension_filter, diff --git a/tests/script_runtime_error.rs b/tests/script_runtime_error.rs index a219a7f67f..5d8433d4fa 100644 --- a/tests/script_runtime_error.rs +++ b/tests/script_runtime_error.rs @@ -17,8 +17,8 @@ use std::io::prelude::*; use tremor_common::file; use tremor_runtime::errors::*; use tremor_script::{ - highlighter::Dumb, module::Manager, prelude::*, utils::*, AggrType, EventContext, Script, - FN_REGISTRY, + highlighter::Dumb, module::Manager, prelude::*, utils::*, AggrType, Script, FN_REGISTRY, + NO_CONTEXT, }; use tremor_value::{Object, Value}; @@ -55,7 +55,7 @@ macro_rules! test_cases { let err = err.trim(); if let Some(mut json) = in_json.pop() { - let context = EventContext::new(0, None); + let context = NO_CONTEXT; let mut meta = Value::from(Object::default()); let mut state = Value::null(); let s = script.run(&context, AggrType::Tick, &mut json, &mut state, &mut meta); @@ -109,7 +109,7 @@ macro_rules! ignore_cases { let mut state = Value::null(); if let Some(mut json) = in_json.pop() { - let context = EventContext::new(0, None); + let context = NO_CONTEXT; let mut meta = Value::object(); let s = script.run(&context, AggrType::Tick, &mut json, &mut state, &mut meta); if let Err(e) = s { diff --git a/tests/scripts/complex_for/in b/tests/scripts/complex_for/in new file mode 100644 index 0000000000..8b032864c6 --- /dev/null +++ b/tests/scripts/complex_for/in @@ -0,0 +1,4 @@ +[1] +[1,2] +[1,2,3] +[1,2,3,4] \ No newline at end of file diff --git a/tests/scripts/complex_for/out b/tests/scripts/complex_for/out new file mode 100644 index 0000000000..236a46b0f6 --- /dev/null +++ b/tests/scripts/complex_for/out @@ -0,0 +1,4 @@ +[["one",1],["two",null],["three",null]] +[["one",1],["two",2],["three",null]] +[["one",1],["two",2],["three",3]] +[["one",1],["two",2],["three",3]] \ No newline at end of file diff --git a/tests/scripts/complex_for/script.tremor b/tests/scripts/complex_for/script.tremor new file mode 100644 index 0000000000..1c4a0a41fd --- /dev/null +++ b/tests/scripts/complex_for/script.tremor @@ -0,0 +1,5 @@ +let state = ["one", "two", "three"]; +for state of + case (idx, name) when present event[idx] => [name, event[idx]] + case (idx, name) => [name, null] +end diff --git a/tremor-api/.gitignore b/tremor-api/.gitignore deleted file mode 100644 index 3571a1684c..0000000000 --- a/tremor-api/.gitignore +++ /dev/null @@ -1 +0,0 @@ -coyote.html diff --git a/tremor-api/Cargo.toml b/tremor-api/Cargo.toml deleted file mode 100644 index 5df5d25b2c..0000000000 --- a/tremor-api/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -authors = ["The Tremor Team"] -description = "Tremor Api Callbacks" -edition = "2021" -license = "Apache-2.0" -name = "tremor-api" -version = "0.13.0-rc.16" - -[dependencies] -halfbrown = "0.2" -http-types = "2" -log = "0.4" -serde = "1" -serde_yaml = "0.9" -simd-json = "0.13" -tokio = { version = "1.32", features = ["full"] } -# we don't need sessions or cookies or shitty logging middleware -tide = { version = "0.16", default-features = false, features = ["h1-server"] } -tremor-pipeline = { version = "0.13.0-rc.16", path = "../tremor-pipeline" } -tremor-runtime = { version = "0.13.0-rc.16", path = "../" } -tremor-script = { version = "0.13.0-rc.16", path = "../tremor-script" } -tremor-value = { version = "0.13.0-rc.16", path = "../tremor-value" } -tremor-common = { version = "0.13.0-rc.16", path = "../tremor-common" } - -[dev-dependencies] -surf = { version = "2.3", default-features = false, features = [ - "h1-client-rustls", -] } -env_logger = "0.10" diff --git a/tremor-api/LICENSE b/tremor-api/LICENSE deleted file mode 100644 index f830e3aef4..0000000000 --- a/tremor-api/LICENSE +++ /dev/null @@ -1,72 +0,0 @@ -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of this License; and You must cause any modified files to carry prominent notices stating that You changed the files; and You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. - -Copyright 2018 Wayfair GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/tremor-api/src/api.rs b/tremor-api/src/api.rs deleted file mode 100644 index 8dc8f453f3..0000000000 --- a/tremor-api/src/api.rs +++ /dev/null @@ -1,541 +0,0 @@ -// Copyright 2022, The Tremor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// #![cfg_attr(coverage, no_coverage)] - -use std::time::Duration; - -use crate::errors::Error; -use http_types::{ - headers::{self, HeaderValue, ToHeaderValues}, - StatusCode, -}; -use serde::{Deserialize, Serialize}; -use tide::Response; -use tokio::task::JoinHandle; -use tremor_runtime::instance::State as InstanceState; -use tremor_runtime::system::World; - -pub mod flow; -pub mod model; -pub mod prelude; -pub mod status; -pub mod version; - -pub type Request = tide::Request; -pub type Result = std::result::Result; - -/// Default API timeout applied to operations triggered by the API. E.g. get flow status -pub const DEFAULT_API_TIMEOUT: Duration = Duration::from_secs(5); - -#[derive(Clone)] -pub struct State { - pub world: World, -} - -#[derive(Clone, Copy, Debug)] -pub enum ResourceType { - Json, - Yaml, - Trickle, - Troy, -} - -impl std::fmt::Display for ResourceType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.as_str().fmt(f) - } -} -impl ResourceType { - #[must_use] - pub fn as_str(self) -> &'static str { - match self { - Self::Yaml => "application/yaml", - Self::Json => "application/json", - Self::Trickle => "application/vnd.trickle", - Self::Troy => "application/vnd.troy", - } - } -} - -impl ToHeaderValues for ResourceType { - type Iter = std::iter::Once; - - fn to_header_values(&self) -> http_types::Result { - HeaderValue::from_bytes(self.as_str().as_bytes().to_vec()).map(std::iter::once) - } -} - -#[must_use] -pub fn content_type(req: &Request) -> Option { - match req - .header(&headers::CONTENT_TYPE) - .map(headers::HeaderValues::last) - .map(headers::HeaderValue::as_str) - { - Some("application/yaml") => Some(ResourceType::Yaml), - Some("application/json") => Some(ResourceType::Json), - Some("application/vnd.trickle") => Some(ResourceType::Trickle), - Some("application/vnd.troy") => Some(ResourceType::Troy), - _ => None, - } -} - -#[must_use] -pub fn accept(req: &Request) -> ResourceType { - // TODO implement correctly / RFC compliance - match req - .header(headers::ACCEPT) - .map(headers::HeaderValues::last) - .map(headers::HeaderValue::as_str) - { - Some("application/yaml") => ResourceType::Yaml, - Some("application/vnd.trickle") => ResourceType::Trickle, - Some("application/vnd.troy") => ResourceType::Troy, - _ => ResourceType::Json, - } -} - -pub fn serialize(t: ResourceType, d: &T, code: StatusCode) -> Result { - match t { - ResourceType::Yaml => Ok(Response::builder(code) - .header(headers::CONTENT_TYPE, t.as_str()) - .body(serde_yaml::to_string(d)?) - .build()), - ResourceType::Json => Ok(Response::builder(code) - .header(headers::CONTENT_TYPE, t.as_str()) - .body(simd_json::to_string(d)?) - .build()), - _ => Err(Error::new( - StatusCode::InternalServerError, - format!("Unsuported formatting: {t}"), - )), - } -} - -pub fn serialize_error(t: ResourceType, d: Error) -> Result { - match t { - ResourceType::Json | ResourceType::Yaml => serialize(t, &d, d.code), - // formatting errors as trickle does not make sense so for this - // fall back to the error's conversion into tide response - ResourceType::Trickle | ResourceType::Troy => Ok(d.into()), - } -} - -pub fn reply( - req: &Request, - result_in: T, - ok_code: StatusCode, -) -> Result { - serialize(accept(req), &result_in, ok_code) -} - -async fn handle_api_request< - G: std::future::Future>, - F: Fn(Request) -> G, ->( - req: Request, - handler_func: F, -) -> tide::Result { - let resource_type = accept(&req); - let path = req.url().path().to_string(); - let method = req.method(); - - let r = match handler_func(req).await { - Err(e) => { - error!("[API {method} {path}] Error: {e}"); - Err(e) - } - Ok(r) => Ok(r), - }; - - r.or_else(|api_error| { - serialize_error(resource_type, api_error).or_else(|e| Ok(Into::::into(e))) - }) -} - -/// server the tremor API in a separately spawned task -#[must_use] -pub fn serve(host: String, world: &World) -> JoinHandle> { - let mut v1_app = tide::Server::with_state(State { - world: world.clone(), - }); - v1_app - .at("/version") - .get(|r| handle_api_request(r, version::get)); - v1_app - .at("/status") - .get(|r| handle_api_request(r, status::get_runtime_status)); - v1_app - .at("/flows") - .get(|r| handle_api_request(r, flow::list_flows)); - v1_app - .at("/flows/:id") - .get(|r| handle_api_request(r, flow::get_flow)) - .patch(|r| handle_api_request(r, flow::patch_flow_status)); - v1_app - .at("/flows/:id/connectors") - .get(|r| handle_api_request(r, flow::get_flow_connectors)); - v1_app - .at("/flows/:id/connectors/:connector") - .get(|r| handle_api_request(r, flow::get_flow_connector_status)) - .patch(|r| handle_api_request(r, flow::patch_flow_connector_status)); - - let mut app = tide::Server::new(); - app.at("/v1").nest(v1_app); - - // spawn API listener - tokio::task::spawn(async move { - let res = app.listen(host).await; - warn!("API stopped."); - if let Err(e) = res { - error!("API Error: {}", e); - Err(e.into()) - } else { - Ok(()) - } - }) -} - -#[cfg(test)] -mod tests { - use http_types::Url; - use std::time::Instant; - use tokio::net::TcpListener; - use tremor_runtime::{ - errors::Result as RuntimeResult, - instance::State as InstanceState, - system::{ShutdownMode, WorldConfig}, - }; - use tremor_script::{aggr_registry, ast::DeployStmt, deploy::Deploy, FN_REGISTRY}; - use tremor_value::{prelude::*, value::StaticValue}; - - use crate::api::model::{ApiFlowStatusReport, PatchStatus}; - - use super::*; - - #[allow(clippy::too_many_lines)] // this is a test - #[tokio::test(flavor = "multi_thread")] - async fn test_api() -> RuntimeResult<()> { - let _: std::result::Result<_, _> = env_logger::try_init(); - let config = WorldConfig { - debug_connectors: true, - }; - let (world, world_handle) = World::start(config).await?; - - let free_port = { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let port = listener.local_addr()?.port(); - drop(listener); - port - }; - let host = format!("127.0.0.1:{free_port}"); - let api_handle = serve(host.clone(), &world); - info!("Listening on: {}", host); - - let src = r#" - define flow api_test - flow - define pipeline main - pipeline - select event from in into out; - end; - create pipeline main; - - define connector my_null from `null`; - create connector my_null; - - connect /connector/my_null to /pipeline/main; - connect /pipeline/main to /connector/my_null; - end; - deploy flow api_test; - "#; - let aggr_reg = aggr_registry(); - let deployable = Deploy::parse(&src, &*FN_REGISTRY.read()?, &aggr_reg)?; - let deploy = deployable - .deploy - .stmts - .into_iter() - .find_map(|stmt| match stmt { - DeployStmt::DeployFlowStmt(deploy_flow) => Some((*deploy_flow).clone()), - _other => None, - }) - .expect("No deploy in the given troy file"); - world.start_flow(&deploy).await?; - - // check the status endpoint - let start = Instant::now(); - let client: surf::Client = surf::Config::new() - .set_base_url(Url::parse(&format!("http://{host}/"))?) - .try_into() - .expect("Could not create surf client"); - - let mut body = client - .get("/v1/status") - .await? - .body_json::() - .await - .map(StaticValue::into_value); - while body.is_err() || !body.get_bool("all_running").unwrap_or_default() { - if start.elapsed() > Duration::from_secs(2) { - panic!("Timeout waiting for all flows to run: {body:?}"); - } else { - body = client - .get("/v1/status") - .await? - .body_json::() - .await - .map(StaticValue::into_value); - } - } - assert_eq!( - literal!({ - "flows": { - "running": 1_u64 - }, - "num_flows": 1_u64, - "all_running": true - }), - body? - ); - - // check the version endpoint - let body = client - .get("/v1/version") - .await? - .body_json::() - .await? - .into_value(); - assert!(body.contains_key("version")); - assert!(body.contains_key("debug")); - - // list flows - let body = client - .get("/v1/flows") - .await? - .body_json::>() - .await?; - assert_eq!(1, body.len()); - assert_eq!("api_test", body[0].alias.as_str()); - assert_eq!(InstanceState::Running, body[0].status); - assert_eq!(1, body[0].connectors.len()); - assert_eq!(String::from("my_null"), body[0].connectors[0]); - - // get flow - let res = client.get("/v1/flows/i_do_not_exist").await?; - assert_eq!(StatusCode::NotFound, res.status()); - - let body = client - .get("/v1/flows/api_test") - .await? - .body_json::() - .await?; - - assert_eq!("api_test", body.alias.as_str()); - assert_eq!(InstanceState::Running, body.status); - assert_eq!(1, body.connectors.len()); - assert_eq!(String::from("my_null"), body.connectors[0]); - - // patch flow status - let body = client - .patch("/v1/flows/api_test") - .body_json(&PatchStatus { - status: InstanceState::Paused, - })? - .await? - .body_json::() - .await?; - - assert_eq!("api_test", body.alias.as_str()); - assert_eq!(InstanceState::Paused, body.status); - assert_eq!(1, body.connectors.len()); - assert_eq!(String::from("my_null"), body.connectors[0]); - - // invalid patch - let mut res = client - .patch("/v1/flows/api_test") - .body_json(&PatchStatus { - status: InstanceState::Failed, - })? - .await?; - assert_eq!(StatusCode::BadRequest, res.status()); - res.body_bytes().await?; // consume the body - - // resume - let body = client - .patch("/v1/flows/api_test") - .body_json(&PatchStatus { - status: InstanceState::Running, - })? - .await? - .body_json::() - .await?; - - assert_eq!("api_test", body.alias.as_str()); - assert_eq!(InstanceState::Running, body.status); - assert_eq!(1, body.connectors.len()); - assert_eq!(String::from("my_null"), body.connectors[0]); - - // list flow connectors - let body = client - .get("/v1/flows/api_test/connectors") - .await? - .body_json::() - .await? - .into_value(); - assert_eq!( - literal!([ - { - "alias": "my_null", - "status": "running", - "connectivity": "connected", - "pipelines": { - "out": [ - { - "port": "in", - "alias": "main" - } - ], - "in": [ - { - "port": "out", - "alias": "main" - } - ] - } - } - ]), - body - ); - - // get flow connector - let mut res = client - .get("/v1/flows/api_test/connectors/i_do_not_exist") - .await?; - assert_eq!(StatusCode::NotFound, res.status()); - res.body_bytes().await?; //consume body - - let body = client - .get("/v1/flows/api_test/connectors/my_null") - .await? - .body_json::() - .await? - .into_value(); - assert_eq!( - literal!({ - "alias": "my_null", - "status": "running", - "connectivity": "connected", - "pipelines": { - "out": [ - { - "port": "in", - "alias": "main" - } - ], - "in": [ - { - "port": "out", - "alias": "main" - } - ] - } - }), - body - ); - - // patch flow connector status - let body = client - .patch("/v1/flows/api_test/connectors/my_null") - .body_json(&PatchStatus { - status: InstanceState::Paused, - })? - .await? - .body_json::() - .await? - .into_value(); - - assert_eq!( - literal!({ - "alias": "my_null", - "status": "paused", - "connectivity": "connected", - "pipelines": { - "out": [ - { - "port": "in", - "alias": "main" - } - ], - "in": [ - { - "port": "out", - "alias": "main" - } - ] - } - }), - body - ); - - // invalid patch - let mut res = client - .patch("/v1/flows/api_test/connectors/my_null") - .body_json(&PatchStatus { - status: InstanceState::Failed, - })? - .await?; - assert_eq!(StatusCode::BadRequest, res.status()); - res.body_bytes().await?; // consume the body - - // resume - let body = client - .patch("/v1/flows/api_test/connectors/my_null") - .body_json(&PatchStatus { - status: InstanceState::Running, - })? - .await? - .body_json::() - .await? - .into_value(); - assert_eq!( - literal!({ - "alias": "my_null", - "status": "running", - "connectivity": "connected", - "pipelines": { - "out": [ - { - "port": "in", - "alias": "main" - } - ], - "in": [ - { - "port": "out", - "alias": "main" - } - ] - } - }), - body - ); - - // cleanup - world.stop(ShutdownMode::Graceful).await?; - world_handle.abort(); - api_handle.abort(); - Ok(()) - } -} diff --git a/tremor-api/src/api/flow.rs b/tremor-api/src/api/flow.rs deleted file mode 100644 index 09bc85adc9..0000000000 --- a/tremor-api/src/api/flow.rs +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2022, The Tremor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Flow API - -use crate::{ - api::prelude::*, - model::{ApiConnectorStatusReport, ApiFlowStatusReport, PatchStatus}, -}; - -pub(crate) async fn list_flows(req: Request) -> Result { - let world = &req.state().world; - let flows = world.get_flows().await?; - let mut result: Vec = Vec::with_capacity(flows.len()); - for flow in flows { - let status = flow.report_status().await?; - result.push(status.into()); - } - reply(&req, result, StatusCode::Ok) -} - -pub(crate) async fn get_flow(req: Request) -> Result { - let world = &req.state().world; - let flow_id = req.param("id")?.to_string(); - let flow = world.get_flow(flow_id).await?; - let report = flow.report_status().await?; - reply(&req, ApiFlowStatusReport::from(report), StatusCode::Ok) -} - -pub(crate) async fn patch_flow_status(mut req: Request) -> Result { - let patch_status_payload: PatchStatus = req.body_json().await?; - let world = &req.state().world; - let flow_id = req.param("id")?.to_string(); - let flow = world.get_flow(flow_id.clone()).await?; - let current_status = flow.report_status().await?; - let report = match (current_status.status, patch_status_payload.status) { - (state1, state2) if state1 == state2 => { - // desired status == current status - current_status - } - (InstanceState::Running, InstanceState::Paused) => { - flow.pause().await?; - flow.report_status().await? - } - - (InstanceState::Paused, InstanceState::Running) => { - flow.resume().await?; - flow.report_status().await? - } - // TODO: we could stop a flow by patching its status to `Stopped` - (current, desired) => { - // throw error - return Err(Error::bad_request(format!( - "Cannot patch status of {flow_id} to from {current} to {desired}" - ))); - } - }; - reply(&req, ApiFlowStatusReport::from(report), StatusCode::Ok) -} - -pub(crate) async fn get_flow_connectors(req: Request) -> Result { - let world = &req.state().world; - let flow_id = req.param("id")?.to_string(); - let flow = world.get_flow(flow_id).await?; - let connectors = flow.get_connectors().await?; - let mut result: Vec = Vec::with_capacity(connectors.len()); - for connector in connectors { - let status = connector.report_status().await?; - result.push(status.into()); - } - reply(&req, result, StatusCode::Ok) -} - -pub(crate) async fn get_flow_connector_status(req: Request) -> Result { - let world = &req.state().world; - let flow_id = req.param("id")?.to_string(); - let connector_id = req.param("connector")?.to_string(); - let flow = world.get_flow(flow_id).await?; - - let connector = flow.get_connector(connector_id).await?; - let report = connector.report_status().await?; - reply(&req, ApiConnectorStatusReport::from(report), StatusCode::Ok) -} - -pub(crate) async fn patch_flow_connector_status(mut req: Request) -> Result { - let patch_status_payload: PatchStatus = req.body_json().await?; - let flow_id = req.param("id")?.to_string(); - let connector_id = req.param("connector")?.to_string(); - - let world = &req.state().world; - let flow = world.get_flow(flow_id.clone()).await?; - let connector = flow.get_connector(connector_id.clone()).await?; - let current_status = connector.report_status().await?; - let report = match (current_status.status(), patch_status_payload.status) { - (state1, state2) if *state1 == state2 => { - // desired status == current status - current_status - } - (InstanceState::Running, InstanceState::Paused) => { - connector.pause().await?; - connector.report_status().await? - } - - (InstanceState::Paused, InstanceState::Running) => { - connector.resume().await?; - connector.report_status().await? - } - // TODO: we could stop a deployment by patching its status to `Stopped` - (current, desired) => { - // throw error - return Err(Error::bad_request(format!("Cannot patch status of connector {connector_id} in flow {flow_id} from {current} to {desired}"))); - } - }; - reply(&req, ApiConnectorStatusReport::from(report), StatusCode::Ok) -} diff --git a/tremor-api/src/api/model.rs b/tremor-api/src/api/model.rs deleted file mode 100644 index 2d2f5c3b18..0000000000 --- a/tremor-api/src/api/model.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2022, The Tremor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::api::prelude::*; -use halfbrown::HashMap; -use tremor_common::alias; -use tremor_runtime::{ - connectors::{Connectivity, StatusReport as ConnectorStatusReport}, - instance::State, - system::flow::StatusReport as FlowStatusReport, -}; -use tremor_script::ast::DeployEndpoint; - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub(crate) struct ApiFlowStatusReport { - pub(crate) alias: alias::Flow, - pub(crate) status: State, - pub(crate) connectors: Vec, -} - -impl From for ApiFlowStatusReport { - fn from(sr: FlowStatusReport) -> Self { - Self { - alias: sr.alias, - status: sr.status, - connectors: sr - .connectors - .into_iter() - .map(|ca| ca.connector_alias().to_string()) - .collect(), - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub(crate) struct Pipeline { - alias: String, - port: String, -} - -impl From<&DeployEndpoint> for Pipeline { - fn from(de: &DeployEndpoint) -> Self { - Self { - alias: de.alias().to_string(), - port: de.port().to_string(), - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub(crate) struct ApiConnectorStatusReport { - pub(crate) alias: String, - pub(crate) status: State, - pub(crate) connectivity: Connectivity, - pub(crate) pipelines: HashMap>, -} - -impl From for ApiConnectorStatusReport { - fn from(csr: ConnectorStatusReport) -> Self { - Self { - alias: csr.alias().connector_alias().to_string(), - status: *csr.status(), - connectivity: *csr.connectivity(), - pipelines: csr - .pipelines() - .iter() - .map(|(k, v)| { - ( - k.to_string(), - v.iter().map(Pipeline::from).collect::>(), - ) - }) - .collect(), - } - } -} - -#[derive(Serialize, Deserialize, Debug)] -pub(crate) struct PatchStatus { - pub(crate) status: InstanceState, -} diff --git a/tremor-api/src/api/status.rs b/tremor-api/src/api/status.rs deleted file mode 100644 index 53e423e5ff..0000000000 --- a/tremor-api/src/api/status.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2022, The Tremor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Runtime status API - -use halfbrown::HashMap; -use tremor_runtime::instance::State; - -use crate::api::prelude::*; - -#[derive(Serialize, Debug)] -struct RuntimeStatus { - all_running: bool, - num_flows: usize, - flows: HashMap, -} - -impl RuntimeStatus { - fn new(num_flows: usize) -> Self { - Self { - num_flows, - ..Self::default() - } - } - fn add_flow(&mut self, status: State) { - *self.flows.entry(status).or_insert(0) += 1; - self.all_running = self.all_running && status == State::Running; - } -} - -impl Default for RuntimeStatus { - fn default() -> Self { - Self { - all_running: true, - num_flows: 0, - flows: HashMap::new(), - } - } -} - -pub(crate) async fn get_runtime_status(req: Request) -> Result { - let world = &req.state().world; - - let flows = world.get_flows().await?; - let mut runtime_status = RuntimeStatus::new(flows.len()); - let mut all_in_good_state: bool = true; - - for flow in &flows { - let status = flow.report_status().await?; - runtime_status.add_flow(status.status); - - // report OK if all flows are in a good/intended state (Running) - all_in_good_state = all_in_good_state - && matches!( - status.status, - State::Running | State::Paused | State::Initializing - ); - } - let code = if all_in_good_state { - StatusCode::Ok - } else { - StatusCode::ServiceUnavailable - }; - reply(&req, runtime_status, code) -} diff --git a/tremor-api/src/api/version.rs b/tremor-api/src/api/version.rs deleted file mode 100644 index f639e1e855..0000000000 --- a/tremor-api/src/api/version.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2020-2021, The Tremor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::api::prelude::*; -use tremor_runtime::version::{DEBUG, VERSION}; - -#[derive(Serialize, Deserialize)] -pub struct Version { - version: &'static str, - debug: bool, -} - -impl Default for Version { - fn default() -> Self { - Self { - version: VERSION, - debug: DEBUG, - } - } -} - -// ALLOW: We allow this since it's required for generalizing accept fuinctions -#[allow(clippy::unused_async)] -pub(crate) async fn get(req: Request) -> Result { - reply(&req, Version::default(), StatusCode::Ok) -} diff --git a/tremor-api/src/errors.rs b/tremor-api/src/errors.rs deleted file mode 100644 index 714c72785c..0000000000 --- a/tremor-api/src/errors.rs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2022, The Tremor Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use http_types::{headers, StatusCode}; -use serde::Serialize; -use std::sync::{MutexGuard, PoisonError}; -use tide::Response; -use tremor_runtime::errors::{Error as TremorError, Kind as ErrorKind}; - -/// Tremor API error -#[derive(Debug, Serialize)] -pub struct Error { - pub code: StatusCode, - pub error: String, -} - -impl Error { - /// Create new error from the given status code and error string - #[must_use] - pub fn new(code: StatusCode, error: String) -> Self { - Self { code, error } - } - - /// Convenience function for creating artefact not found errors - pub fn artefact_not_found() -> Self { - Self::new(StatusCode::NotFound, "Artefact not found".into()) - } - - /// Convenience function for creating instance not found errors - pub fn instance_not_found() -> Self { - Self::new(StatusCode::NotFound, "Instance not found".into()) - } - /// Convenience function for denoting bad requests - pub fn bad_request(msg: String) -> Self { - Self::new(StatusCode::BadRequest, msg) - } -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.error) - } -} -impl std::error::Error for Error {} - -impl From for Response { - fn from(err: Error) -> Response { - Response::builder(err.code) - .header(headers::CONTENT_TYPE, crate::api::ResourceType::Json) - .body(format!( - r#"{{"code":{},"error":"{}"}}"#, - err.code, err.error - )) - .build() - } -} - -impl From for Error { - fn from(e: std::io::Error) -> Self { - Self::new(StatusCode::InternalServerError, format!("IO error: {e}")) - } -} - -impl From for Error { - fn from(e: tremor_common::Error) -> Self { - Self::new(StatusCode::InternalServerError, format!("{e}")) - } -} - -impl From for Error { - fn from(_e: tokio::time::error::Elapsed) -> Self { - Self::new(StatusCode::InternalServerError, "Request timed out".into()) - } -} - -impl From for Error { - fn from(e: simd_json::Error) -> Self { - Self::new(StatusCode::BadRequest, format!("JSON error: {e}")) - } -} - -impl From for Error { - fn from(e: serde_yaml::Error) -> Self { - Self::new(StatusCode::BadRequest, format!("YAML error: {e}")) - } -} - -impl From for Error { - fn from(e: http_types::Error) -> Self { - Self::new( - StatusCode::InternalServerError, - format!("HTTP type error: {e}"), - ) - } -} - -impl From for Error { - fn from(e: tremor_pipeline::errors::Error) -> Self { - Self::new(StatusCode::BadRequest, format!("Pipeline error: {e}")) - } -} - -impl From>> for Error { - fn from(e: PoisonError>) -> Self { - Self::new( - StatusCode::InternalServerError, - format!("Locking error: {e}"), - ) - } -} - -impl From for Error { - fn from(e: TremorError) -> Self { - match e.0 { - ErrorKind::FlowNotFound(id) => { - Error::new(StatusCode::NotFound, format!("Flow {id} not found")) - } - ErrorKind::ConnectorNotFound(flow_id, id) => Error::new( - StatusCode::NotFound, - format!("Connector {id} not found in Flow {flow_id}"), - ), - _e => Error::new( - StatusCode::InternalServerError, - "Internal server error".into(), - ), - } - } -} diff --git a/tremor-cli/Cargo.toml b/tremor-cli/Cargo.toml index 296db27f91..3f82865d67 100644 --- a/tremor-cli/Cargo.toml +++ b/tremor-cli/Cargo.toml @@ -45,7 +45,6 @@ surf = { version = "=2.3.2", default-features = false, features = [ "h1-client-rustls", "middleware-logger", ] } -tremor-api = { version = "0.13.0-rc.16", path = "../tremor-api" } tremor-codec = { version = "0.13.0-rc.16", path = "../tremor-codec" } tremor-interceptor = { version = "0.13.0-rc.16", path = "../tremor-interceptor" } tremor-common = { version = "0.13.0-rc.16", path = "../tremor-common" } @@ -60,6 +59,8 @@ port_scanner = "0.1" shell-words = "1" tch = { version = "0.13", optional = true } termcolor = "1.2" +serde_json = "1.0" + [[bin]] name = "tremor" @@ -78,7 +79,6 @@ test = false # levarage features # -snmalloc = [] bert = ["tremor-runtime/bert", "tch"] default = [] stdalloc = [] diff --git a/tremor-cli/src/_template/lib/flows.tremor b/tremor-cli/src/_template/lib/flows.tremor deleted file mode 100644 index 76b00c60e9..0000000000 --- a/tremor-cli/src/_template/lib/flows.tremor +++ /dev/null @@ -1,12 +0,0 @@ -define flow main -flow - use lib::pipelines; - use tremor::connectors; - - create connector console from connectors::console; - create pipeline main from pipelines::main; - - connect /connector/console to /pipeline/main; - connect /pipeline/main to /connector/console; - -end; \ No newline at end of file diff --git a/tremor-cli/src/_template/main.troy b/tremor-cli/src/_template/main.troy index 305e992777..76b00c60e9 100644 --- a/tremor-cli/src/_template/main.troy +++ b/tremor-cli/src/_template/main.troy @@ -1,2 +1,12 @@ -use lib::flows; -deploy flow main from flows::main; \ No newline at end of file +define flow main +flow + use lib::pipelines; + use tremor::connectors; + + create connector console from connectors::console; + create pipeline main from pipelines::main; + + connect /connector/console to /pipeline/main; + connect /pipeline/main to /connector/console; + +end; \ No newline at end of file diff --git a/tremor-cli/src/cli.rs b/tremor-cli/src/cli.rs index 39a9021071..4cffa2353c 100644 --- a/tremor-cli/src/cli.rs +++ b/tremor-cli/src/cli.rs @@ -53,8 +53,198 @@ pub(crate) enum Command { #[clap( value_parser = clap::value_parser!(String))] name: String, }, + + /// cluster commands + Cluster(Cluster), +} + +#[derive(Parser, Debug)] +pub(crate) struct Cluster { + #[clap(subcommand)] + pub(crate) command: ClusterCommand, } +#[derive(Parser, Clone, Debug)] +pub(crate) enum ClusterCommand { + /// Bootstraps a new initial cluster + Bootstrap { + /// Database dir to store raft data in + #[clap(short, long, value_parser = clap::value_parser!(String))] + db_dir: String, + /// Raft RPC IP and endpoint to listen to + #[clap(short, long, value_parser = clap::value_parser!(String))] + rpc: String, + /// Raft API IP and endpoint to listen to + #[clap(short, long, value_parser = clap::value_parser!(String))] + api: String, + /// Removes the node from the cluster when it's shut down with SIGTERM + #[clap(long, action = clap::ArgAction::SetTrue)] + remove_on_sigterm: bool, + }, + /// Starts a cluster node that is part as a cluster already + Start { + /// Database dir to store raft data in + #[clap(short, long, value_parser = clap::value_parser!(String))] + db_dir: String, + /// optional Raft RPC IP and endpoint to listen to + /// if not provided, it will be read from the previously initialized cluster state + #[clap(short, long, value_parser = clap::value_parser!(String))] + rpc: Option, + /// optional Raft API IP and endpoint to listen to + /// if not provided, it will be read from the previously initialized cluster state + #[clap(short, long, value_parser = clap::value_parser!(String))] + api: Option, + /// Zero or more endpoints of the cluster to join + /// if no value is provided this node will not attempt to explicitly join any cluster + /// except the one it is already part of + #[clap(short, long, value_parser = clap::value_parser!(String))] + join: Vec, + /// If set to true the node will join the cluster as learner and not take part in voting + #[clap(long, action = clap::ArgAction::SetTrue)] + passive: bool, + /// Removes the node from the cluster when it's shut down with SIGTERM + #[clap(long, action = clap::ArgAction::SetTrue)] + remove_on_sigterm: bool, + }, + /// Removes a node from the cluster (the node still has to be stopped after this) + Remove { + /// Raft API IP and endpoint to send to + #[clap(short, long, value_parser = clap::value_parser!(String))] + api: Option, + /// Node to remove + #[clap(value_parser = clap::value_parser!(u64))] + node: u64, + }, + /// Displays the cluster status + Status { + /// Raft API IP and endpoint to send to + #[clap(short, long, value_parser = clap::value_parser!(String))] + api: Option, + /// Show JSON output instead of human readable information + #[clap(short, long, action = clap::ArgAction::SetTrue)] + json: bool, + }, + /// App related tommands + Apps { + /// Raft API IP and endpoint to send to, defaults to `TREMOR_API_ADDRESS` + #[clap(short, long, value_parser = clap::value_parser!(String))] + api: Option, + #[clap(subcommand)] + command: AppsCommands, + }, + /// Packages a tremor application + Package { + /// the file to load + #[clap(short, long, value_parser = clap::value_parser!(String))] + name: Option, + #[clap(short, long, value_parser = clap::value_parser!(String))] + out: String, + #[clap( value_parser = clap::value_parser!(String))] + entrypoint: String, + }, + Kv { + /// Raft API IP and endpoint to send to, defaults to `TREMOR_API_ADDRESS` + #[clap(short, long, value_parser = clap::value_parser!(String))] + api: Option, + #[clap(subcommand)] + command: KvCommands, + }, +} + +// TODO: do we want to split out start/stop/pause/resume into a own sub command +#[derive(Parser, Clone, Debug)] +pub(crate) enum AppsCommands { + /// lists all installed apps + List { + /// Show JSON output instead of human readable information + #[clap(short, long, action = clap::ArgAction::SetTrue)] + json: bool, + }, + /// Installs new app + Install { + /// the file to load + #[clap( value_parser = clap::value_parser!(String))] + file: String, + }, + /// Installs new app + Uninstall { + /// The app + #[clap(value_parser = clap::value_parser!(String))] + app: String, + }, + /// Starts an instance of a flow in an installed app + Start { + /// The app containing the flow + #[clap(value_parser = clap::value_parser!(String))] + app: String, + /// the instance of the flow to start + #[clap(value_parser = clap::value_parser!(String))] + instance: String, + /// The flow to start defaults to `main` + #[clap(long, short, value_parser = clap::value_parser!(String))] + flow: Option, + /// Configuration for the flow instance to start + #[clap(long, short, value_parser = clap::value_parser!(String))] + config: Option, + /// Deploys the pipeline in paused state + #[clap(short, long, action = clap::ArgAction::SetTrue)] + paused: bool, + /// Deploys the flow in single node mode, this means the flow will only run + /// on one node in the cluster and might be balanced to other nodes on resize + #[clap(short, long, action = clap::ArgAction::SetTrue)] + single_node: bool, + }, + + /// Stops and removes an instance of a flow in an installed app + Stop { + /// The app + #[clap(value_parser = clap::value_parser!(String))] + app: String, + /// the instance to stop + #[clap(value_parser = clap::value_parser!(String))] + instance: String, + }, + /// Pauses an instance of a flow in an installed app + Pause { + /// The app + #[clap(value_parser = clap::value_parser!(String))] + app: String, + /// the instance to stop + #[clap(value_parser = clap::value_parser!(String))] + instance: String, + }, + /// Resumes an instance of a flow in an installed app + Resume { + /// The app + #[clap(value_parser = clap::value_parser!(String))] + app: String, + /// the instance to stop + #[clap(value_parser = clap::value_parser!(String))] + instance: String, + }, +} + +#[derive(Parser, Clone, Debug)] +pub(crate) enum KvCommands { + /// Reads a value from the KV store + Get { + /// The key to get + #[clap(value_parser = clap::value_parser!(String))] + key: String, + /// Performs the read on the leader to guarnatee a non stale read + #[clap(short, long, action = clap::ArgAction::SetTrue)] + consistant: bool, + }, + /// Writes a value to the KV store + Set { + /// The key to set + #[clap(value_parser = clap::value_parser!(String))] + key: String, + /// The value to set (valid json) + #[clap(value_parser = clap::value_parser!(String))] + value: String, + }, +} /// Shell type #[derive(ValueEnum, Clone, Copy, Debug)] pub(crate) enum Shell { diff --git a/tremor-cli/src/cluster.rs b/tremor-cli/src/cluster.rs new file mode 100644 index 0000000000..1281ff9d42 --- /dev/null +++ b/tremor-cli/src/cluster.rs @@ -0,0 +1,402 @@ +// Copyright 2020-2021, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + cli::{AppsCommands, Cluster, ClusterCommand, KvCommands}, + errors::{Error, Result}, +}; +use futures::StreamExt; +use signal_hook::{ + consts::signal::{SIGINT, SIGQUIT, SIGTERM}, + low_level::signal_name, +}; +use signal_hook_tokio::Signals; +use simd_json::OwnedValue; +use std::{collections::HashMap, path::Path}; +use tokio::{io::AsyncReadExt, task}; +use tremor_common::{alias, asy::file}; +use tremor_runtime::{ + raft::{ + api::client::{print_metrics, Tremor as Client}, + archive, + node::{Addr, ClusterNodeKillSwitch, Node}, + remove_node, + store::TremorInstanceState, + ClusterError, NodeId, + }, + system::ShutdownMode, +}; + +/// returns either the defined API address or the default defined from the environment +/// variable `TREMOR_API_ADDRESS` +fn get_api(input: Option) -> Result { + input.map_or_else( + || Ok(std::env::var("TREMOR_API_ADDRESS").map_err(|e| e.to_string())?), + Ok, + ) +} + +async fn handle_signals( + signals: Signals, + kill_switch: ClusterNodeKillSwitch, + node_to_remove_on_term: Option<(NodeId, Addr)>, +) { + let mut signals = signals.fuse(); + + while let Some(signal) = signals.next().await { + info!( + "Received SIGNAL: {}", + signal_name(signal).unwrap_or(&signal.to_string()) + ); + match signal { + SIGTERM => { + // In k8s SIGTERM is sent to the container to indicate that it should shut down + // gracefully as part of auto scaling events. + // if we want tremor to auto-scale automatically we need to be able to join and + // leave clusters as part of the process. + // This is not always teh intended behaviour (read: only in k8s) so we have it as + // an option. + // It's noteworthy that as API adress we use the own nodes address, as it's not shut + // down yet it has the full cluster informaiton and we know for use it is up. + // Otherwise if we'd cache a API endpoint that endpoint might have been autoscaled + // away or replaced by now. + if let Some((node_id, ref addr)) = node_to_remove_on_term { + println!("SIGTERM received, removing {node_id} from cluster"); + if let Err(e) = remove_node(node_id, addr.api()).await { + error!("Cluster leave failed: {e}"); + }; + } + if let Err(_e) = kill_switch.stop(ShutdownMode::Graceful) { + if let Err(e) = signal_hook::low_level::emulate_default_handler(signal) { + error!("Error handling signal {}: {}", signal, e); + } + } + } + SIGINT => { + if let Err(_e) = kill_switch.stop(ShutdownMode::Graceful) { + if let Err(e) = signal_hook::low_level::emulate_default_handler(signal) { + error!("Error handling signal {}: {}", signal, e); + } + } + } + SIGQUIT => { + if let Err(_e) = kill_switch.stop(ShutdownMode::Forceful) { + if let Err(e) = signal_hook::low_level::emulate_default_handler(signal) { + error!("Error handling signal {}: {}", signal, e); + } + } + } + signal => { + if let Err(e) = signal_hook::low_level::emulate_default_handler(signal) { + error!("Error handling signal {}: {}", signal, e); + } + } + } + } +} + +impl Cluster { + /// Run the cluster command + pub(crate) async fn run(self) -> Result<()> { + match self.command { + // rm -r temp/test-db*; cargo run -p tremor-cli -- cluster bootstrap --db-dir temp/test-db1 --api 127.0.0.1:8001 --rpc 127.0.0.1:9001 + ClusterCommand::Bootstrap { + rpc, + api, + db_dir, + remove_on_sigterm, + } => { + // the bootstrap node always starts with a node_id of 0 + // it will be assigned a new one during bootstrap + let node_id = NodeId::default(); + info!("Boostrapping Tremor node with id {node_id}..."); + + let addr = Addr::new(api.clone(), rpc); + let mut node = Node::new(db_dir, tremor_runtime::raft::config()?); + let running_node = node.bootstrap_as_single_node_cluster(addr.clone()).await?; + + // install signal handler + let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; + let signal_handle = signals.handle(); + let node_to_remove_on_term = remove_on_sigterm.then_some((node_id, addr)); + + let signal_handler_task = task::spawn(handle_signals( + signals, + running_node.kill_switch(), + node_to_remove_on_term, + )); + + info!("Tremor node bootstrapped as single node cluster."); + // wait for the node to be finished + if let Err(e) = running_node.join().await { + error!("Error: {e}"); + } + info!("Tremor stopped."); + signal_handle.close(); + signal_handler_task.abort(); + } + // target/debug/tremor cluster start --db-dir temp/test-db2 --api 127.0.0.1:8002 --rpc 127.0.0.1:9002 --join 127.0.0.1:8001 + // target/debug/tremor cluster start --db-dir temp/test-db3 --api 127.0.0.1:8003 --rpc 127.0.0.1:9003 --join 127.0.0.1:8001 + ClusterCommand::Start { + db_dir, + rpc, + api, + join, + remove_on_sigterm, + passive, + } => { + // start db and statemachine + // start raft + // node_id assigned during bootstrap or join + let running_node = if Path::new(&db_dir).exists() + && !tremor_common::file::is_empty(&db_dir)? + { + if !join.is_empty() { + // TODO: check if join nodes are part of the known cluster, if so, don't error + return Err(Error::from( + "Cannot join another cluster with existing db directory", + )); + } + info!("Loading existing Tremor node state from {db_dir}."); + Node::load_from_store(&db_dir, tremor_runtime::raft::config()?).await? + } else { + // db dir does not exist + let rpc_addr = rpc.ok_or_else(|| ClusterError::from("missing rpc address"))?; + let api_addr = api.ok_or_else(|| ClusterError::from("missing api address"))?; + let addr = Addr::new(api_addr, rpc_addr); + + info!("Bootstrapping cluster node with addr {addr} and db_dir {db_dir}"); + let mut node = Node::new(&db_dir, tremor_runtime::raft::config()?); + + // attempt to join any one of the given endpoints, stop once we joined one + node.try_join(addr, join, !passive).await? + }; + + // install signal handler + let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; + let signal_handle = signals.handle(); + let node_to_remove_on_term = remove_on_sigterm.then_some(running_node.node_data()); + let signal_handler_task = task::spawn(handle_signals( + signals, + running_node.kill_switch(), + node_to_remove_on_term, + )); + + // wait for the node to be finished + if let Err(e) = running_node.join().await { + error!("Error: {e}"); + } + signal_handle.close(); + signal_handler_task.abort(); + } + ClusterCommand::Remove { node, api } => { + let api_addr = get_api(api)?; + + remove_node(node, &api_addr).await?; + } + // target/debug/tremor cluster status --api 127.0.0.1:8001 + ClusterCommand::Status { api, json } => { + let client = Client::new(&get_api(api)?)?; + let r = client.metrics().await.map_err(|e| format!("error: {e}"))?; + if json { + println!("{}", serde_json::to_string_pretty(&r)?); + } else { + print_metrics(&r); + } + } + ClusterCommand::Apps { api, command } => { + command.run(get_api(api)?.as_str()).await?; + } + ClusterCommand::Package { + name, + out, + entrypoint, + } => { + archive::package(&out, &entrypoint, name.clone()).await?; + } + ClusterCommand::Kv { api, command } => { + command.run(get_api(api)?.as_str()).await?; + } + } + Ok(()) + } +} + +impl AppsCommands { + pub(crate) async fn run(self, api: &str) -> Result<()> { + let client = Client::new(api)?; + match self { + AppsCommands::List { json } => { + let r = client.list().await.map_err(|e| format!("error: {e}"))?; + if json { + println!("{}", serde_json::to_string_pretty(&r)?); + } else { + for a in r.values() { + println!("{a}"); + } + } + } + AppsCommands::Install { file } => { + let mut file = file::open(&file).await?; + let mut buf = Vec::new(); + file.read_to_end(&mut buf).await?; + match client.install(&buf).await { + Ok(app_id) => println!("Application `{app_id}` successfully installed",), + Err(e) => eprintln!("Application failed to install: {e}"), + } + } + AppsCommands::Uninstall { app } => { + let app_id = alias::App(app); + match client.uninstall_app(&app_id).await { + Ok(app_id) => println!("App `{app_id}` successfully uninstalled",), + Err(e) => eprintln!("Application `{app_id}` failed to be uninstalled: {e}"), + } + } + AppsCommands::Start { + app, + flow, + instance, + config, + paused, + single_node, + } => { + let config: HashMap = + config.map_or_else(|| Ok(HashMap::new()), |c| serde_json::from_str(&c))?; + let flow = flow.map_or_else( + || alias::FlowDefinition::from("main"), + alias::FlowDefinition, + ); + let app_id = AppId(app); + let instance_id = alias::Flow::new(app_id, instance); + match client + .start(&flow, &instance_id, config, !paused, single_node) + .await + { + Ok(instance_id) => println!("Instance `{instance_id}` successfully started",), + Err(e) => eprintln!("Instance `{instance_id}` failed to start: {e}"), + } + } + AppsCommands::Stop { app, instance } => { + let instance_id = alias::Flow::new(app, instance); + match client.stop_instance(&instance_id).await { + Ok(instance_id) => println!("Instance `{instance_id}` stopped"), + Err(e) => eprintln!("Instance `{instance_id}` failed to stop: {e}"), + } + } + AppsCommands::Pause { app, instance } => { + let app_id = alias::App(app); + let instance_id = alias::Flow::new(app_id, instance); + match client + .change_instance_state(&instance_id, TremorInstanceState::Pause) + .await + { + Ok(instance_id) => println!("Instance `{instance_id}` successfully paused"), + Err(e) => eprintln!("Instance `{instance_id}` failed to pause: {e}"), + } + } + AppsCommands::Resume { app, instance } => { + let instance_id = alias::Flow::new(app, instance); + match client + .change_instance_state(&instance_id, TremorInstanceState::Resume) + .await + { + Ok(instance_id) => println!("Instance `{instance_id}` successfully resumed"), + Err(e) => eprintln!("Instance `{instance_id}` failed to resume: {e}"), + } + } + } + Ok(()) + } +} + +impl KvCommands { + async fn run(self, api: &str) -> Result<()> { + let client = Client::new(api)?; + match self { + KvCommands::Get { + key, + consistant: true, + } => { + let value = client.read(&key).await.map_err(|e| format!("error: {e}"))?; + println!("{}", simd_json::to_string_pretty(&value)?); + } + KvCommands::Get { key, .. } => { + let value = client + .consistent_read(&key) + .await + .map_err(|e| format!("error: {e}"))?; + println!("{}", simd_json::to_string_pretty(&value)?); + } + KvCommands::Set { key, mut value } => { + let v = unsafe { value.as_bytes_mut() }; + let value: OwnedValue = simd_json::from_slice(v)?; + let ts = tremor_runtime::raft::api::kv::KVSet { key, value }; + let r = client.write(&ts).await.map_err(|e| format!("error: {e}"))?; + println!("{}", simd_json::to_string_pretty(&r)?); + } + } + Ok(()) + } +} +// # Cluster +// +// ## Management Commands +// +// ### `tremor cluster init` +// initializes a new cluster, sets up the DB and promote itself to leader +// +// ### `tremor cluster join` +// joins an existing cluster, first becomes a learner then joins the cluster as a full member. +// +// ### `tremor cluster start` +// starts a cluster node that was already set/ +// +// ### `tremor cluster status` +// gets information from the cluster about it's current state +// +// ### `tremor cluster remove` +// removes a node from the cluster +// +// ## Packaging commands +// +// ### `tremor cluster package` +// packages a `.troy` file into a fully self contained tremor-application +// +// ### `tremor cluster download` --app my-app +// downloads a tremor application from th cluister +// +// ### `tremor cluster reinstall` --app my-app appv2.tar +// Updates an application / reinstalls it (for later versions of clustering) +// +// ## Application Commands +// +// ### `tremor cluster apps list` +// curl -v localhost:8001/api/apps +// +// ### `tremor cluster apps install` (or app.tar) +// installs a tremor-application into the cluster, but does not start any flows. +// +// ### `tremor cluster apps uninstall` --app my-app +// uninstalls a tremor-application from the cluster if it has no running instances. +// +// ### `tremor cluster apps start` --app my-app flowymcflowface from the_flow --args '{...}' +// deploys a flow from an app with a given set of arguments. (aka starts the appllication) +// curl -v -X POST localhost:8001/api/apps/tick/tick -d '{"instance": "i2", "config":{"interval":10000000000}}' +// +// ### `tremor cluster apps stop` --app my-app flowymcflowface +// undeploys a flow from an app. (aka stops the application) +// +// ### `tremor cluster apps pause` --app my-app flowymcflowface +// undeploys a flow from an app. (aka stops the application) +// diff --git a/tremor-cli/src/debug.rs b/tremor-cli/src/debug.rs index ecd43009f5..fa2f7e770d 100644 --- a/tremor-cli/src/debug.rs +++ b/tremor-cli/src/debug.rs @@ -23,7 +23,8 @@ use lexer::Lexer; use std::io::Write; use std::io::{self, Read}; use termcolor::{Color, ColorSpec}; -use tremor_common::ids::OperatorIdGen; +use tremor_common::uids::OperatorUIdGen; +use tremor_pipeline::MetricsChannel; use tremor_script::highlighter::{Dumb as TermNoHighlighter, Highlighter, Term as TermHighlighter}; use tremor_script::lexer::{self, Token}; use tremor_script::pos::{Span, Spanned}; @@ -263,9 +264,9 @@ impl DbgDot { let env = env::setup()?; match Query::parse(&data.raw, &env.fun, &env.aggr) { Ok(runnable) => { - let mut idgen = OperatorIdGen::new(); - let g = - tremor_pipeline::query::Query(runnable).to_executable_graph(&mut idgen)?; + let mut idgen = OperatorUIdGen::new(); + let g = tremor_pipeline::query::Query(runnable) + .to_executable_graph(&mut idgen, &MetricsChannel::default())?; println!("{}", g.dot); } diff --git a/tremor-cli/src/doc.rs b/tremor-cli/src/doc.rs index 3f38745df9..16c924a81a 100644 --- a/tremor-cli/src/doc.rs +++ b/tremor-cli/src/doc.rs @@ -18,8 +18,11 @@ use crate::util::visit_path_str; use std::ffi::OsStr; use std::io::Read; use std::path::{Path, PathBuf}; -use tremor_script::arena::Arena; -use tremor_script::module::{Id, Module}; +use tremor_script::{arena::Arena, ast::NodeId, module::PreCachedNodes}; +use tremor_script::{ + module::{Id, Module}, + NodeMeta, +}; fn push_line(line: &str, buf: &mut String) { buf.push_str(line); @@ -167,7 +170,14 @@ fn gen_doc( let module_id = Id::from(raw.as_bytes()); let mut ids = Vec::new(); let (aid, raw) = Arena::insert(&raw)?; - Module::load(module_id, &mut ids, aid, raw)? + Module::load( + NodeId::new("doc".to_string(), Vec::new(), NodeMeta::dummy()), + module_id, + &mut ids, + aid, + raw, + &PreCachedNodes::new(), + )? } Some(_) | None => return Ok(()), }; diff --git a/tremor-cli/src/errors.rs b/tremor-cli/src/errors.rs index a5849e08d2..6884becd5f 100644 --- a/tremor-cli/src/errors.rs +++ b/tremor-cli/src/errors.rs @@ -37,6 +37,11 @@ impl From> for Error { Self::from("Send Error") } } +impl From for Error { + fn from(e: tremor_runtime::raft::ClusterError) -> Self { + ErrorKind::ClusterError(e).into() + } +} error_chain! { links { @@ -49,6 +54,7 @@ error_chain! { foreign_links { Value(tremor_value::Error); YamlError(serde_yaml::Error) #[doc = "Error during yaml parsing"]; + SerdeJsonError(serde_json::Error) #[doc = "Error during json parsing"]; JsonError(simd_json::Error) #[doc = "Error during json parsing"]; Io(std::io::Error) #[doc = "Error during std::io"]; Fmt(std::fmt::Error) #[doc = "Error during std::fmt"]; @@ -62,6 +68,10 @@ error_chain! { JoinError(tokio::task::JoinError); } errors { + ClusterError(e: tremor_runtime::raft::ClusterError) { + description("Cluster Error") + display("Cluster Error: {}", e) + } TestFailures(stats: crate::test::stats::Stats) { description("Some tests failed") display("{} out of {} tests failed.\nFailed tests: {}", diff --git a/tremor-cli/src/main.rs b/tremor-cli/src/main.rs index 253c4284dc..4ad3cbed11 100644 --- a/tremor-cli/src/main.rs +++ b/tremor-cli/src/main.rs @@ -39,17 +39,15 @@ use tremor_common::file; // use tremor_runtime::errors; mod alloc; +pub(crate) mod cli; +mod cluster; mod completions; mod debug; mod doc; mod env; mod errors; -// mod explain; -pub(crate) mod cli; -mod report; mod run; mod server; -pub(crate) mod status; mod target_process; mod test; mod util; @@ -124,6 +122,7 @@ async fn run(cli: Cli) -> Result<()> { command.run().await; Ok(()) } + Command::Cluster(command) => command.run().await, Command::Test(t) => t.run().await, Command::Dbg(d) => d.run(), Command::Run(r) => r.run().await, @@ -134,7 +133,6 @@ async fn run(cli: Cli) -> Result<()> { fn create_template(mut path: PathBuf, name: &str) -> Result<()> { const MAIN_TROY: &str = include_str!("_template/main.troy"); - const FLOWS: &str = include_str!("_template/lib/flows.tremor"); const PIPELINES: &str = include_str!("_template/lib/pipelines.tremor"); const SCRIPTS: &str = include_str!("_template/lib/scripts.tremor"); @@ -148,15 +146,10 @@ fn create_template(mut path: PathBuf, name: &str) -> Result<()> { fs::create_dir(&lib_path)?; let mut file_path = path.clone(); - file_path.push("main.troy"); + file_path.push(format!("{name}.troy")); print!("."); fs::write(&file_path, MAIN_TROY)?; - let mut file_path = lib_path.clone(); - file_path.push("flows.tremor"); - print!("."); - fs::write(&file_path, FLOWS)?; - let mut file_path = lib_path.clone(); file_path.push("pipelines.tremor"); print!("."); @@ -169,7 +162,7 @@ fn create_template(mut path: PathBuf, name: &str) -> Result<()> { println!("done.\n"); println!( - "To run:\n TREMOR_PATH=\"${{TREMOR_PATH}}:${{PWD}}/{name}\" tremor run {name}/main.troy" + "To run:\n TREMOR_PATH=\"${{TREMOR_PATH}}:${{PWD}}/{name}\" tremor run {name}/{name}.troy" ); Ok(()) diff --git a/tremor-cli/src/run.rs b/tremor-cli/src/run.rs index 0d825dd0c6..5bceda5379 100644 --- a/tremor-cli/src/run.rs +++ b/tremor-cli/src/run.rs @@ -22,13 +22,13 @@ use std::io::{self, BufReader, BufWriter}; use tremor_codec::Codec; use tremor_common::{ file, - ids::OperatorIdGen, ports::{Port, IN}, time::nanotime, + uids::OperatorUIdGen, }; use tremor_interceptor::{postprocessor, preprocessor}; use tremor_pipeline::{Event, EventId}; -use tremor_runtime::system::{World, WorldConfig}; +use tremor_runtime::system::{Runtime, WorldConfig}; use tremor_script::{ arena::Arena, highlighter::{Error as HighlighterError, Highlighter, Term as TermHighlighter}, @@ -232,7 +232,7 @@ impl Run { let env = env::setup()?; let mut h = TermHighlighter::stderr(); - match Script::parse(raw, &env.fun) { + match Script::parse(&raw, &env.fun) { Ok(mut script) => { script.format_warnings_with(&mut h)?; @@ -249,7 +249,7 @@ impl Run { let mut global_map = Value::object(); let mut event = event.clone_static(); match runnable.run( - &EventContext::new(at, None), + &EventContext::new(at, None, 0), AggrType::Tick, &mut event, state, @@ -338,8 +338,8 @@ impl Run { let mut egress = Egress::from_args(self)?; let runnable = tremor_pipeline::query::Query(runnable); - let mut idgen = OperatorIdGen::new(); - let mut pipeline = runnable.to_executable_graph(&mut idgen)?; + let mut idgen = OperatorUIdGen::new(); + let mut pipeline = runnable.to_executable_graph(&mut idgen, &MetricsChannel::new(128))?; let id = 0_u64; ingress @@ -354,6 +354,7 @@ impl Run { let mut continuation = vec![]; if let Err(e) = runnable.enqueue( + 0, IN, Event { id: EventId::from_id(0, 0, *id), @@ -423,7 +424,7 @@ impl Run { let config = WorldConfig { debug_connectors: true, }; - let (world, handle) = World::start(config).await?; + let (world, handle) = Runtime::start(config).await?; tremor_runtime::load_troy_file(&world, &self.script).await?; handle.await??; Ok(()) diff --git a/tremor-cli/src/server.rs b/tremor-cli/src/server.rs index 7c1177394e..ac878cc0d0 100644 --- a/tremor-cli/src/server.rs +++ b/tremor-cli/src/server.rs @@ -17,15 +17,14 @@ use crate::{ cli::{ServerCommand, ServerRun}, errors::{Error, ErrorKind, Result}, }; -use futures::{future, StreamExt}; +use futures::StreamExt; use signal_hook::consts::signal::{SIGINT, SIGQUIT, SIGTERM}; use signal_hook::low_level::signal_name; use signal_hook_tokio::Signals; use std::io::Write; use std::sync::atomic::Ordering; -use tremor_api as api; use tremor_common::file; -use tremor_runtime::system::{ShutdownMode, World}; +use tremor_runtime::system::{Runtime, ShutdownMode}; use tremor_runtime::{self, version}; macro_rules! log_and_print_error { @@ -35,7 +34,7 @@ macro_rules! log_and_print_error { }; } -async fn handle_signals(mut signals: Signals, world: World) { +async fn handle_signals(mut signals: Signals, world: Runtime) { while let Some(signal) = signals.next().await { info!( "Received SIGNAL: {}", @@ -98,7 +97,7 @@ impl ServerRun { debug_connectors: self.debug_connectors, }; - let (world, handle) = World::start(config).await?; + let (world, handle) = Runtime::start(config).await?; // signal handling let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; @@ -125,36 +124,12 @@ impl ServerRun { } } - let api_handle = if self.no_api { - // dummy task never finishing - tokio::task::spawn(async move { - future::pending::<()>().await; - Ok(()) - }) - } else { - eprintln!("Listening at: http://{}", &self.api_host); - info!("Listening at: http://{}", &self.api_host); - api::serve(self.api_host.clone(), &world) - }; // waiting for either - match future::select(handle, api_handle).await { - future::Either::Left((manager_res, api_handle)) => { - // manager stopped - if let Err(e) = manager_res { - error!("Manager failed with: {}", e); - result = 1; - } - api_handle.abort(); - } - future::Either::Right((_api_res, manager_handle)) => { - // api stopped - if let Err(e) = world.stop(ShutdownMode::Graceful).await { - error!("Error shutting down gracefully: {}", e); - result = 2; - } - manager_handle.abort(); - } - }; + + if let Err(e) = handle.await { + error!("Manager failed with: {}", e); + result = 1; + } signal_handle.close(); signal_handler_task.abort(); warn!("Tremor stopped."); @@ -163,6 +138,7 @@ impl ServerRun { async fn run(&self) { version::print(); + match self.run_dun().await { Err(e) => { match e { diff --git a/tremor-cli/src/test.rs b/tremor-cli/src/test.rs index 216961686b..472440b69e 100644 --- a/tremor-cli/src/test.rs +++ b/tremor-cli/src/test.rs @@ -12,17 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::report; -use crate::status; -use crate::test::command::suite_command; use crate::util::{basename, slurp_string}; use crate::{ cli::Test, errors::{Error, ErrorKind, Result}, }; use crate::{cli::TestMode, target_process}; +use command::suite_command; use globwalk::{FileType, GlobWalkerBuilder}; use metadata::Meta; +use process::run_process; use std::collections::HashMap; use std::io::Write; use std::path::{Path, PathBuf}; @@ -38,7 +37,9 @@ mod before; mod command; mod metadata; mod process; +mod report; pub mod stats; +mod status; pub mod tag; mod unit; @@ -92,7 +93,7 @@ async fn run_bench( let cwd = std::env::current_dir()?; file::set_current_dir(&root)?; status::tags(&tags, Some(&matched), Some(&config.excludes))?; - let test_report = process::run_process( + let test_report = run_process( "bench", config.base_directory.as_path(), &cwd.join(&root), diff --git a/tremor-cli/src/test/assert.rs b/tremor-cli/src/test/assert.rs index 9aea874df1..8b25e8ea8d 100644 --- a/tremor-cli/src/test/assert.rs +++ b/tremor-cli/src/test/assert.rs @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::{report, status}; use crate::errors::{self, Result}; +use crate::open_file; use crate::test::stats; use crate::util::slurp_string; -use crate::{open_file, report, status}; use difference::Changeset; use errors::Error; use serde::{Deserialize, Deserializer}; diff --git a/tremor-cli/src/test/command.rs b/tremor-cli/src/test/command.rs index 55fb38c2c3..6edda21cc5 100644 --- a/tremor-cli/src/test/command.rs +++ b/tremor-cli/src/test/command.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::status; use crate::errors::{Error, Result}; -use crate::status; use crate::target_process; use crate::test::after; use crate::test::assert; diff --git a/tremor-cli/src/test/process.rs b/tremor-cli/src/test/process.rs index 65858dd2cf..229822ce66 100644 --- a/tremor-cli/src/test/process.rs +++ b/tremor-cli/src/test/process.rs @@ -12,18 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::status; -use super::after; -use super::assert; -use super::before; -use super::stats; -use super::tag; -use super::target_process; -use crate::report; -use crate::util::slurp_string; +use super::{ + after, assert, before, + report::{self, TestReport}, + stats, status, tag, target_process, +}; use crate::{ errors::{Error, Result}, - report::TestReport, + util::slurp_string, }; use futures::StreamExt; use globwalk::GlobWalkerBuilder; diff --git a/tremor-cli/src/report.rs b/tremor-cli/src/test/report.rs similarity index 100% rename from tremor-cli/src/report.rs rename to tremor-cli/src/test/report.rs diff --git a/tremor-cli/src/test/stats.rs b/tremor-cli/src/test/stats.rs index 054844850a..2718f6331b 100644 --- a/tremor-cli/src/test/stats.rs +++ b/tremor-cli/src/test/stats.rs @@ -1,5 +1,3 @@ -use crate::report::StatusKind; - // Copyright 2020-2021, The Tremor Team // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +11,7 @@ use crate::report::StatusKind; // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +use super::report::StatusKind; /// Test statistics #[derive(Serialize, Debug, Clone, Default)] diff --git a/tremor-cli/src/status.rs b/tremor-cli/src/test/status.rs similarity index 100% rename from tremor-cli/src/status.rs rename to tremor-cli/src/test/status.rs diff --git a/tremor-cli/src/test/unit.rs b/tremor-cli/src/test/unit.rs index 7e554a1d6b..ae7048eec7 100644 --- a/tremor-cli/src/test/unit.rs +++ b/tremor-cli/src/test/unit.rs @@ -12,27 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::TestConfig; -use crate::errors::{Error, Result}; -use crate::test; -use crate::test::stats; -use crate::test::status; -use crate::{env, report}; -use report::TestSuite; -use std::fmt::Write as _; // import without risk of name clashing -use std::io::Read; -use std::{collections::HashMap, path::Path}; -use test::tag; +use super::{ + report::{self, TestSuite}, + stats, status, tag, TestConfig, +}; +use crate::{ + env, + errors::{Error, Result}, + test, +}; +use std::{collections::HashMap, fmt::Write as _, io::Read, path::Path}; use tremor_common::time::nanotime; -use tremor_script::highlighter::{Dumb as DumbHighlighter, Highlighter, Term as TermHighlighter}; -use tremor_script::interpreter::{AggrType, Env, ExecOpts, LocalStack}; -use tremor_script::prelude::*; use tremor_script::{ ast::{Expr, ImutExpr, Invoke, List, Record}, - NO_AGGRS, + ctx::EventContext, + highlighter::{Dumb as DumbHighlighter, Highlighter, Term as TermHighlighter}, + interpreter::{AggrType, Env, ExecOpts, LocalStack}, + prelude::*, + NO_AGGRS, NO_CONSTS, }; -use tremor_script::{ctx::EventContext, NO_CONSTS}; use tremor_value::Value; + const EXEC_OPTS: ExecOpts = ExecOpts { result_needed: true, aggr: AggrType::Tick, @@ -234,7 +234,7 @@ pub(crate) fn run_suite( let script = runnable.script; - let context = &EventContext::new(nanotime(), None); + let context = &EventContext::new(nanotime(), None, 0); let env = Env { context, consts: NO_CONSTS.run(), diff --git a/tremor-cli/tests/integration/system-metrics/config.troy b/tremor-cli/tests/integration/system-metrics/config.troy index fa60bfedee..4845bfcef5 100644 --- a/tremor-cli/tests/integration/system-metrics/config.troy +++ b/tremor-cli/tests/integration/system-metrics/config.troy @@ -9,6 +9,7 @@ flow use integration; use std::time::nanos; use tremor::connectors; + use tremor::pipelines; # connector definitions define connector in from metronome @@ -78,15 +79,15 @@ flow end; # update state counters match event of - case matched = %{ tags ~= %{`pipeline` == "main1::main1", port == "in" }, fields ~= %{ present count } } => + case matched = %{ tags ~= %{`pipeline` == "default/main1::main1", port == "in" }, fields ~= %{ present count } } => let state.pipeline_in = matched.fields.count - case matched = %{ tags ~= %{ `pipeline` == "main1::main1", port == "out"}, fields ~= %{ present count } } => + case matched = %{ tags ~= %{ `pipeline` == "default/main1::main1", port == "out"}, fields ~= %{ present count } } => let state.pipeline_out = matched.fields.count - case matched = %{ tags ~= %{ `connector` == "main1::in_here", port == "out" }, fields ~= %{ present count } } => + case matched = %{ tags ~= %{ `connector` == "default/main1::in_here", port == "out" }, fields ~= %{ present count } } => let state.source_out = matched.fields.count - case matched = %{ tags ~= %{ `connector` == "main1::in_here", port == "err" }, fields ~= %{ present count } } => + case matched = %{ tags ~= %{ `connector` == "default/main1::in_here", port == "err" }, fields ~= %{ present count } } => let state.source_err = matched.fields.count - case matched = %{ tags ~= %{ `connector` == "main1::out_here", port == "in" }, fields ~= %{ present count } } => + case matched = %{ tags ~= %{ `connector` == "default/main1::out_here", port == "in" }, fields ~= %{ present count } } => let state.sink_in = matched.fields.count case _ => null end; diff --git a/tremor-codec/src/lib.rs b/tremor-codec/src/lib.rs index 795b01345f..294ead488b 100644 --- a/tremor-codec/src/lib.rs +++ b/tremor-codec/src/lib.rs @@ -129,9 +129,9 @@ pub fn resolve(config: &Config) -> Result> { "binflux" => Ok(Box::::default()), "csv" => Ok(Box::new(csv::Csv {})), "dogstatsd" => Ok(Box::::default()), + "influx" => Ok(Box::new(influx::Influx {})), "json" => json::from_config(config.config.as_ref()), "msgpack" => Ok(Box::new(msgpack::MsgPack {})), - "influx" => Ok(Box::new(influx::Influx {})), "null" => Ok(Box::new(null::Null {})), "statsd" => Ok(Box::::default()), "string" => Ok(Box::new(string::String {})), diff --git a/tremor-common/Cargo.toml b/tremor-common/Cargo.toml index aa58b09efd..f3b5627f93 100644 --- a/tremor-common/Cargo.toml +++ b/tremor-common/Cargo.toml @@ -23,3 +23,4 @@ lazy_static = "*" [dev-dependencies] tokio = { version = "1", features = ["full"] } test-case = "3" +tempfile = { version = "3.2" } diff --git a/tremor-common/src/alias.rs b/tremor-common/src/alias.rs index 8dbd46a4b2..95ec302c0b 100644 --- a/tremor-common/src/alias.rs +++ b/tremor-common/src/alias.rs @@ -14,63 +14,237 @@ use serde::{Deserialize, Serialize}; -/// unique identifier of a flow instance within a tremor instance +/// Types of aliases +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] +pub enum Type { + /// a connector + Connector, + /// a pipeline + Pipeline, +} + +impl std::fmt::Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Connector => write!(f, "connector"), + Self::Pipeline => write!(f, "pipeline"), + } + } +} + +/// generic behaviour of an alias +pub trait Generic { + /// the type if the alias + fn alias_type(&self) -> Type; + + /// the alias itself + fn alias(&self) -> &str; +} + +#[derive(Clone, Debug)] +/// Precomputed prefix for logging for Soiurce context +pub struct SourceContext(String); + +impl SourceContext { + /// construct a new `ConnectorId` from the id of the containing flow and the connector instance id + pub fn new(alias: impl Into) -> Self { + Self(alias.into()) + } +} +impl std::fmt::Display for SourceContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// An instance +#[derive(Serialize, Deserialize, Debug, Clone, Hash, Eq, PartialEq, PartialOrd)] +pub struct Instance(pub String); +impl std::fmt::Display for Instance { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for Instance { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for Instance { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +impl From<&String> for Instance { + fn from(value: &String) -> Self { + Self(value.to_string()) + } +} +impl From<&Instance> for Instance { + fn from(value: &Instance) -> Self { + value.clone() + } +} + +impl From<&FlowDefinition> for Instance { + fn from(value: &FlowDefinition) -> Self { + Self(value.0.clone()) + } +} +impl From for Instance { + fn from(value: FlowDefinition) -> Self { + Self(value.0) + } +} + +/// An `App` is an isolated container that is defined by +/// a troy file with possibly multiple flow definitions. +/// An `App` needs to have a unique name inside a tremor cluster. +/// Flow instances (and thus connector and pipeline instances) are spawned in the context +/// of an app and thus can have similar aliases/ids +#[derive(Serialize, Deserialize, Debug, Clone, Hash, Eq, PartialEq, PartialOrd)] +pub struct App(pub String); +impl std::fmt::Display for App { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for App { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for App { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +/// This default implementation should not be used in the clustered context +impl Default for App { + fn default() -> Self { + Self("default".to_string()) + } +} + +/// Identifier of a Flow definition +#[derive(Debug, PartialEq, PartialOrd, Eq, Hash, Clone, Serialize, Deserialize)] +pub struct FlowDefinition(pub String); +impl std::fmt::Display for FlowDefinition { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for FlowDefinition { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From for String { + fn from(value: FlowDefinition) -> Self { + value.0 + } +} + +impl From<&FlowDefinition> for String { + fn from(value: &FlowDefinition) -> Self { + value.0.clone() + } +} +impl From<&str> for FlowDefinition { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +impl From<&String> for FlowDefinition { + fn from(value: &String) -> Self { + Self(value.to_string()) + } +} + +impl From<&Instance> for FlowDefinition { + fn from(value: &Instance) -> Self { + FlowDefinition(value.0.clone()) + } +} +impl From for FlowDefinition { + fn from(value: Instance) -> Self { + FlowDefinition(value.0) + } +} + +/// Unique identifier of a `Flow` instance within a tremor cluster +/// A flow instance is always part of an `App` and thus always needs an `alias::App` to be fully qualified. +/// The `Flow` id needs to be unique within the App, regardless of the flow definition this instance is based upon. +/// An actual running instance of a flow #[derive(Debug, PartialEq, PartialOrd, Eq, Hash, Clone, Serialize, Deserialize)] -pub struct Flow(String); +pub struct Flow { + app_id: App, + instance_id: Instance, +} impl Flow { /// construct a new flow if from some stringy thingy - pub fn new(alias: impl Into) -> Self { - Self(alias.into()) + pub fn new(app_id: impl Into, instance_id: impl Into) -> Self { + Self { + app_id: app_id.into(), + instance_id: instance_id.into(), + } } - /// reference this id as a stringy thing again + /// Returns a reference to the appid #[must_use] - pub fn as_str(&self) -> &str { - self.0.as_str() + pub fn app_id(&self) -> &App { + &self.app_id + } + + /// returns a reference to the alias + #[must_use] + pub fn instance_id(&self) -> &Instance { + &self.instance_id } } impl From<&str> for Flow { - fn from(e: &str) -> Self { - Self(e.to_string()) + fn from(alias: &str) -> Self { + Self::new(App::default(), alias) } } impl From for Flow { fn from(alias: String) -> Self { - Self(alias) + Self::new(App::default(), alias) } } impl std::fmt::Display for Flow { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) + write!(f, "{}/{}", &self.app_id, &self.instance_id) } } /// unique instance alias/id of a connector within a deployment #[derive(Debug, PartialEq, PartialOrd, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Connector { - flow_alias: Flow, connector_alias: String, } impl Connector { /// construct a new `ConnectorId` from the id of the containing flow and the connector instance id - pub fn new(flow_alias: impl Into, connector_alias: impl Into) -> Self { + pub fn new(connector_alias: impl Into) -> Self { Self { - flow_alias: flow_alias.into(), connector_alias: connector_alias.into(), } } - /// get a reference to the flow alias - #[must_use] - pub fn flow_alias(&self) -> &Flow { - &self.flow_alias - } - /// get a reference to the connector alias #[must_use] pub fn connector_alias(&self) -> &str { @@ -80,35 +254,57 @@ impl Connector { impl std::fmt::Display for Connector { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}::{}", self.flow_alias, self.connector_alias) + self.connector_alias.fmt(f) + } +} + +impl From<&T> for Connector +where + T: ToString + ?Sized, +{ + fn from(s: &T) -> Self { + Self { + connector_alias: s.to_string(), + } + } +} + +impl Generic for Connector { + fn alias_type(&self) -> Type { + Type::Connector + } + + fn alias(&self) -> &str { + &self.connector_alias } } /// unique instance alias/id of a pipeline within a deployment #[derive(Debug, PartialEq, PartialOrd, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Pipeline { - flow_alias: Flow, pipeline_alias: String, } impl Pipeline { /// construct a new `Pipeline` from the id of the containing flow and the pipeline instance id - pub fn new(flow_alias: impl Into, pipeline_alias: impl Into) -> Self { + pub fn new(pipeline_alias: impl Into) -> Self { Self { - flow_alias: flow_alias.into(), pipeline_alias: pipeline_alias.into(), } } +} +impl Generic for Pipeline { + fn alias_type(&self) -> Type { + Type::Pipeline + } - /// get a reference to the Pipeline alias - #[must_use] - pub fn pipeline_alias(&self) -> &str { - self.pipeline_alias.as_str() + fn alias(&self) -> &str { + &self.pipeline_alias } } impl std::fmt::Display for Pipeline { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}::{}", self.flow_alias, self.pipeline_alias) + self.alias().fmt(f) } } diff --git a/tremor-common/src/asy/file.rs b/tremor-common/src/asy/file.rs index b6366d46f7..134c624d49 100644 --- a/tremor-common/src/asy/file.rs +++ b/tremor-common/src/asy/file.rs @@ -77,6 +77,20 @@ where }) } +/// A wrapper around `fs::canonicalize` that will give a better error (including the filename) +/// +/// # Errors +/// * if the path does not exist +pub async fn read_to_string(path: &S) -> Result +where + S: AsRef + ?Sized, +{ + tokio::fs::read_to_string(path).await.map_err(|e| { + let p: &Path = path.as_ref(); + Error::FileOpen(e, p.to_string_lossy().to_string()) + }) +} + pub use crate::file::extension; #[cfg(test)] diff --git a/tremor-common/src/base64.rs b/tremor-common/src/base64.rs index d76d288a62..081690687c 100644 --- a/tremor-common/src/base64.rs +++ b/tremor-common/src/base64.rs @@ -15,13 +15,30 @@ use base64::alphabet::STANDARD as STANDARD_ALPHABET; use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}; pub use base64::{DecodeError, Engine}; -/** - * Our very own base64 engine, that produces base64 with padding, but accepts base64 with and without padding. - * `BASE64` doesn't allow decoding without padding. - */ +/// Our very own base64 engine, that produces base64 with padding, but accepts base64 with and without padding. +/// `BASE64` doesn't allow decoding without padding. pub const BASE64: GeneralPurpose = GeneralPurpose::new( &STANDARD_ALPHABET, GeneralPurposeConfig::new() .with_encode_padding(true) .with_decode_padding_mode(DecodePaddingMode::Indifferent), ); + +/// Encode a byte slice into a base64 string. +#[must_use] +pub fn encode(data: T) -> String +where + T: AsRef<[u8]>, +{ + BASE64.encode(data) +} + +/// Decode a base64 string into a byte vector. +/// # Errors +/// Returns an error if the input is not valid base64. +pub fn decode(data: T) -> Result, base64::DecodeError> +where + T: AsRef<[u8]>, +{ + BASE64.decode(data) +} diff --git a/tremor-common/src/file.rs b/tremor-common/src/file.rs index 72c3522f18..723642d24e 100644 --- a/tremor-common/src/file.rs +++ b/tremor-common/src/file.rs @@ -68,7 +68,7 @@ where /// * if the file couldn't be created pub fn set_current_dir(path: &S) -> Result<(), Error> where - S: AsRef, + S: AsRef + ?Sized, { std::env::set_current_dir(path).map_err(|e| { let p: &Path = path.as_ref(); @@ -80,10 +80,28 @@ pub fn extension(path: &str) -> Option<&str> { Path::new(path).extension().and_then(OsStr::to_str) } +/// Returns true if the directory exists and is empty +/// +/// # Errors +/// +/// if the given path is not a directory or cannot be read or does not exist etc etc etc +pub fn is_empty(dir: &D) -> Result +where + D: AsRef + ?Sized, +{ + let dir = dir.as_ref(); + Ok(dir + .read_dir() + .map_err(|e| Error::FileOpen(e, dir.display().to_string()))? + .next() + .is_none()) +} + #[cfg(test)] mod test { #![allow(clippy::unwrap_used)] use crate::errors::Error; + use std::{fs::File, io::Write, path::PathBuf}; #[test] fn set_current_dir() -> Result<(), Error> { @@ -141,4 +159,33 @@ mod test { assert!(matches!(err, Error::FileOpen(_, bad) if bad == p)); Ok(()) } + + #[test] + fn is_empty_does_not_exist() -> Result<(), std::io::Error> { + let dir = tempfile::tempdir()?; + let mut dir_buf = PathBuf::from(dir.path()); + dir_buf.push("non-existent"); + assert!(super::is_empty(&dir_buf).is_err()); + Ok(()) + } + + #[test] + fn is_empty() -> Result<(), std::io::Error> { + let dir = tempfile::tempdir()?; + assert!(super::is_empty(&dir).expect("Expected empty dir to be empty")); + Ok(()) + } + + #[test] + fn is_not_empty() -> Result<(), std::io::Error> { + let dir = tempfile::tempdir()?; + let mut buf = PathBuf::from(dir.path()); + buf.push("element"); + let mut f = File::create(&buf)?; + f.write_all(b"snot")?; + f.flush()?; + drop(f); + assert!(!super::is_empty(&dir).expect("Expected dir to not be empty")); + Ok(()) + } } diff --git a/tremor-common/src/lib.rs b/tremor-common/src/lib.rs index 1eea6c2996..485d302195 100644 --- a/tremor-common/src/lib.rs +++ b/tremor-common/src/lib.rs @@ -31,8 +31,8 @@ pub mod file; /// Time related functions pub mod time; -/// common id handling -pub mod ids; +/// common uid handling +pub mod uids; /// Random numbers pub mod rand; diff --git a/tremor-common/src/ids.rs b/tremor-common/src/uids.rs similarity index 74% rename from tremor-common/src/ids.rs rename to tremor-common/src/uids.rs index 3fe3fb8b6f..b8bbea35d1 100644 --- a/tremor-common/src/ids.rs +++ b/tremor-common/src/uids.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -// common id handling +//! Collection of common unique numeric identifiers for internal use. +//! Those are more efficient than stupid strings. use std::{marker::PhantomData, ops::Deref}; -/// operator id +/// operator uid #[derive( Debug, PartialEq, @@ -30,9 +31,9 @@ use std::{marker::PhantomData, ops::Deref}; simd_json_derive::Serialize, simd_json_derive::Deserialize, )] -pub struct OperatorId(u64); +pub struct OperatorUId(u64); -impl Id for OperatorId { +impl UId for OperatorUId { fn new(id: u64) -> Self { Self(id) } @@ -42,13 +43,13 @@ impl Id for OperatorId { } } -impl std::fmt::Display for OperatorId { +impl std::fmt::Display for OperatorUId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } } -impl std::str::FromStr for OperatorId { +impl std::str::FromStr for OperatorUId { type Err = std::num::ParseIntError; fn from_str(s: &str) -> Result { @@ -57,7 +58,7 @@ impl std::str::FromStr for OperatorId { } } -/// connector id +/// connector uid #[derive( Debug, PartialEq, @@ -71,8 +72,8 @@ impl std::str::FromStr for OperatorId { simd_json_derive::Serialize, simd_json_derive::Deserialize, )] -pub struct ConnectorId(u64); -impl Id for ConnectorId { +pub struct ConnectorUId(u64); +impl UId for ConnectorUId { fn new(id: u64) -> Self { Self(id) } @@ -82,13 +83,13 @@ impl Id for ConnectorId { } } -impl std::fmt::Display for ConnectorId { +impl std::fmt::Display for ConnectorUId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } } -impl Deref for ConnectorId { +impl Deref for ConnectorUId { type Target = u64; fn deref(&self) -> &Self::Target { @@ -96,7 +97,7 @@ impl Deref for ConnectorId { } } -impl AsRef for ConnectorId { +impl AsRef for ConnectorUId { fn as_ref(&self) -> &u64 { &self.0 } @@ -116,16 +117,16 @@ impl AsRef for ConnectorId { simd_json_derive::Serialize, simd_json_derive::Deserialize, )] -pub struct SinkId(ConnectorId); -impl From for SinkId { - fn from(cid: ConnectorId) -> Self { +pub struct SinkUId(ConnectorUId); +impl From for SinkUId { + fn from(cid: ConnectorUId) -> Self { Self(cid) } } -impl Id for SinkId { +impl UId for SinkUId { fn new(id: u64) -> Self { - Self(ConnectorId::new(id)) + Self(ConnectorUId::new(id)) } fn id(&self) -> u64 { @@ -133,7 +134,7 @@ impl Id for SinkId { } } -impl std::fmt::Display for SinkId { +impl std::fmt::Display for SinkUId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Sink({})", self.0) } @@ -153,11 +154,11 @@ impl std::fmt::Display for SinkId { simd_json_derive::Serialize, simd_json_derive::Deserialize, )] -pub struct SourceId(ConnectorId); +pub struct SourceUId(ConnectorUId); -impl Id for SourceId { +impl UId for SourceUId { fn new(id: u64) -> Self { - Self(ConnectorId::new(id)) + Self(ConnectorUId::new(id)) } fn id(&self) -> u64 { @@ -165,19 +166,19 @@ impl Id for SourceId { } } -impl From for SourceId { - fn from(cid: ConnectorId) -> Self { +impl From for SourceUId { + fn from(cid: ConnectorUId) -> Self { Self(cid) } } -impl std::fmt::Display for SourceId { +impl std::fmt::Display for SourceUId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Source({})", self.0) } } -/// Identifier trait used everywhere within tremor -pub trait Id { +/// Unique numeric Identifier trait used everywhere within tremor +pub trait UId { /// constructor from a unique integer fn new(id: u64) -> Self; @@ -187,12 +188,12 @@ pub trait Id { #[derive(Debug)] /// id generator -pub struct IdGen { +pub struct UIdGen { current: u64, _marker: PhantomData, } -impl IdGen { +impl UIdGen { #[must_use] /// constructor pub fn new() -> Self { @@ -208,7 +209,7 @@ impl IdGen { } } -impl Default for IdGen { +impl Default for UIdGen { fn default() -> Self { Self::new() } @@ -216,9 +217,9 @@ impl Default for IdGen { /// operator id generator - generates consecutive u64 values /// this one will be shared by all pipelines - so we ensure unique operator ids -pub type OperatorIdGen = IdGen; +pub type OperatorUIdGen = UIdGen; /// connector id generator - generates consecutive u64 values -pub type ConnectorIdGen = IdGen; +pub type ConnectorUIdGen = UIdGen; #[cfg(test)] mod tests { @@ -227,8 +228,8 @@ mod tests { #[test] fn id_gen() { - let mut idgen = IdGen::::default(); - let ids: Vec = std::iter::repeat_with(|| idgen.next_id()) + let mut idgen = UIdGen::::default(); + let ids: Vec = std::iter::repeat_with(|| idgen.next_id()) .take(100) .collect(); @@ -244,8 +245,8 @@ mod tests { #[test] fn id_can_be_created_from_string() { - let id = OperatorId::from_str("1024"); + let id = OperatorUId::from_str("1024"); - assert_eq!(Ok(OperatorId(1024)), id); + assert_eq!(Ok(OperatorUId(1024)), id); } } diff --git a/tremor-erl/src/tremor_binding.erl b/tremor-erl/src/tremor_binding.erl deleted file mode 100644 index bdf4e3da57..0000000000 --- a/tremor-erl/src/tremor_binding.erl +++ /dev/null @@ -1,77 +0,0 @@ --module(tremor_binding). - --export([ - example/0, - list/1, - find/2, - publish/2, - unpublish/2 - ]). - --xref_ignore([ - example/0, - list/3, - find/2, - publish/2, - unpublish/2 - ]). - --define(ENDPOINT, "binding"). - - - -%%-------------------------------------------------------------------- -%% @doc -%% @end -%%-------------------------------------------------------------------- --spec list(tremor_api:connection()) -> - {ok, JSON :: binary()}. - -list(C) -> - tremor_http:get(?ENDPOINT, C). - - -%%-------------------------------------------------------------------- -%% @doc -%% @end -%%-------------------------------------------------------------------- --spec find(ID :: binary(), tremor_api:connection()) -> - {ok, JSON :: binary()}. - -find(UUID, C) -> - tremor_http:get([?ENDPOINT, $/, UUID], C). - - - -%%-------------------------------------------------------------------- -%% @doc -%% @end -%%-------------------------------------------------------------------- --spec publish(Spec :: map(), tremor_api:connection()) -> - {error, duplicate} | - {error, bad_content} | - ok. - -publish(Spec, C) -> - tremor_http:post([?ENDPOINT], Spec, C). - -%%-------------------------------------------------------------------- -%% @doc -%% @end -%%-------------------------------------------------------------------- --spec unpublish(ID :: map(), tremor_api:connection()) -> - {error, duplicate} | - {error, bad_content} | - ok. - -unpublish(Id, C) -> - tremor_http:delete([?ENDPOINT, $/, Id], [], C). - -example() -> - #{ - id => <<"test">>, - type => <<"file">>, - config => #{ - source => <<"file.txt">> - } - }. diff --git a/tremor-erl/src/tremor_flow.erl b/tremor-erl/src/tremor_flow.erl index 77e160a1e8..83783ebc05 100644 --- a/tremor-erl/src/tremor_flow.erl +++ b/tremor-erl/src/tremor_flow.erl @@ -16,6 +16,7 @@ -define(API_VERSION, "v1"). -define(ENDPOINT, "flows"). +-define(APP, "default"). -export([ list/1, get/2, @@ -43,7 +44,7 @@ list(Conn) -> %%------------------------------------------------------------------------ -spec get(Alias :: binary(), tremor_api:connection()) -> {ok, JSON::binary()}. get(Alias, Conn) -> - tremor_http:get([?API_VERSION, $/, ?ENDPOINT, $/, Alias], Conn). + tremor_http:get([?API_VERSION, $/, ?ENDPOINT, $/, ?APP, $/, Alias], Conn). %%------------------------------------------------------------------------ @@ -53,7 +54,7 @@ get(Alias, Conn) -> %%------------------------------------------------------------------------ -spec pause(Alias :: binary(), tremor_api:connection()) -> {ok, JSON::binary()}. pause(Alias, Conn) -> - tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, Alias], #{ status => <<"paused">> }, Conn). + tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, ?APP, $/, Alias], #{ status => <<"paused">> }, Conn). %%------------------------------------------------------------------------ %% @doc @@ -62,7 +63,7 @@ pause(Alias, Conn) -> %%------------------------------------------------------------------------ -spec resume(Alias :: binary(), tremor_api:connection()) -> {ok, JSON::binary()}. resume(Alias, Conn) -> - tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, Alias], #{ status => <<"running">> }, Conn). + tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, ?APP, $/, Alias], #{ status => <<"running">> }, Conn). %%------------------------------------------------------------------------ @@ -72,7 +73,7 @@ resume(Alias, Conn) -> %%------------------------------------------------------------------------ -spec list_connectors(FlowAlias :: binary(), tremor_api:connection()) -> {ok, JSON::binary()}. list_connectors(FlowAlias, Conn) -> - tremor_http:get([?API_VERSION, $/, ?ENDPOINT, $/, FlowAlias , $/, <<"connectors">>], Conn). + tremor_http:get([?API_VERSION, $/, ?ENDPOINT, $/, ?APP, $/, FlowAlias , $/, <<"connectors">>], Conn). %%------------------------------------------------------------------------ %% @doc @@ -82,7 +83,7 @@ list_connectors(FlowAlias, Conn) -> %%------------------------------------------------------------------------ -spec get_connector(FlowAlias :: binary(), ConnectorAlias :: binary(), tremor_api:connection()) -> {ok, JSON::binary()}. get_connector(FlowAlias, ConnectorAlias, Conn) -> - tremor_http:get([?API_VERSION, $/, ?ENDPOINT, $/, FlowAlias, $/, <<"connectors">>, $/, ConnectorAlias], Conn). + tremor_http:get([?API_VERSION, $/, ?ENDPOINT, $/, ?APP, $/, FlowAlias, $/, <<"connectors">>, $/, ConnectorAlias], Conn). %%------------------------------------------------------------------------ %% @doc @@ -92,7 +93,7 @@ get_connector(FlowAlias, ConnectorAlias, Conn) -> %%------------------------------------------------------------------------ -spec pause_connector(FlowAlias :: binary(), ConnectorAlias :: binary(), tremor_api:connection()) -> {ok, JSON::binary()}. pause_connector(FlowAlias, ConnectorAlias, Conn) -> - tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, FlowAlias, $/, <<"connectors">>, $/, ConnectorAlias], #{status => <<"paused">>}, Conn). + tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, ?APP, $/, FlowAlias, $/, <<"connectors">>, $/, ConnectorAlias], #{status => <<"paused">>}, Conn). %%------------------------------------------------------------------------ %% @doc @@ -102,4 +103,4 @@ pause_connector(FlowAlias, ConnectorAlias, Conn) -> %%------------------------------------------------------------------------ -spec resume_connector(FlowAlias :: binary(), ConnectorAlias :: binary(), tremor_api:connection()) -> {ok, JSON::binary()}. resume_connector(FlowAlias, ConnectorAlias, Conn) -> - tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, FlowAlias, $/, <<"connectors">>, $/, ConnectorAlias], #{status => <<"running">>}, Conn). + tremor_http:patch([?API_VERSION, $/, ?ENDPOINT, $/, ?APP, $/, FlowAlias, $/, <<"connectors">>, $/, ConnectorAlias], #{status => <<"running">>}, Conn). diff --git a/tremor-interceptor/src/postprocessor/base64.rs b/tremor-interceptor/src/postprocessor/base64.rs index 920e38a02d..436abe0d6c 100644 --- a/tremor-interceptor/src/postprocessor/base64.rs +++ b/tremor-interceptor/src/postprocessor/base64.rs @@ -16,7 +16,7 @@ use super::Postprocessor; use crate::errors::Result; -use tremor_common::base64::{Engine, BASE64}; +use tremor_common::base64; #[derive(Default)] pub(crate) struct Base64 {} @@ -26,6 +26,6 @@ impl Postprocessor for Base64 { } fn process(&mut self, _ingres_ns: u64, _egress_ns: u64, data: &[u8]) -> Result>> { - Ok(vec![BASE64.encode(data).as_bytes().to_vec()]) + Ok(vec![base64::encode(data).as_bytes().to_vec()]) } } diff --git a/tremor-interceptor/src/preprocessor.rs b/tremor-interceptor/src/preprocessor.rs index 65b95b0b0a..74ed580e8a 100644 --- a/tremor-interceptor/src/preprocessor.rs +++ b/tremor-interceptor/src/preprocessor.rs @@ -30,7 +30,7 @@ pub(crate) mod prelude { use self::prelude::*; use crate::errors::Result; use log::error; -use tremor_common::alias::Connector as Alias; +use tremor_common::alias; /// Configuration for a preprocessor pub type Config = tremor_config::NameWithConfig; @@ -126,16 +126,16 @@ pub fn preprocess( ingest_ns: &mut u64, data: Vec, meta: Value<'static>, - alias: &Alias, + prefix: &alias::SourceContext, ) -> Result, Value<'static>)>> { let mut data = vec![(data, meta)]; let mut data1 = Vec::new(); for pp in preprocessors { - for (i, (d, m)) in data.drain(..).enumerate() { + for (d, m) in data.drain(..) { match pp.process(ingest_ns, &d, m) { Ok(mut r) => data1.append(&mut r), Err(e) => { - error!("[Connector::{alias}] Preprocessor [{i}] error: {e}"); + error!("{prefix}[{}] Preprocessor error: {e}", pp.name()); return Err(e); } } @@ -152,16 +152,13 @@ pub fn preprocess( /// * If a preprocessor failed pub fn finish( preprocessors: &mut [Box], - alias: &Alias, + prefix: &alias::SourceContext, ) -> Result, Value<'static>)>> { if let Some((head, tail)) = preprocessors.split_first_mut() { let mut data = match head.finish(None, None) { Ok(d) => d, Err(e) => { - error!( - "[Connector::{alias}] Preprocessor '{}' finish error: {e}", - head.name() - ); + error!("[{prefix}][{}] Preprocessor finish error: {e}", head.name()); return Err(e); } }; @@ -171,10 +168,7 @@ pub fn finish( match pp.finish(Some(&d), Some(m)) { Ok(mut r) => data1.append(&mut r), Err(e) => { - error!( - "[Connector::{alias}] Preprocessor '{}' finish error: {e}", - pp.name() - ); + error!("[{prefix}][{}] Preprocessor finish error: {e}", pp.name()); return Err(e); } } @@ -225,6 +219,7 @@ mod test { } use proptest::prelude::*; + use tremor_common::alias; // generate multiple chopped length-prefixed strings fn multiple_textual_lengths(max_elements: usize) -> BoxedStrategy<(Vec, Vec)> { @@ -362,7 +357,7 @@ mod test { let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; let wire = post_p.process(0, 0, &data)?; let (start, end) = wire[0].split_at(7); - let alias = Alias::new("test", "test"); + let alias = alias::SourceContext::new("test"); let mut pps: Vec> = vec![Box::new(pre_p)]; let recv = preprocess( pps.as_mut_slice(), @@ -675,10 +670,7 @@ mod test { #[test] fn single_pre_process_head_ok() { let pre = Box::new(BadPreprocessor {}); - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); + let alias = alias::SourceContext::new("chucky"); let mut ingest_ns = 0_u64; let r = preprocess( &mut [pre], @@ -695,10 +687,7 @@ mod test { let noop = Box::new(NoOp {}); assert_eq!("nily", noop.name()); let pre = Box::new(BadPreprocessor {}); - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); + let alias = alias::SourceContext::new("chucky"); let mut ingest_ns = 0_u64; let r = preprocess( &mut [noop, pre], @@ -713,10 +702,7 @@ mod test { #[test] fn single_pre_finish_ok() { let pre = Box::new(BadPreprocessor {}); - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); + let alias = alias::SourceContext::new("chucky"); let r = finish(&mut [pre], &alias); assert!(r.is_ok()); } @@ -730,10 +716,7 @@ mod test { #[test] fn preprocess_finish_head_fail() { - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); + let alias = alias::SourceContext::new("chucky"); let pre = Box::new(BadFinisher {}); let r = finish(&mut [pre], &alias); assert!(r.is_err()); @@ -741,10 +724,7 @@ mod test { #[test] fn preprocess_finish_tail_fail() { - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("chucky"), - "chucky".to_string(), - ); + let alias = alias::SourceContext::new("chucky"); let noop = Box::new(NoOp {}); let pre = Box::new(BadFinisher {}); let r = finish(&mut [noop, pre], &alias); @@ -753,10 +733,7 @@ mod test { #[test] fn preprocess_finish_multi_ok() { - let alias = tremor_common::alias::Connector::new( - tremor_common::alias::Flow::new("xyz"), - "xyz".to_string(), - ); + let alias = alias::SourceContext::new("xyz"); let noop1 = Box::new(NoOp {}); let noop2 = Box::new(NoOp {}); let noop3 = Box::new(NoOp {}); diff --git a/tremor-pipeline/Cargo.toml b/tremor-pipeline/Cargo.toml index a040381bae..ebb6347b8e 100644 --- a/tremor-pipeline/Cargo.toml +++ b/tremor-pipeline/Cargo.toml @@ -14,7 +14,6 @@ error-chain = "0.12" halfbrown = "0.2" indexmap = "2" rand = { version = "0.8", features = ["small_rng"] } -lazy_static = "1" log = "0.4" lru = "0.12" petgraph = "0.6" diff --git a/tremor-pipeline/src/event.rs b/tremor-pipeline/src/event.rs index d98dbd3133..013ddd6374 100644 --- a/tremor-pipeline/src/event.rs +++ b/tremor-pipeline/src/event.rs @@ -14,8 +14,8 @@ use crate::{CbAction, EventId, OpMeta, SignalKind}; use std::mem::swap; -use tremor_common::ids::SourceId; use tremor_common::time::nanotime; +use tremor_common::uids::SourceUId; use tremor_script::prelude::*; use tremor_script::{EventOriginUri, EventPayload}; @@ -58,7 +58,7 @@ impl Event { /// create a drain signal event originating at the connector with the given `source_id` #[must_use] - pub fn signal_drain(source_id: SourceId) -> Self { + pub fn signal_drain(source_id: SourceUId) -> Self { Self { ingest_ns: nanotime(), kind: Some(SignalKind::Drain(source_id)), @@ -68,7 +68,7 @@ impl Event { /// create start signal for the given `SourceId` #[must_use] - pub fn signal_start(uid: SourceId) -> Self { + pub fn signal_start(uid: SourceUId) -> Self { Self { ingest_ns: nanotime(), kind: Some(SignalKind::Start(uid)), @@ -464,7 +464,7 @@ mod test { use super::*; use crate::Result; use simd_json::{ObjectHasher, OwnedValue}; - use tremor_common::ids::{Id, OperatorId}; + use tremor_common::uids::{OperatorUId, UId}; use tremor_script::ValueAndMeta; use tremor_value::Object; @@ -606,7 +606,7 @@ mod test { assert_eq!(e.insight_fail().cb, CbAction::Fail); let mut clone = e.clone(); - let op_id = OperatorId::new(1); + let op_id = OperatorUId::new(1); clone.op_meta.insert(op_id, OwnedValue::null()); let ack_with_timing = clone.insight_ack_with_timing(100); assert_eq!(ack_with_timing.cb, CbAction::Ack); @@ -614,7 +614,7 @@ mod test { let (_, m) = ack_with_timing.data.parts(); assert_eq!(Some(100), m.get_u64("time")); - e.op_meta.insert(OperatorId::new(42), OwnedValue::null()); + e.op_meta.insert(OperatorUId::new(42), OwnedValue::null()); } #[test] diff --git a/tremor-pipeline/src/executable_graph.rs b/tremor-pipeline/src/executable_graph.rs index 64cca0160e..4cf30b6b56 100644 --- a/tremor-pipeline/src/executable_graph.rs +++ b/tremor-pipeline/src/executable_graph.rs @@ -23,7 +23,7 @@ use crate::{op::EventAndInsights, Event, NodeKind, Operator}; use halfbrown::HashMap; use simd_json::ObjectHasher; use std::{fmt, fmt::Display}; -use tremor_common::{ids::OperatorId, ports::Port, stry}; +use tremor_common::{ports::Port, stry, uids::OperatorUId}; use tremor_config::Map as ConfigMap; use tremor_script::{ast::Helper, ast::Stmt}; use tremor_value::{Object, Value}; @@ -82,7 +82,7 @@ impl PartialEq for NodeConfig { impl NodeConfig { pub(crate) fn to_op( &self, - uid: OperatorId, + uid: OperatorUId, resolver: NodeLookupFn, helper: &mut Helper<'static, '_>, ) -> Result { @@ -115,7 +115,7 @@ pub struct OperatorNode { /// The executable operator pub op: Box, /// Tremor unique identifyer - pub uid: OperatorId, + pub uid: OperatorUId, /// The original config pub config: NodeConfig, } @@ -126,12 +126,13 @@ impl Operator for OperatorNode { } fn on_event( &mut self, - _uid: OperatorId, + node_id: u64, + _uid: OperatorUId, port: &Port<'static>, state: &mut Value<'static>, event: Event, ) -> Result { - self.op.on_event(self.uid, port, state, event) + self.op.on_event(node_id, self.uid, port, state, event) } fn handles_signal(&self) -> bool { @@ -139,17 +140,18 @@ impl Operator for OperatorNode { } fn on_signal( &mut self, - _uid: OperatorId, + node_id: u64, + _uid: OperatorUId, state: &mut Value<'static>, signal: &mut Event, ) -> Result { - self.op.on_signal(self.uid, state, signal) + self.op.on_signal(node_id, self.uid, state, signal) } fn handles_contraflow(&self) -> bool { self.op.handles_contraflow() } - fn on_contraflow(&mut self, _uid: OperatorId, contraevent: &mut Event) { + fn on_contraflow(&mut self, _uid: OperatorUId, contraevent: &mut Event) { self.op.on_contraflow(self.uid, contraevent); } @@ -295,148 +297,13 @@ impl ExecutableGraph { Some(()) } - fn optimize_(&mut self) -> Option { - let mut did_chage = false; - // remove skippable nodes from contraflow and signalflow - self.contraflow = (*self.contraflow) - .iter() - .filter(|id| { - !self - .graph - .get(**id) - .map(Operator::skippable) - .unwrap_or_default() - }) - .copied() - .collect(); - self.signalflow = (*self.signalflow) - .iter() - .filter(|id| { - !self - .graph - .get(**id) - .map(Operator::skippable) - .unwrap_or_default() - }) - .copied() - .collect(); - - let mut input_ids: Vec = Vec::new(); - - // first we check the inputs, if an input points to a skippable - // node and does not connect to more then one other node with the same - // input name it can be removed. - for (input_name, target) in &mut self.inputs.iter_mut() { - let target_node = self.graph.get(*target)?; - input_ids.push(*target); - // the target of the input is skippable - if target_node.skippable() { - let mut next_nodes = self - .port_indexes - .iter() - .filter(|((from_id, _), _)| from_id == target); - - if let Some((_, dsts)) = next_nodes.next() { - // we only connect from one output - if next_nodes.next().is_none() && dsts.len() == 1 { - let (next_id, next_input) = dsts.first()?; - if next_input == input_name { - *target = *next_id; - } - } - } - } - } - // The id's of all nodes that are skippable - let skippables: Vec<_> = self - .graph - .iter() - .enumerate() - .filter_map(|(id, e)| { - if e.skippable() && e.kind.skippable() { - Some(id) - } else { - None - } - }) - .collect(); - - for skippable_id in &skippables { - // We iterate over all the skippable ID's - - // We collect all the outputs that the skippable node - // sends data to. - // So of a passthrough sends data to node 4 and 5 - // 4 and 5 becomes the output - let outputs: Vec<_> = self - .port_indexes - .iter() - .filter_map(|((from_id, _), outputs)| { - if from_id == skippable_id { - Some(outputs) - } else { - None - } - }) - .flatten() - .cloned() - .collect(); - - // Find all nodes that connect to the skippable - // so if node 1 and 7 connect to the passthrough - // then this is going to be 1 and 7 - let inputs: Vec<_> = self - .port_indexes - .iter() - .filter_map(|(from, connections)| { - if connections - .iter() - .any(|(target_id, _)| target_id == skippable_id) - { - Some(from) - } else { - None - } - }) - .cloned() - .collect(); - - // We iterate over all nodes that connect to the skippable - // we're handling. - - for i in inputs { - // Take the nodes connections for the indexes - let srcs: Vec<_> = self.port_indexes.remove(&i)?; - let mut srcs1 = Vec::new(); - - // We then iterate over all the destinations that input - // node connects to - for (src_id, src_port) in srcs { - if src_id == *skippable_id { - did_chage = true; - // If it is the skippable node replace this entry - // with all the outputs the skippable had - for o in &outputs { - srcs1.push(o.clone()); - } - } else { - // Otherwise keep the connection untoucehd. - srcs1.push((src_id, src_port)); - } - } - // Add the node back in - self.port_indexes.insert(i, srcs1); - } - } - Some(did_chage) - } - /// This is a performance critial function! /// /// # Errors /// Errors if the event can not be processed, or an operator fails pub fn enqueue( &mut self, + node_id: u64, stream_name: Port<'static>, event: Event, returns: &mut Returns, @@ -459,12 +326,48 @@ impl ExecutableGraph { )) })); self.stack.push((input, stream_name, event)); - self.run(returns) + self.run(node_id, returns) + } + + /// Enqueue a signal + /// + /// # Errors + /// if the singal fails to be processed in the singal flow or if any forward going + /// events spawned by this signal fail to be processed + pub fn enqueue_signal( + &mut self, + node_id: u64, + signal: Event, + returns: &mut Returns, + ) -> Result<()> { + if stry!(self.signalflow(node_id, signal)) { + stry!(self.run(node_id, returns)); + } + Ok(()) + } + + fn signalflow(&mut self, node_id: u64, mut signal: Event) -> Result { + let mut has_events = false; + // We can't use an iterator over signalfow here + // rust refuses to let us use enqueue_events if we do + for idx in 0..self.signalflow.len() { + // ALLOW: we guarantee that idx exists above + let i = unsafe { *self.signalflow.get_unchecked(idx) }; + let EventAndInsights { events, insights } = { + let op = unsafe { self.graph.get_unchecked_mut(i) }; // We know this exists + let state = unsafe { self.states.ops.get_unchecked_mut(i) }; // we know this has been initialized + stry!(op.on_signal(node_id, op.uid, state, &mut signal)) + }; + self.insights.extend(insights.into_iter().map(|cf| (i, cf))); + has_events = has_events || !events.is_empty(); + self.enqueue_events(i, events); + } + Ok(has_events) } #[inline] - fn run(&mut self, returns: &mut Returns) -> Result<()> { - while match self.next(returns) { + fn run(&mut self, node_id: u64, returns: &mut Returns) -> Result<()> { + while match self.next(node_id, returns) { Ok(res) => res, Err(e) => { // if we error handling an event, we need to clear the stack @@ -479,12 +382,12 @@ impl ExecutableGraph { } #[inline] - fn next(&mut self, returns: &mut Returns) -> Result { + fn next(&mut self, node_id: u64, returns: &mut Returns) -> Result { if let Some((idx, port, event)) = self.stack.pop() { // If we have emitted a signal event we got to handle it as a signal flow // the signal flow will if event.kind.is_some() { - stry!(self.signalflow(event)); + stry!(self.signalflow(node_id, event)); } else { // count ingres let node = unsafe { self.graph.get_unchecked_mut(idx) }; @@ -494,7 +397,7 @@ impl ExecutableGraph { // ALLOW: We know the state was initiated let state = unsafe { self.states.ops.get_unchecked_mut(idx) }; let EventAndInsights { events, insights } = - stry!(node.on_event(node.uid, &port, state, event)); + stry!(node.on_event(node_id, node.uid, &port, state, event)); for (out_port, _) in &events { unsafe { self.metrics.get_unchecked_mut(idx) }.inc_output(out_port); @@ -571,36 +474,312 @@ impl ExecutableGraph { } insight } - /// Enqueue a signal - /// - /// # Errors - /// if the singal fails to be processed in the singal flow or if any forward going - /// events spawned by this signal fail to be processed - pub fn enqueue_signal(&mut self, signal: Event, returns: &mut Returns) -> Result<()> { - if stry!(self.signalflow(signal)) { - stry!(self.run(returns)); + fn optimize_(&mut self) -> Option { + let mut did_chage = false; + // remove skippable nodes from contraflow and signalflow + self.contraflow = (*self.contraflow) + .iter() + .filter(|id| { + !self + .graph + .get(**id) + .map(Operator::skippable) + .unwrap_or_default() + }) + .copied() + .collect(); + self.signalflow = (*self.signalflow) + .iter() + .filter(|id| { + !self + .graph + .get(**id) + .map(Operator::skippable) + .unwrap_or_default() + }) + .copied() + .collect(); + + let mut input_ids: Vec = Vec::new(); + + // first we check the inputs, if an input points to a skippable + // node and does not connect to more then one other node with the same + // input name it can be removed. + for (input_name, target) in &mut self.inputs.iter_mut() { + let target_node = self.graph.get(*target)?; + input_ids.push(*target); + // the target of the input is skippable + if target_node.skippable() { + let mut next_nodes = self + .port_indexes + .iter() + .filter(|((from_id, _), _)| from_id == target); + + if let Some((_, dsts)) = next_nodes.next() { + // we only connect from one output + if next_nodes.next().is_none() && dsts.len() == 1 { + let (next_id, next_input) = dsts.first()?; + if next_input == input_name { + *target = *next_id; + } + } + } + } } - Ok(()) - } + // The id's of all nodes that are skippable + let skippables: Vec<_> = self + .graph + .iter() + .enumerate() + .filter_map(|(id, e)| { + if e.skippable() && e.kind.skippable() { + Some(id) + } else { + None + } + }) + .collect(); - fn signalflow(&mut self, mut signal: Event) -> Result { - let mut has_events = false; - // We can't use an iterator over signalfow here - // rust refuses to let us use enqueue_events if we do - for idx in 0..self.signalflow.len() { - // ALLOW: we guarantee that idx exists above - let i = unsafe { *self.signalflow.get_unchecked(idx) }; - let EventAndInsights { events, insights } = { - let op = unsafe { self.graph.get_unchecked_mut(i) }; // We know this exists - let state = unsafe { self.states.ops.get_unchecked_mut(i) }; // we know this has been initialized - stry!(op.on_signal(op.uid, state, &mut signal)) - }; - self.insights.extend(insights.into_iter().map(|cf| (i, cf))); - has_events = has_events || !events.is_empty(); - self.enqueue_events(i, events); + for skippable_id in &skippables { + // We iterate over all the skippable ID's + + // We collect all the outputs that the skippable node + // sends data to. + // So of a passthrough sends data to node 4 and 5 + // 4 and 5 becomes the output + let outputs: Vec<_> = self + .port_indexes + .iter() + .filter_map(|((from_id, _), outputs)| { + if from_id == skippable_id { + Some(outputs) + } else { + None + } + }) + .flatten() + .cloned() + .collect(); + + // Find all nodes that connect to the skippable + // so if node 1 and 7 connect to the passthrough + // then this is going to be 1 and 7 + let inputs: Vec<_> = self + .port_indexes + .iter() + .filter_map(|(from, connections)| { + if connections + .iter() + .any(|(target_id, _)| target_id == skippable_id) + { + Some(from) + } else { + None + } + }) + .cloned() + .collect(); + + // We iterate over all nodes that connect to the skippable + // we're handling. + + for i in inputs { + // Take the nodes connections for the indexes + let srcs: Vec<_> = self.port_indexes.remove(&i)?; + let mut srcs1 = Vec::new(); + + // We then iterate over all the destinations that input + // node connects to + for (src_id, src_port) in srcs { + if src_id == *skippable_id { + did_chage = true; + // If it is the skippable node replace this entry + // with all the outputs the skippable had + for o in &outputs { + srcs1.push(o.clone()); + } + } else { + // Otherwise keep the connection untoucehd. + srcs1.push((src_id, src_port)); + } + } + // Add the node back in + self.port_indexes.insert(i, srcs1); + } } - Ok(has_events) + Some(did_chage) } + + // /// This is a performance critial function! + // /// + // /// # Errors + // /// Errors if the event can not be processed, or an operator fails + // pub fn enqueue( + // &mut self, + // stream_name: Port<'static>, + // event: Event, + // returns: &mut Returns, + // ) -> Result<()> { + // // Resolve the input stream or entrypoint for this enqueue operation + // if self + // .metric_interval + // .map(|ival| event.ingest_ns - self.last_metrics > ival) + // .unwrap_or_default() + // { + // let mut tags = HashMap::with_capacity_and_hasher(8, ObjectHasher::default()); + // tags.insert("pipeline".into(), common_cow(&self.id).into()); + // self.send_metrics("events", tags, event.ingest_ns); + // self.last_metrics = event.ingest_ns; + // } + // let input = *stry!(self.inputs.get(&stream_name).ok_or_else(|| { + // Error::from(ErrorKind::InvalidInputStreamName( + // stream_name.to_string(), + // self.id.clone(), + // )) + // })); + // self.stack.push((input, stream_name, event)); + // self.run(returns) + // } + + // #[inline] + // fn run(&mut self, returns: &mut Returns) -> Result<()> { + // while match self.next(returns) { + // Ok(res) => res, + // Err(e) => { + // // if we error handling an event, we need to clear the stack + // // In the case of branching where the stack has > 1 element + // // we would keep the old errored event around for the numbers of branches that havent been executed + // self.stack.clear(); + // return Err(e); + // } + // } {} + // returns.reverse(); + // Ok(()) + // } + + // #[inline] + // fn next(&mut self, returns: &mut Returns) -> Result { + // if let Some((idx, port, event)) = self.stack.pop() { + // // If we have emitted a signal event we got to handle it as a signal flow + // // the signal flow will + // if event.kind.is_some() { + // stry!(self.signalflow(event)); + // } else { + // // count ingres + // let node = unsafe { self.graph.get_unchecked_mut(idx) }; + // if let NodeKind::Output(port) = &node.kind { + // returns.push((port.clone(), event)); + // } else { + // // ALLOW: We know the state was initiated + // let state = unsafe { self.states.ops.get_unchecked_mut(idx) }; + // let EventAndInsights { events, insights } = + // stry!(node.on_event(node.uid, &port, state, event)); + + // for (out_port, _) in &events { + // unsafe { self.metrics.get_unchecked_mut(idx) }.inc_output(out_port); + // } + // for insight in insights { + // self.insights.push((idx, insight)); + // } + // self.enqueue_events(idx, events); + // }; + // } + // Ok(!self.stack.is_empty()) + // } else { + // error!("next was called on an empty graph stack, this should never happen"); + // Ok(false) + // } + // } + + // fn send_metrics(&mut self, metric_name: &str, mut tags: Object<'static>, ingest_ns: u64) { + // for (i, m) in self.metrics.iter().enumerate() { + // tags.insert("node".into(), unsafe { + // self.graph.get_unchecked(i).id.clone().into() + // }); + // if let Ok(metrics) = unsafe { self.graph.get_unchecked(i) }.metrics(&tags, ingest_ns) { + // for value in metrics { + // if let Err(e) = self.metrics_channel.send(MetricsMsg { + // payload: value.into(), + // origin_uri: None, + // }) { + // error!("Failed to send metrics: {}", e); + // }; + // } + // } + + // for value in m.to_value(metric_name, &mut tags, ingest_ns) { + // if let Err(e) = self.metrics_channel.send(MetricsMsg { + // payload: value.into(), + // origin_uri: None, + // }) { + // error!("Failed to send metrics: {}", e); + // }; + // } + // } + // } + // // Takes the output of one operator, identified by `idx` and puts them on the stack + // // for the connected operators to pick up. + // // If the output is not connected we register this as a dropped event + // #[inline] + // fn enqueue_events(&mut self, idx: usize, events: Vec<(Port<'static>, Event)>) { + // for (out_port, event) in events { + // if let Some((last, rest)) = self + // .port_indexes + // .get(&(idx, out_port)) + // .and_then(|o| o.split_last()) + // { + // for (idx, in_port) in rest { + // unsafe { self.metrics.get_unchecked_mut(*idx) }.inc_input(in_port); + // self.stack.push((*idx, in_port.clone(), event.clone())); + // } + // let (idx, in_port) = last; + // unsafe { self.metrics.get_unchecked_mut(*idx) }.inc_input(in_port); + // self.stack.push((*idx, in_port.clone(), event)); + // } + // } + // } + // /// Enque a contraflow insight + // pub fn contraflow(&mut self, mut skip_to: Option, mut insight: Event) -> Event { + // for idx in &self.contraflow { + // if skip_to.is_none() { + // let op = unsafe { self.graph.get_unchecked_mut(*idx) }; // We know this exists + // op.on_contraflow(op.uid, &mut insight); + // } else if skip_to == Some(*idx) { + // skip_to = None; + // } + // } + // insight + // } + // /// Enqueue a signal + // /// + // /// # Errors + // /// if the singal fails to be processed in the singal flow or if any forward going + // /// events spawned by this signal fail to be processed + // pub fn enqueue_signal(&mut self, signal: Event, returns: &mut Returns) -> Result<()> { + // if stry!(self.signalflow(signal)) { + // stry!(self.run(returns)); + // } + // Ok(()) + // } + + // fn signalflow(&mut self, mut signal: Event) -> Result { + // let mut has_events = false; + // // We can't use an iterator over signalfow here + // // rust refuses to let us use enqueue_events if we do + // for idx in 0..self.signalflow.len() { + // // ALLOW: we guarantee that idx exists above + // let i = unsafe { *self.signalflow.get_unchecked(idx) }; + // let EventAndInsights { events, insights } = { + // let op = unsafe { self.graph.get_unchecked_mut(i) }; // We know this exists + // let state = unsafe { self.states.ops.get_unchecked_mut(i) }; // we know this has been initialized + // stry!(op.on_signal(op.uid, state, &mut signal)) + // }; + // self.insights.extend(insights.into_iter().map(|cf| (i, cf))); + // has_events = has_events || !events.is_empty(); + // self.enqueue_events(i, events); + // } + // Ok(has_events) + // } } #[cfg(test)] @@ -611,11 +790,11 @@ mod test { identity::PassthroughFactory, prelude::{IN, OUT}, }, - Result, METRICS_CHANNEL, + MetricsChannel, Result, }; - use tremor_common::ids::Id; + use tremor_common::uids::UId; use tremor_script::prelude::*; - fn pass(uid: OperatorId, id: &'static str) -> Result { + fn pass(uid: OperatorUId, id: &'static str) -> Result { let config = NodeConfig::from_config(&"passthrough", None); Ok(OperatorNode { id: id.into(), @@ -628,7 +807,7 @@ mod test { } #[test] fn operator_node() -> Result<()> { - let op_id = OperatorId::new(0); + let op_id = OperatorUId::new(0); let mut n = pass(op_id, "passthrough")?; assert!(!n.handles_contraflow()); assert!(!n.handles_signal()); @@ -638,7 +817,7 @@ mod test { n.on_contraflow(op_id, &mut e); assert_eq!(e, Event::default()); assert_eq!( - n.on_signal(op_id, &mut state, &mut e)?, + n.on_signal(0, op_id, &mut state, &mut e)?, EventAndInsights::default() ); assert_eq!(e, Event::default()); @@ -679,7 +858,8 @@ mod test { impl Operator for AllOperator { fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, event: Event, @@ -692,7 +872,8 @@ mod test { fn on_signal( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _state: &mut Value<'static>, _signal: &mut Event, ) -> Result { @@ -704,7 +885,7 @@ mod test { true } - fn on_contraflow(&mut self, _uid: OperatorId, _insight: &mut Event) {} + fn on_contraflow(&mut self, _uid: OperatorUId, _insight: &mut Event) {} fn metrics(&self, _tags: &Object<'static>, _timestamp: u64) -> Result>> { Ok(Vec::new()) @@ -721,7 +902,7 @@ mod test { kind: NodeKind::Operator, op_type: "test".into(), op: Box::new(AllOperator {}), - uid: OperatorId::default(), + uid: OperatorUId::default(), config: NodeConfig::default(), } } @@ -774,9 +955,9 @@ mod test { #[tokio::test(flavor = "multi_thread")] async fn eg_metrics() -> Result<()> { - let mut in_n = pass(OperatorId::new(1), "in")?; + let mut in_n = pass(OperatorUId::new(1), "in")?; in_n.kind = NodeKind::Input; - let mut out_n = pass(OperatorId::new(2), "out")?; + let mut out_n = pass(OperatorUId::new(2), "out")?; out_n.kind = NodeKind::Output(OUT); // The graph is in -> 1 -> 2 -> out, we pre-stack the edges since we do not @@ -811,7 +992,8 @@ mod test { let states = State { ops: vec![Value::null(), Value::null(), Value::null(), Value::null()], }; - let mut rx = METRICS_CHANNEL.rx(); + let metrics_channel = MetricsChannel::new(128); + let mut rx = metrics_channel.rx(); let mut g = ExecutableGraph { id: "flow::pipe".into(), graph, @@ -828,13 +1010,13 @@ mod test { metric_interval: Some(1), insights: vec![], dot: String::new(), - metrics_channel: METRICS_CHANNEL.tx(), + metrics_channel: metrics_channel.tx(), }; // Test with one event let e = Event::default(); let mut returns = vec![]; - g.enqueue(IN, e, &mut returns)?; + g.enqueue(0, IN, e, &mut returns)?; assert_eq!(returns.len(), 1); returns.clear(); @@ -852,13 +1034,13 @@ mod test { // Test with two events let e = Event::default(); let mut returns = vec![]; - g.enqueue(IN, e, &mut returns)?; + g.enqueue(0, IN, e, &mut returns)?; assert_eq!(returns.len(), 1); returns.clear(); let e = Event::default(); let mut returns = vec![]; - g.enqueue(IN, e, &mut returns)?; + g.enqueue(0, IN, e, &mut returns)?; assert_eq!(returns.len(), 1); returns.clear(); @@ -877,18 +1059,18 @@ mod test { #[tokio::test(flavor = "multi_thread")] async fn eg_optimize() -> Result<()> { - let mut in_n = pass(OperatorId::new(0), "in")?; + let mut in_n = pass(OperatorUId::new(0), "in")?; in_n.kind = NodeKind::Input; - let mut out_n = pass(OperatorId::new(1), "out")?; + let mut out_n = pass(OperatorUId::new(1), "out")?; out_n.kind = NodeKind::Output(OUT); // The graph is in -> 1 -> 2 -> out, we pre-stack the edges since we do not // need to have order in here. let graph = vec![ - in_n, // 0 - all_op("all-1"), // 1 - pass(OperatorId::new(2), "nop")?, // 2 - all_op("all-2"), // 3 - out_n, // 4 + in_n, // 0 + all_op("all-1"), // 1 + pass(OperatorUId::new(2), "nop")?, // 2 + all_op("all-2"), // 3 + out_n, // 4 ]; let mut inputs = HashMap::new(); @@ -922,6 +1104,7 @@ mod test { Value::null(), ], }; + let metrics_channel = MetricsChannel::new(128); let mut g = ExecutableGraph { id: "test".into(), graph, @@ -938,13 +1121,13 @@ mod test { metric_interval: None, insights: vec![], dot: String::new(), - metrics_channel: METRICS_CHANNEL.tx(), + metrics_channel: metrics_channel.tx(), }; assert!(g.optimize().is_some()); // Test with one event let e = Event::default(); let mut returns = vec![]; - g.enqueue(IN, e, &mut returns)?; + g.enqueue(0, IN, e, &mut returns)?; assert_eq!(returns.len(), 1); returns.clear(); diff --git a/tremor-pipeline/src/lib.rs b/tremor-pipeline/src/lib.rs index f5240137af..f25bbf6eae 100644 --- a/tremor-pipeline/src/lib.rs +++ b/tremor-pipeline/src/lib.rs @@ -35,7 +35,6 @@ use crate::errors::{ErrorKind, Result}; use either::Either; use executable_graph::NodeConfig; use halfbrown::HashMap; -use lazy_static::lazy_static; use petgraph::graph; use simd_json::OwnedValue; use std::collections::{BTreeMap, HashSet}; @@ -46,8 +45,8 @@ use std::str::FromStr; use std::{borrow::Borrow, cmp::Ordering}; use tokio::sync::broadcast::{self, Receiver, Sender}; use tremor_common::{ - ids::{Id, OperatorId, SinkId, SourceId}, ports::Port, + uids::{OperatorUId, SinkUId, SourceUId, UId}, }; use tremor_script::{ ast::{self, Helper}, @@ -79,7 +78,7 @@ pub(crate) type ExecPortIndexMap = HashMap<(usize, Port<'static>), Vec<(usize, P /// A lookup function to used to look up operators pub type NodeLookupFn = fn( config: &NodeConfig, - uid: OperatorId, + uid: OperatorUId, node: Option<&ast::Stmt<'static>>, helper: &mut Helper<'static, '_>, ) -> Result; @@ -90,8 +89,16 @@ pub struct MetricsChannel { tx: Sender, } +impl Default for MetricsChannel { + fn default() -> Self { + Self::new(128) + } +} + impl MetricsChannel { - pub(crate) fn new(qsize: usize) -> Self { + /// Creates a new metrics channel + #[must_use] + pub fn new(qsize: usize) -> Self { let (tx, _) = broadcast::channel(qsize); Self { tx } } @@ -130,11 +137,6 @@ impl MetricsMsg { /// Sender for metrics pub type MetricsSender = Sender; -lazy_static! { - /// TODO do we want to change this number or can we make it configurable? - pub static ref METRICS_CHANNEL: MetricsChannel = MetricsChannel::new(128); -} - /// Stringified numeric key /// from #[derive(Clone, Copy, Ord, PartialOrd, Eq, PartialEq, Debug)] @@ -194,23 +196,23 @@ where // TODO: optimization: - use two Vecs, one for operator ids, one for operator metadata (Values) // - make it possible to trace operators with and without metadata // - insert with bisect (numbers of operators tracked will be low single digit numbers most of the time) -pub struct OpMeta(BTreeMap, OwnedValue>); +pub struct OpMeta(BTreeMap, OwnedValue>); impl OpMeta { /// inserts a value - pub fn insert(&mut self, key: OperatorId, value: V) -> Option + pub fn insert(&mut self, key: OperatorUId, value: V) -> Option where OwnedValue: From, { self.0.insert(PrimStr(key), OwnedValue::from(value)) } /// reads a value - pub fn get(&mut self, key: OperatorId) -> Option<&OwnedValue> { + pub fn get(&mut self, key: OperatorUId) -> Option<&OwnedValue> { self.0.get(&PrimStr(key)) } /// checks existance of a key #[must_use] - pub fn contains_key(&self, key: OperatorId) -> bool { + pub fn contains_key(&self, key: OperatorUId) -> bool { self.0.contains_key(&PrimStr(key)) } @@ -285,9 +287,9 @@ pub enum CbAction { Fail, /// Notify all upstream sources that this sink has started, notifying them of its existence. /// Will be used for tracking for which sinks to wait during Drain. - SinkStart(SinkId), + SinkStart(SinkUId), /// answer to a `SignalKind::Drain(uid)` signal from a connector with the same uid - Drained(SourceId, SinkId), + Drained(SourceUId, SinkUId), } impl Default for CbAction { fn default() -> Self { @@ -824,30 +826,30 @@ impl EventIdGenerator { #[must_use] /// create a new generator for the `Source` identified by `source_id` using the default stream id - pub fn new(source_id: SourceId) -> Self { + pub fn new(source_id: SourceUId) -> Self { Self(source_id.id(), DEFAULT_STREAM_ID, 0) } #[must_use] /// create a new generator for the `Operator` identified by `operator_id` using the default stream id - pub fn for_operator(operator_id: OperatorId) -> Self { + pub fn for_operator(operator_id: OperatorUId) -> Self { Self(operator_id.id(), DEFAULT_STREAM_ID, 0) } #[must_use] /// create a new generator for the `Operator` identified by `operator_id` with `stream_id` - pub fn for_operator_with_stream(operator_id: OperatorId, stream_id: u64) -> Self { + pub fn for_operator_with_stream(operator_id: OperatorUId, stream_id: u64) -> Self { Self(operator_id.id(), stream_id, 0) } #[must_use] /// create a new generator using the given source and stream id - pub fn new_with_stream(source_id: SourceId, stream_id: u64) -> Self { + pub fn new_with_stream(source_id: SourceUId, stream_id: u64) -> Self { Self(source_id.id(), stream_id, 0) } /// set the source id - pub fn set_source(&mut self, source_id: SourceId) { + pub fn set_source(&mut self, source_id: SourceUId) { self.0 = source_id.id(); } @@ -869,7 +871,7 @@ impl EventIdGenerator { pub enum SignalKind { // Lifecycle /// Start signal, containing the source uid which just started - Start(SourceId), + Start(SourceUId), /// Shutdown Signal Shutdown, // Pause, TODO debug trace @@ -883,7 +885,7 @@ pub enum SignalKind { /// This signal must be answered with a Drain contraflow event containing the same uid (u64) /// this way a contraflow event will not be interpreted by connectors for which it isn't meant /// reception of such Drain contraflow event notifies the signal sender that the intermittent pipeline is drained and can be safely disconnected - Drain(SourceId), + Drain(SourceUId), } // We ignore this since it's a simple lookup table @@ -922,7 +924,7 @@ fn factory(node: &NodeConfig) -> Result> { Ok(factory) } -fn operator(uid: OperatorId, node: &NodeConfig) -> Result> { +fn operator(uid: OperatorUId, node: &NodeConfig) -> Result> { factory(node)?.node_to_operator(uid, node) } @@ -966,9 +968,9 @@ mod test { #[test] fn op_meta_merge() { - let op_id1 = OperatorId::new(1); - let op_id2 = OperatorId::new(2); - let op_id3 = OperatorId::new(3); + let op_id1 = OperatorUId::new(1); + let op_id2 = OperatorUId::new(2); + let op_id3 = OperatorUId::new(3); let mut m1 = OpMeta::default(); let mut m2 = OpMeta::default(); m1.insert(op_id1, 1); diff --git a/tremor-pipeline/src/macros.rs b/tremor-pipeline/src/macros.rs index f688040592..91cae21965 100644 --- a/tremor-pipeline/src/macros.rs +++ b/tremor-pipeline/src/macros.rs @@ -26,7 +26,7 @@ macro_rules! op { impl $crate::op::InitializableOperator for $factory { fn node_to_operator( &self, - $uid: tremor_common::ids::OperatorId, + $uid: tremor_common::uids::OperatorUId, $node: &$crate::NodeConfig, ) -> $crate::errors::Result> { $constructor diff --git a/tremor-pipeline/src/op.rs b/tremor-pipeline/src/op.rs index 37251f272b..b128d019f5 100644 --- a/tremor-pipeline/src/op.rs +++ b/tremor-pipeline/src/op.rs @@ -25,18 +25,9 @@ pub mod trickle; use self::prelude::OUT; use super::{Event, NodeConfig}; use crate::errors::Result; -use regex::Regex; -use tremor_common::{ids::OperatorId, ports::Port}; +use tremor_common::{ports::Port, uids::OperatorUId}; use tremor_value::{Object, Value}; -lazy_static::lazy_static! { - static ref LINE_REGEXP: Regex = { - #[allow(clippy::unwrap_used)] - // ALLOW: we tested this - Regex::new(r" at line \d+ column \d+$").unwrap() - }; -} - /// Response type for operator callbacks returning both events and insights #[derive(Default, Clone, PartialEq, Debug)] pub struct EventAndInsights { @@ -79,7 +70,8 @@ pub trait Operator: std::fmt::Debug + Send + Sync { /// if the event can not be processed fn on_event( &mut self, - uid: OperatorId, + node_id: u64, + uid: OperatorUId, port: &Port<'static>, state: &mut Value<'static>, event: Event, @@ -98,7 +90,8 @@ pub trait Operator: std::fmt::Debug + Send + Sync { /// if the singal can not be processed fn on_signal( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _state: &mut Value<'static>, _signal: &mut Event, ) -> Result { @@ -117,7 +110,7 @@ pub trait Operator: std::fmt::Debug + Send + Sync { /// /// # Errors /// if the insight can not be processed - fn on_contraflow(&mut self, _uid: OperatorId, _insight: &mut Event) { + fn on_contraflow(&mut self, _uid: OperatorUId, _insight: &mut Event) { // Make the trait signature nicer } @@ -148,5 +141,5 @@ pub trait InitializableOperator { /// /// # Errors //// if no operator con be instanciated from the provided NodeConfig - fn node_to_operator(&self, uid: OperatorId, node: &NodeConfig) -> Result>; + fn node_to_operator(&self, uid: OperatorUId, node: &NodeConfig) -> Result>; } diff --git a/tremor-pipeline/src/op/bert/sequence_classification.rs b/tremor-pipeline/src/op/bert/sequence_classification.rs index 0d05f5dec9..a24a6c9849 100644 --- a/tremor-pipeline/src/op/bert/sequence_classification.rs +++ b/tremor-pipeline/src/op/bert/sequence_classification.rs @@ -107,7 +107,8 @@ op!(SequenceClassificationFactory(_uid, node) { impl Operator for SequenceClassification { fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, mut event: Event, @@ -115,7 +116,7 @@ impl Operator for SequenceClassification { event.data.rent_mut(|data| -> Result<()> { let (v, m) = data.parts_mut(); if let Some(s) = v.as_str() { - let labels = self.model.lock()?.predict(&[s]); + let labels = self.model.lock()?.predict([s]); let mut label_meta = Value::object_with_capacity(labels.len()); for label in labels { label_meta.try_insert(label.text, label.score); diff --git a/tremor-pipeline/src/op/bert/summarization.rs b/tremor-pipeline/src/op/bert/summarization.rs index 05b65c7e9d..f40dda6633 100644 --- a/tremor-pipeline/src/op/bert/summarization.rs +++ b/tremor-pipeline/src/op/bert/summarization.rs @@ -63,7 +63,8 @@ op!(SummerizationFactory(_uid, node) { impl Operator for Summerization { fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, mut event: Event, diff --git a/tremor-pipeline/src/op/debug/history.rs b/tremor-pipeline/src/op/debug/history.rs index 00d6c44862..d257992559 100644 --- a/tremor-pipeline/src/op/debug/history.rs +++ b/tremor-pipeline/src/op/debug/history.rs @@ -76,7 +76,8 @@ struct History { impl Operator for History { fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, mut event: Event, @@ -118,7 +119,8 @@ impl Operator for History { } fn on_signal( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _state: &mut Value<'static>, signal: &mut Event, ) -> Result { @@ -156,7 +158,7 @@ impl Operator for History { #[cfg(test)] mod test { - use tremor_common::ids::Id; + use tremor_common::uids::UId; use super::*; use crate::EventId; @@ -169,7 +171,7 @@ mod test { name: "snot".to_string(), }, }; - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let event = Event { id: EventId::from_id(0, 0, 1), ingest_ns: 1, @@ -180,7 +182,7 @@ mod test { let mut state = Value::null(); let (out, event) = op - .on_event(operator_id, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("Failed to run pipeline") .events .pop() @@ -188,7 +190,7 @@ mod test { assert_eq!(out, "out"); let (out, _event) = op - .on_event(operator_id, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("Failed to run pipeline") .events .pop() @@ -203,8 +205,8 @@ mod test { }; let mut state = Value::null(); - op.on_signal(operator_id, &mut state, &mut event)?; - op.on_signal(operator_id, &mut state, &mut event)?; + op.on_signal(0, operator_id, &mut state, &mut event)?; + op.on_signal(0, operator_id, &mut state, &mut event)?; let history = event.data.suffix().meta().get(op.config.name.as_str()); diff --git a/tremor-pipeline/src/op/generic/batch.rs b/tremor-pipeline/src/op/generic/batch.rs index eb03a50fde..9d7f394e55 100644 --- a/tremor-pipeline/src/op/generic/batch.rs +++ b/tremor-pipeline/src/op/generic/batch.rs @@ -99,7 +99,8 @@ impl Operator for Batch { /// with a new event id tracking all events within that batch fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, event: Event, @@ -170,7 +171,8 @@ impl Operator for Batch { fn on_signal( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _state: &mut Value<'static>, signal: &mut Event, ) -> Result { @@ -211,12 +213,12 @@ impl Operator for Batch { mod test { use super::*; use simd_json_derive::Serialize; - use tremor_common::ids::Id; + use tremor_common::uids::UId; use tremor_value::Value; #[test] fn size() { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut idgen = EventIdGenerator::for_operator(operator_id); let mut op = Batch { config: Config { @@ -241,7 +243,7 @@ mod test { let mut state = Value::null(); let r = op - .on_event(operator_id, &Port::In, &mut state, event_1.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event_1.clone()) .expect("could not run pipeline"); assert_eq!(r.len(), 0); @@ -253,7 +255,7 @@ mod test { }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event_2.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event_2.clone()) .expect("could not run pipeline"); assert_eq!(r.len(), 1); let (out, event) = r.events.pop().expect("no results"); @@ -272,14 +274,14 @@ mod test { }; let r = op - .on_event(operator_id, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("could not run pipeline"); assert_eq!(r.len(), 0); } #[test] fn time() -> Result<()> { - let operator_id = OperatorId::new(42); + let operator_id = OperatorUId::new(42); let node_config = NodeConfig::from_config( &"badger", Some(literal!({ @@ -301,7 +303,7 @@ mod test { let mut state = Value::null(); let r = op - .on_event(operator_id, &Port::In, &mut state, event_1.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event_1.clone()) .expect("could not run pipeline"); assert_eq!(r.len(), 0); @@ -314,7 +316,7 @@ mod test { }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event_2.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event_2.clone()) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -336,7 +338,7 @@ mod test { }; let r = op - .on_event(operator_id, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("could not run pipeline"); assert_eq!(r.len(), 0); @@ -348,7 +350,7 @@ mod test { }; let r = op - .on_event(operator_id, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("could not run pipeline"); assert_eq!(r.len(), 0); Ok(()) @@ -356,7 +358,7 @@ mod test { #[test] fn signal() { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut idgen = EventIdGenerator::for_operator(operator_id); let mut op = Batch { config: Config { @@ -382,7 +384,7 @@ mod test { let mut state = Value::null(); let r = op - .on_event(operator_id, &Port::In, &mut state, event_1.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event_1.clone()) .expect("failed to run peipeline"); assert_eq!(r.len(), 0); @@ -394,7 +396,7 @@ mod test { }; let mut r = op - .on_signal(operator_id, &mut state, &mut signal) + .on_signal(0, operator_id, &mut state, &mut signal) .expect("failed to run pipeline") .events; assert_eq!(r.len(), 1); @@ -413,7 +415,7 @@ mod test { }; let r = op - .on_event(operator_id, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("failed to run pipeline"); assert_eq!(r.len(), 0); @@ -425,14 +427,14 @@ mod test { }; let r = op - .on_event(operator_id, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("failed to run piepeline"); assert_eq!(r.len(), 0); } #[test] fn forbid_empty_batches() -> Result<()> { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut idgen = EventIdGenerator::for_operator(operator_id); let mut op = Batch { config: Config { @@ -456,7 +458,9 @@ mod test { ..Event::default() }; - let r = op.on_signal(operator_id, &mut state, &mut signal)?.events; + let r = op + .on_signal(0, operator_id, &mut state, &mut signal)? + .events; assert_eq!(r.len(), 0); let event1 = Event { @@ -465,17 +469,21 @@ mod test { data: Value::from("snot").into(), ..Event::default() }; - let r = op.on_event(operator_id, &Port::In, &mut state, event1)?; + let r = op.on_event(0, operator_id, &Port::In, &mut state, event1)?; assert_eq!(r.len(), 0); signal.ingest_ns = 3_000_000; signal.id = (1, 1, 2).into(); - let r = op.on_signal(operator_id, &mut state, &mut signal)?.events; + let r = op + .on_signal(0, operator_id, &mut state, &mut signal)? + .events; assert_eq!(r.len(), 1); signal.ingest_ns = 4_000_000; signal.id = (1, 1, 3).into(); - let r = op.on_signal(operator_id, &mut state, &mut signal)?.events; + let r = op + .on_signal(0, operator_id, &mut state, &mut signal)? + .events; assert_eq!(r.len(), 0); Ok(()) diff --git a/tremor-pipeline/src/op/generic/counter.rs b/tremor-pipeline/src/op/generic/counter.rs index 99e2dee609..dfb1716c77 100644 --- a/tremor-pipeline/src/op/generic/counter.rs +++ b/tremor-pipeline/src/op/generic/counter.rs @@ -42,7 +42,8 @@ op!(CounterFactory(_uid, _node) { impl Operator for Counter { fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, state: &mut Value<'static>, mut event: Event, diff --git a/tremor-pipeline/src/op/grouper/bucket.rs b/tremor-pipeline/src/op/grouper/bucket.rs index e67bf33105..ccf9d70e9a 100644 --- a/tremor-pipeline/src/op/grouper/bucket.rs +++ b/tremor-pipeline/src/op/grouper/bucket.rs @@ -158,7 +158,8 @@ impl Operator for Grouper { #[allow(clippy::manual_let_else)] // clippy bug fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, event: Event, @@ -230,12 +231,12 @@ impl Operator for Grouper { mod test { use super::*; use simd_json::ObjectHasher; - use tremor_common::ids::Id; + use tremor_common::uids::UId; use tremor_value::Value; #[test] fn bucket() -> Result<()> { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut op = Grouper { buckets: HashMap::new(), }; @@ -249,7 +250,7 @@ mod test { let mut state = Value::null(); let mut r = op - .on_event(operator_id, &Port::In, &mut state, event1.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event1.clone()) .expect("could not run pipeline"); let (port, e) = r.events.pop().ok_or("no data")?; @@ -266,7 +267,7 @@ mod test { }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event2.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event2.clone()) .expect("could not run pipeline"); let (port, e) = r.events.pop().ok_or("no data")?; @@ -275,7 +276,7 @@ mod test { assert_eq!(e, event2); let mut r = op - .on_event(operator_id, &Port::In, &mut state, event2.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event2.clone()) .expect("could not run pipeline"); let (port, e) = r.events.pop().ok_or("no data")?; @@ -284,7 +285,7 @@ mod test { assert_eq!(e, event2); let mut r = op - .on_event(operator_id, &Port::In, &mut state, event2.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event2.clone()) .expect("could not run pipeline"); let (port, e) = r.events.pop().ok_or("no data")?; @@ -300,7 +301,7 @@ mod test { }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event3.clone()) + .on_event(0, operator_id, &Port::In, &mut state, event3.clone()) .expect("could not run pipeline"); let (port, e) = r.events.pop().ok_or("no data")?; diff --git a/tremor-pipeline/src/op/identity/passthrough.rs b/tremor-pipeline/src/op/identity/passthrough.rs index 3d8b0b3709..a7ea33bd8b 100644 --- a/tremor-pipeline/src/op/identity/passthrough.rs +++ b/tremor-pipeline/src/op/identity/passthrough.rs @@ -24,7 +24,8 @@ op!(PassthroughFactory (_uid, _node) { impl Operator for Passthrough { fn on_event( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, event: Event, diff --git a/tremor-pipeline/src/op/qos/backpressure.rs b/tremor-pipeline/src/op/qos/backpressure.rs index 9d037cf0ae..cfda8588fe 100644 --- a/tremor-pipeline/src/op/qos/backpressure.rs +++ b/tremor-pipeline/src/op/qos/backpressure.rs @@ -148,7 +148,8 @@ op!(BackpressureFactory(_uid, node) { impl Operator for Backpressure { fn on_event( &mut self, - uid: OperatorId, + _node_id: u64, + uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, mut event: Event, @@ -179,7 +180,8 @@ impl Operator for Backpressure { fn on_signal( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _state: &mut Value<'static>, signal: &mut Event, ) -> Result { @@ -199,7 +201,7 @@ impl Operator for Backpressure { }) } - fn on_contraflow(&mut self, uid: OperatorId, insight: &mut Event) { + fn on_contraflow(&mut self, uid: OperatorUId, insight: &mut Event) { // If the related event never touched this operator we don't take // action if !insight.op_meta.contains_key(uid) { @@ -234,12 +236,12 @@ mod test { use super::*; use crate::SignalKind; use simd_json::ObjectHasher; - use tremor_common::ids::Id; + use tremor_common::uids::UId; use tremor_value::Object; #[test] fn pass_wo_error() { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut op: Backpressure = Config { timeout: 100_000_000, steps: vec![1, 10, 100], @@ -257,7 +259,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event1) + .on_event(0, operator_id, &Port::In, &mut state, event1) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -273,7 +275,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event2) + .on_event(0, operator_id, &Port::In, &mut state, event2) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -284,7 +286,7 @@ mod test { #[test] fn halt_on_error() { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut op: Backpressure = Config { timeout: 100_000_000, steps: vec![1, 10, 100], @@ -302,7 +304,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event1) + .on_event(0, operator_id, &Port::In, &mut state, event1) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -340,7 +342,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event2) + .on_event(0, operator_id, &Port::In, &mut state, event2) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -355,7 +357,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event3) + .on_event(0, operator_id, &Port::In, &mut state, event3) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -371,7 +373,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event3) + .on_event(0, operator_id, &Port::In, &mut state, event3) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -381,7 +383,7 @@ mod test { #[test] fn halt_on_error_cb() -> Result<()> { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut op: Backpressure = Config { timeout: 100_000_000, steps: vec![1, 10, 100], @@ -399,7 +401,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event1)? + .on_event(0, operator_id, &Port::In, &mut state, event1)? .events; assert_eq!(r.len(), 1); let (out, event) = r.pop().expect("no results"); @@ -435,7 +437,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event2)? + .on_event(0, operator_id, &Port::In, &mut state, event2)? .events; assert_eq!(r.len(), 1); let (out, _event) = r.pop().expect("no results"); @@ -450,7 +452,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event3)? + .on_event(0, operator_id, &Port::In, &mut state, event3)? .events; assert_eq!(r.len(), 1); let (out, event) = r.pop().expect("no results"); @@ -465,7 +467,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(operator_id, &Port::In, &mut state, event3)? + .on_event(0, operator_id, &Port::In, &mut state, event3)? .events; assert_eq!(r.len(), 1); let (out, _event) = r.pop().expect("no results"); @@ -479,7 +481,7 @@ mod test { kind: Some(SignalKind::Tick), ..Event::default() }; - let mut r = op.on_signal(operator_id, &mut state, &mut signal)?; + let mut r = op.on_signal(0, operator_id, &mut state, &mut signal)?; let i = r.insights.pop().expect("No Insight received"); // We receive a restore signal assert_eq!(i.cb, CbAction::Restore); @@ -490,7 +492,7 @@ mod test { #[test] fn walk_backoff() { - let operator_id = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut op: Backpressure = Config { timeout: 100_000_000, steps: vec![1, 10, 100], diff --git a/tremor-pipeline/src/op/qos/percentile.rs b/tremor-pipeline/src/op/qos/percentile.rs index 1276ac89e9..fbd80a7ed5 100644 --- a/tremor-pipeline/src/op/qos/percentile.rs +++ b/tremor-pipeline/src/op/qos/percentile.rs @@ -100,7 +100,8 @@ op!(PercentileFactory(_uid, node) { impl Operator for Percentile { fn on_event( &mut self, - uid: OperatorId, + _node_id: u64, + uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, mut event: Event, @@ -127,7 +128,7 @@ impl Operator for Percentile { true } - fn on_contraflow(&mut self, uid: OperatorId, insight: &mut Event) { + fn on_contraflow(&mut self, uid: OperatorUId, insight: &mut Event) { // If the related event never touched this operator we don't take // action if !insight.op_meta.contains_key(uid) { @@ -152,12 +153,12 @@ mod test { #![allow(clippy::float_cmp)] use super::*; use simd_json::ObjectHasher; - use tremor_common::ids::Id; + use tremor_common::uids::UId; use tremor_value::Object; #[test] fn pass_wo_error() { - let uid = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut op: Percentile = Config { timeout: 100_000_000, step_up: default_step_up(), @@ -175,7 +176,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event1) + .on_event(0, operator_id, &Port::In, &mut state, event1) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -191,7 +192,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event2) + .on_event(0, operator_id, &Port::In, &mut state, event2) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -208,7 +209,7 @@ mod test { step_up: default_step_up(), } .into(); - let uid = OperatorId::new(42); + let operator_id = OperatorUId::new(42); let mut state = Value::null(); @@ -220,7 +221,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event1) + .on_event(0, operator_id, &Port::In, &mut state, event1) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -238,7 +239,7 @@ mod test { let mut insight = event.insight_ack_with_timing(100_000_001); // Verify that we increased our percentage - op.on_contraflow(uid, &mut insight); + op.on_contraflow(operator_id, &mut insight); assert_eq!(0.95, op.perc); // now we have a good and fast event @@ -249,7 +250,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event2) + .on_event(0, operator_id, &Port::In, &mut state, event2) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -259,7 +260,7 @@ mod test { // less than timeout, we reset percentage a little let mut insight2 = event.insight_ack_with_timing(99); - op.on_contraflow(uid, &mut insight2); + op.on_contraflow(operator_id, &mut insight2); assert_eq!(0.951, op.perc); } @@ -271,13 +272,13 @@ mod test { step_up: 0.1, } .into(); - let uid = OperatorId::new(123); + let operator_id = OperatorUId::new(123); // An contraflow that fails the timeout let mut m = Object::with_hasher(ObjectHasher::default()); m.insert("time".into(), 200_000_000.into()); let mut op_meta = OpMeta::default(); - op_meta.insert(uid, OwnedValue::null()); + op_meta.insert(operator_id, OwnedValue::null()); let mut insight = Event { id: (1, 1, 1).into(), @@ -299,10 +300,10 @@ mod test { // Assert initial state assert_eq!(1.0, op.perc); // move one step down - op.on_contraflow(uid, &mut insight); + op.on_contraflow(operator_id, &mut insight); assert_eq!(0.75, op.perc); // move one step down - op.on_contraflow(uid, &mut insight); + op.on_contraflow(operator_id, &mut insight); assert_eq!(0.5, op.perc); let event = Event { @@ -312,7 +313,7 @@ mod test { }; let mut state = Value::null(); let mut events = op - .on_event(uid, &Port::In, &mut state, event) + .on_event(0, operator_id, &Port::In, &mut state, event) .expect("could not run pipeline") .events; @@ -323,7 +324,7 @@ mod test { assert_eq!("overflow", out); // Now we should reset - op.on_contraflow(uid, &mut insight_reset); + op.on_contraflow(operator_id, &mut insight_reset); assert_eq!(0.6, op.perc); } } diff --git a/tremor-pipeline/src/op/qos/roundrobin.rs b/tremor-pipeline/src/op/qos/roundrobin.rs index eef0d620c7..d97a132f3a 100644 --- a/tremor-pipeline/src/op/qos/roundrobin.rs +++ b/tremor-pipeline/src/op/qos/roundrobin.rs @@ -96,7 +96,8 @@ if let Some(map) = &node.config { impl Operator for RoundRobin { fn on_event( &mut self, - uid: OperatorId, + _node_id: u64, + uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, mut event: Event, @@ -126,7 +127,8 @@ impl Operator for RoundRobin { } fn on_signal( &mut self, - _uid: OperatorId, + _node_id: u64, + _uid: OperatorUId, _state: &mut Value<'static>, signal: &mut Event, ) -> Result { @@ -147,7 +149,7 @@ impl Operator for RoundRobin { true } - fn on_contraflow(&mut self, uid: OperatorId, insight: &mut Event) { + fn on_contraflow(&mut self, uid: OperatorUId, insight: &mut Event) { let RoundRobin { ref mut outputs, .. } = *self; @@ -182,13 +184,13 @@ impl Operator for RoundRobin { #[cfg(test)] mod test { - use tremor_common::ids::Id; + use tremor_common::uids::UId; use super::*; #[test] fn multi_output_block() { - let uid = OperatorId::new(0); + let operator_id = OperatorUId::new(0); let mut op: RoundRobin = Config { outputs: vec!["out".into(), "out2".into()], } @@ -204,7 +206,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event1) + .on_event(0, operator_id, &Port::In, &mut state, event1) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -219,7 +221,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event2) + .on_event(0, operator_id, &Port::In, &mut state, event2) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -228,7 +230,7 @@ mod test { // Mark output 0 as broken let mut op_meta = OpMeta::default(); - op_meta.insert(uid, 0); + op_meta.insert(operator_id, 0); let mut insight = Event { id: (1, 1, 1).into(), @@ -239,7 +241,7 @@ mod test { }; // Verify that we are broken on 0 - op.on_contraflow(uid, &mut insight); + op.on_contraflow(operator_id, &mut insight); assert!(!op.outputs[0].open); assert!(op.outputs[1].open); @@ -250,7 +252,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event2) + .on_event(0, operator_id, &Port::In, &mut state, event2) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -264,7 +266,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event3) + .on_event(0, operator_id, &Port::In, &mut state, event3) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); @@ -273,7 +275,7 @@ mod test { // Mark output 1 as restored let mut op_meta = OpMeta::default(); - op_meta.insert(uid, 0); + op_meta.insert(operator_id, 0); let mut insight = Event { id: (1, 1, 1).into(), @@ -284,7 +286,7 @@ mod test { }; // Verify that we now on disabled outputs - op.on_contraflow(uid, &mut insight); + op.on_contraflow(operator_id, &mut insight); assert!(op.outputs[0].open); assert!(op.outputs[1].open); @@ -295,7 +297,7 @@ mod test { ..Event::default() }; let mut r = op - .on_event(uid, &Port::In, &mut state, event3) + .on_event(0, operator_id, &Port::In, &mut state, event3) .expect("could not run pipeline") .events; assert_eq!(r.len(), 1); diff --git a/tremor-pipeline/src/op/trickle/operator.rs b/tremor-pipeline/src/op/trickle/operator.rs index 4bbd28016e..5d8449922c 100644 --- a/tremor-pipeline/src/op/trickle/operator.rs +++ b/tremor-pipeline/src/op/trickle/operator.rs @@ -42,7 +42,7 @@ fn mk_node_config(id: String, op_type: String, config: Value) -> NodeConfig { impl TrickleOperator { pub fn with_stmt( - operator_uid: OperatorId, + operator_uid: OperatorUId, defn: &ast::OperatorDefinition<'static>, helper: &mut Helper, ) -> Result { @@ -64,12 +64,13 @@ impl TrickleOperator { impl Operator for TrickleOperator { fn on_event( &mut self, - uid: OperatorId, + node_id: u64, + uid: OperatorUId, port: &Port<'static>, state: &mut Value<'static>, event: Event, ) -> Result { - self.op.on_event(uid, port, state, event) + self.op.on_event(node_id, uid, port, state, event) } fn handles_signal(&self) -> bool { @@ -77,17 +78,18 @@ impl Operator for TrickleOperator { } fn on_signal( &mut self, - uid: OperatorId, + node_id: u64, + uid: OperatorUId, state: &mut Value<'static>, signal: &mut Event, ) -> Result { - self.op.on_signal(uid, state, signal) + self.op.on_signal(node_id, uid, state, signal) } fn handles_contraflow(&self) -> bool { self.op.handles_contraflow() } - fn on_contraflow(&mut self, uid: OperatorId, contraevent: &mut Event) { + fn on_contraflow(&mut self, uid: OperatorUId, contraevent: &mut Event) { self.op.on_contraflow(uid, contraevent); } diff --git a/tremor-pipeline/src/op/trickle/script.rs b/tremor-pipeline/src/op/trickle/script.rs index c64b841670..609473b75c 100644 --- a/tremor-pipeline/src/op/trickle/script.rs +++ b/tremor-pipeline/src/op/trickle/script.rs @@ -39,12 +39,13 @@ impl Operator for Script { } fn on_event( &mut self, - _uid: OperatorId, + node_id: u64, + _uid: OperatorUId, port: &Port<'static>, state: &mut Value<'static>, mut event: Event, ) -> Result { - let context = EventContext::new(event.ingest_ns, event.origin_uri.as_ref()); + let context = EventContext::new(event.ingest_ns, event.origin_uri.as_ref(), node_id); let dst_port = event.data.rent_mut(|data| { let (unwind_event, event_meta): (&mut Value, &mut Value) = data.parts_mut(); diff --git a/tremor-pipeline/src/op/trickle/select.rs b/tremor-pipeline/src/op/trickle/select.rs index a25fdcc628..d175f3ba9e 100644 --- a/tremor-pipeline/src/op/trickle/select.rs +++ b/tremor-pipeline/src/op/trickle/select.rs @@ -46,7 +46,7 @@ pub(crate) struct Select { impl Select { pub fn from_stmt( - operator_uid: OperatorId, + operator_uid: OperatorUId, windows: Vec<(String, window::Impl)>, select: &ast::SelectStmt<'static>, ) -> Self { @@ -170,7 +170,8 @@ impl Operator for Select { // so the state can never be changed. fn on_event( &mut self, - _uid: OperatorId, + node_id: u64, + _uid: OperatorUId, _port: &Port<'static>, _state: &mut Value<'static>, mut event: Event, @@ -194,7 +195,7 @@ impl Operator for Select { .. } = event; - let mut ctx = EventContext::new(ingest_ns, origin_uri.as_ref()); + let mut ctx = EventContext::new(ingest_ns, origin_uri.as_ref(), node_id); ctx.cardinality = groups.len(); let opts = Self::opts(); @@ -273,7 +274,7 @@ impl Operator for Select { Entry::Occupied(mut o) => { // If we found a group execute it, and remove it if it is not longer // needed - if stry!(o.get_mut().on_event(sel_ctx, consts, event, &mut events)) { + if stry!(o.get_mut().on_event(node_id, sel_ctx, consts, event, &mut events)) { o.remove(); } } @@ -283,7 +284,7 @@ impl Operator for Select { dflt_group.value = group_value; dflt_group.value.try_push(v.key().to_string()); // execute it - if !stry!(dflt_group.on_event(sel_ctx, consts, event, &mut events)) { + if !stry!(dflt_group.on_event(node_id, sel_ctx, consts, event, &mut events)) { // if we can't delete it check if we're having too many groups, // if so, error. if ctx.cardinality >= *max_groups { @@ -310,7 +311,8 @@ impl Operator for Select { fn on_signal( &mut self, - _uid: OperatorId, + node_id: u64, + _uid: OperatorUId, _state: &mut Value<'static>, signal: &mut Event, ) -> Result { @@ -349,13 +351,13 @@ impl Operator for Select { consts.group = Value::const_null(); consts.args = Value::const_null(); - let mut ctx = EventContext::new(ingest_ns, None); + let mut ctx = EventContext::new(ingest_ns, None, node_id); ctx.cardinality = groups.len(); let mut to_remove = vec![]; for (group_str, g) in groups.iter_mut() { if let Some(w) = &mut g.windows { - let window_event = w.window.on_tick(ingest_ns)?; + let window_event = w.window.on_tick(node_id, ingest_ns)?; let mut can_remove = window_event.emit; if window_event.emit { @@ -395,6 +397,7 @@ impl Operator for Select { if let Some(next) = &mut w.next { can_remove = next.on_event( + node_id, &mut ctx, run, &mut data, diff --git a/tremor-pipeline/src/op/trickle/select/test.rs b/tremor-pipeline/src/op/trickle/select/test.rs index f01048f1c9..171365a211 100644 --- a/tremor-pipeline/src/op/trickle/select/test.rs +++ b/tremor-pipeline/src/op/trickle/select/test.rs @@ -21,7 +21,7 @@ use crate::EventId; use super::*; -use tremor_common::ids::Id; +use tremor_common::uids::UId; use tremor_script::ast::{self, Helper, Ident, Literal}; use tremor_script::{ast::Consts, NodeMeta}; use tremor_script::{ @@ -30,8 +30,8 @@ use tremor_script::{ }; use tremor_value::{literal, Value}; -fn test_uid() -> OperatorId { - OperatorId::new(42) +fn test_uid() -> OperatorUId { + OperatorUId::new(42) } fn test_select_stmt(stmt: tremor_script::ast::Select) -> SelectStmt { @@ -102,7 +102,7 @@ fn test_event_tx(s: u64, transactional: bool, group: u64) -> Event { } } -fn test_select(uid: OperatorId, stmt: &SelectStmt<'static>) -> Select { +fn test_select(uid: OperatorUId, stmt: &SelectStmt<'static>) -> Select { let windows = vec![ ( "w15s".into(), @@ -128,7 +128,7 @@ fn test_select(uid: OperatorId, stmt: &SelectStmt<'static>) -> Select { fn try_enqueue(op: &mut Select, event: Event) -> Result, Event)>> { let mut state = Value::null(); - let mut action = op.on_event(test_uid(), &Port::In, &mut state, event)?; + let mut action = op.on_event(0, test_uid(), &Port::In, &mut state, event)?; let first = action.events.pop(); if action.events.is_empty() { Ok(first) @@ -140,7 +140,7 @@ fn try_enqueue(op: &mut Select, event: Event) -> Result, E #[allow(clippy::type_complexity)] fn try_enqueue_two(op: &mut Select, event: Event) -> Result, Event); 2]>> { let mut state = Value::null(); - let mut action = op.on_event(test_uid(), &Port::In, &mut state, event)?; + let mut action = op.on_event(0, test_uid(), &Port::In, &mut state, event)?; let r = action .events .pop() @@ -155,7 +155,7 @@ fn try_enqueue_two(op: &mut Select, event: Event) -> Result Result { let reg = tremor_script::registry(); let aggr_reg = tremor_script::aggr_registry(); - let query = tremor_script::query::Query::parse(query, ®, &aggr_reg)?; + let query = tremor_script::query::Query::parse(&query, ®, &aggr_reg)?; let stmt = query .query .stmts @@ -203,7 +203,7 @@ fn as_select<'a, 'b>(stmt: &'a Stmt<'b>) -> Option<&'a SelectStmt<'b>> { fn select_stmt_from_query(query_str: &str) -> Result