diff --git a/Makefile b/Makefile index a86168488..305519239 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ FRONTEND_INSTALL_WIZARD_SRC := $(shell find frontend/projects/install-wizard) PATCH_DB_CLIENT_SRC := $(shell find patch-db/client -not -path patch-db/client/dist -and -not -path patch-db/client/node_modules) GZIP_BIN := $(shell which pigz || which gzip) TAR_BIN := $(shell which gtar || which tar) -ALL_TARGETS := $(EMBASSY_BINS) system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(EMBASSY_SRC) $(shell if [ "$(OS_ARCH)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep; fi) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE) +ALL_TARGETS := $(EMBASSY_BINS) system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(EMBASSY_SRC) $(shell if [ "$(OS_ARCH)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep; fi) $(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]; then echo cargo-deps/$(ARCH)-unknown-linux-gnu/release/tokio-console; fi') $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE) ifeq ($(REMOTE),) mkdir = mkdir -p $1 @@ -75,7 +75,7 @@ format: sdk: cd backend/ && ./install-sdk.sh -startos_raspberrypi.img: $(BUILD_SRC) startos.raspberrypi.squashfs $(VERSION_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep | sudo +startos_raspberrypi.img: $(BUILD_SRC) startos.raspberrypi.squashfs $(VERSION_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) | sudo ./build/raspberrypi/make-image.sh # For creating os images. DO NOT USE @@ -88,6 +88,7 @@ install: $(ALL_TARGETS) $(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/avahi-alias) $(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/embassy-cli) if [ "$(OS_ARCH)" = "raspberrypi" ]; then $(call cp,cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep,$(DESTDIR)/usr/bin/pi-beep); fi + if /bin/bash -c '[[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]'; then $(call cp,cargo-deps/$(ARCH)-unknown-linux-gnu/release/tokio-console,$(DESTDIR)/usr/bin/tokio-console); fi $(call mkdir,$(DESTDIR)/usr/lib) $(call rm,$(DESTDIR)/usr/lib/embassy) @@ -197,4 +198,7 @@ ui: frontend/dist/raw/ui backend: $(EMBASSY_BINS) cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep: - ARCH=aarch64 ./build-cargo-dep.sh pi-beep \ No newline at end of file + ARCH=aarch64 ./build-cargo-dep.sh pi-beep + +cargo-deps/$(ARCH)-unknown-linux-gnu/release/tokio-console: + ARCH=$(ARCH) ./build-cargo-dep.sh tokio-console \ No newline at end of file diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3c82288fa..8b7eeebac 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -250,6 +250,51 @@ dependencies = [ "libc", ] +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.68" @@ -655,6 +700,43 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "console-api" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime 2.1.0", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.4" @@ -802,6 +884,16 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -1816,6 +1908,19 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "hdrhistogram" +version = "7.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +dependencies = [ + "base64 0.13.1", + "byteorder", + "flate2", + "nom 7.1.3", + "num-traits", +] + [[package]] name = "heck" version = "0.4.1" @@ -1970,6 +2075,18 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2607,6 +2724,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "mbrman" version = "0.5.2" @@ -3500,6 +3623,38 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "prost" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.18", +] + +[[package]] +name = "prost-types" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +dependencies = [ + "prost", +] + [[package]] name = "psl-types" version = "2.0.11" @@ -4642,6 +4797,7 @@ dependencies = [ "clap 3.2.25", "color-eyre", "console", + "console-subscriber", "cookie", "cookie_store 0.19.1", "current_platform", @@ -5221,6 +5377,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "tap" version = "1.0.1" @@ -5416,9 +5578,20 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -5543,6 +5716,33 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.2", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "torut" version = "0.2.1" @@ -5563,6 +5763,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 461ae48c7..a1fc3914d 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -28,7 +28,7 @@ path = "src/main.rs" avahi = ["avahi-sys"] default = ["cli", "sdk", "daemon", "js_engine"] dev = [] -unstable = ["patch-db/unstable"] +unstable = ["console-subscriber", "tokio/tracing"] avahi-alias = ["avahi"] cli = [] sdk = [] @@ -55,6 +55,7 @@ bytes = "1" chrono = { version = "0.4.19", features = ["serde"] } clap = "3.2.8" color-eyre = "0.6.1" +console-subscriber = { version = "0.2", optional = true } cookie = "0.16.2" cookie_store = "0.19.0" current_platform = "0.2.0" diff --git a/backend/build-prod.sh b/backend/build-prod.sh index 97ca33279..c679fdf98 100755 --- a/backend/build-prod.sh +++ b/backend/build-prod.sh @@ -22,49 +22,38 @@ if tty -s; then USE_TTY="-it" fi -alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64' -alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl' - cd .. FLAGS="" +RUSTFLAGS="" if [[ "$ENVIRONMENT" =~ (^|-)podman($|-) ]]; then FLAGS="podman,$FLAGS" fi if [[ "$ENVIRONMENT" =~ (^|-)unstable($|-) ]]; then FLAGS="unstable,$FLAGS" + RUSTFLAGS="$RUSTFLAGS --cfg tokio_unstable" fi if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then FLAGS="dev,$FLAGS" fi +alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64' +alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl' + set +e fail= -if [[ "$FLAGS" = "" ]]; then - rust-gnu-builder sh -c "(cd backend && cargo build --release --locked --features avahi-alias, --target=$ARCH-unknown-linux-gnu)" - if test $? -ne 0; then - fail=true - fi - for ARCH in x86_64 aarch64 - do - rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init )" - if test $? -ne 0; then - fail=true - fi - done -else - echo "FLAGS=$FLAGS" - rust-gnu-builder sh -c "(cd backend && cargo build --release --features avahi-alias,$FLAGS --locked --target=$ARCH-unknown-linux-gnu)" +echo "FLAGS=\"$FLAGS\"" +echo "RUSTFLAGS=\"$RUSTFLAGS\"" +rust-gnu-builder sh -c "(cd backend && cargo build --release --features avahi-alias,$FLAGS --locked --target=$ARCH-unknown-linux-gnu)" +if test $? -ne 0; then + fail=true +fi +for ARCH in x86_64 aarch64 +do + rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init)" if test $? -ne 0; then fail=true fi - for ARCH in x86_64 aarch64 - do - rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init)" - if test $? -ne 0; then - fail=true - fi - done -fi +done set -e cd backend diff --git a/backend/src/net/vhost.rs b/backend/src/net/vhost.rs index e8e754baf..00ffc6c59 100644 --- a/backend/src/net/vhost.rs +++ b/backend/src/net/vhost.rs @@ -16,12 +16,13 @@ use tokio::sync::{Mutex, RwLock}; use tokio_rustls::rustls::server::Acceptor; use tokio_rustls::rustls::{RootCertStore, ServerConfig}; use tokio_rustls::{LazyConfigAcceptor, TlsConnector}; +use tracing::instrument; use crate::net::keys::Key; use crate::net::ssl::SslManager; use crate::net::utils::SingleAccept; +use crate::prelude::*; use crate::util::io::{BackTrackingReader, TimeoutStream}; -use crate::Error; // not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353 @@ -36,6 +37,7 @@ impl VHostController { servers: Mutex::new(BTreeMap::new()), } } + #[instrument(skip_all)] pub async fn add( &self, key: Key, @@ -63,6 +65,7 @@ impl VHostController { writable.insert(external, server); Ok(rc?) } + #[instrument(skip_all)] pub async fn gc(&self, hostname: Option, external: u16) -> Result<(), Error> { let mut writable = self.servers.lock().await; if let Some(server) = writable.remove(&external) { @@ -93,6 +96,7 @@ struct VHostServer { _thread: NonDetachingJoinHandle<()>, } impl VHostServer { + #[instrument(skip_all)] async fn new(port: u16, ssl: Arc) -> Result { // check if port allowed let listener = TcpListener::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port)) diff --git a/backend/src/util/logger.rs b/backend/src/util/logger.rs index 8208f7de6..c7ab41ba2 100644 --- a/backend/src/util/logger.rs +++ b/backend/src/util/logger.rs @@ -10,13 +10,28 @@ impl EmbassyLogger { use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; - let filter_layer = EnvFilter::from_default_env(); + let filter_layer = EnvFilter::builder() + .with_default_directive( + format!("{}=info", std::module_path!().split("::").next().unwrap()) + .parse() + .unwrap(), + ) + .from_env_lossy(); + #[cfg(feature = "unstable")] + let filter_layer = filter_layer + .add_directive("tokio=trace".parse().unwrap()) + .add_directive("runtime=trace".parse().unwrap()); let fmt_layer = fmt::layer().with_target(true); - tracing_subscriber::registry() + let sub = tracing_subscriber::registry() .with(filter_layer) .with(fmt_layer) - .with(ErrorLayer::default()) + .with(ErrorLayer::default()); + + #[cfg(feature = "unstable")] + let sub = sub.with(console_subscriber::spawn()); + + sub } pub fn init() -> Self { Self::base_subscriber().init();