From 0e0b7b82a266b57aac850aae32cb8016a451bb8f Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 15 Aug 2024 15:09:50 -0600 Subject: [PATCH 01/29] wip: enabling support for wireguard and firewall --- build/dpkg-deps/depends | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/dpkg-deps/depends b/build/dpkg-deps/depends index 3ccaee4d6..62428bfdc 100644 --- a/build/dpkg-deps/depends +++ b/build/dpkg-deps/depends @@ -10,6 +10,7 @@ cifs-utils cryptsetup curl dmidecode +dnsutils dosfstools e2fsprogs ecryptfs-utils @@ -53,6 +54,8 @@ systemd-resolved systemd-sysv systemd-timesyncd tor +ufw util-linux vim +wireguard-tools wireless-tools From 25aa6273314690444cdde123cd34c2afb6886343 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Fri, 22 Nov 2024 17:30:00 -0700 Subject: [PATCH 02/29] wip --- build/lib/scripts/dhclient-exit-hook | 1 - core/Cargo.lock | 405 +++++++- core/models/Cargo.toml | 1 + core/models/src/errors.rs | 7 + core/startos/Cargo.toml | 1 + core/startos/src/db/model/public.rs | 14 +- core/startos/src/init.rs | 1 - core/startos/src/lxc/dev.rs | 6 +- core/startos/src/lxc/mod.rs | 3 +- core/startos/src/net/dhcp.rs | 99 -- core/startos/src/net/host/binding.rs | 13 +- core/startos/src/net/mod.rs | 10 +- core/startos/src/net/net_controller.rs | 245 ++--- core/startos/src/net/network_interface.rs | 154 +++ core/startos/src/net/service_interface.rs | 3 +- core/startos/src/net/vhost.rs | 898 +++++++++--------- core/startos/src/service/effects/net/bind.rs | 4 +- .../src/service/effects/subcontainer/mod.rs | 5 +- core/startos/src/service/mod.rs | 4 +- core/startos/src/util/mod.rs | 6 +- 20 files changed, 1205 insertions(+), 675 deletions(-) delete mode 100755 build/lib/scripts/dhclient-exit-hook delete mode 100644 core/startos/src/net/dhcp.rs create mode 100644 core/startos/src/net/network_interface.rs diff --git a/build/lib/scripts/dhclient-exit-hook b/build/lib/scripts/dhclient-exit-hook deleted file mode 100755 index 8c4a97746..000000000 --- a/build/lib/scripts/dhclient-exit-hook +++ /dev/null @@ -1 +0,0 @@ -start-cli net dhcp update $interface \ No newline at end of file diff --git a/core/Cargo.lock b/core/Cargo.lock index 498e5903e..db5055e87 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -254,6 +254,18 @@ dependencies = [ "x509-parser", ] +[[package]] +name = "async-broadcast" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -261,10 +273,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue 2.5.0", - "event-listener", + "event-listener 2.5.3", "futures-core", ] +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue 2.5.0", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.17" @@ -279,6 +303,108 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-executor" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec" +dependencies = [ + "async-task", + "concurrent-queue 2.5.0", + "fastrand", + "futures-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcd09b382f40fcd159c2d695175b2ae620ffa5f3bd6f664131efff4e8b9e04a" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-io" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" +dependencies = [ + "async-lock", + "cfg-if", + "concurrent-queue 2.5.0", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" +dependencies = [ + "async-channel 2.3.1", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener 5.3.1", + "futures-lite", + "rustix", + "tracing", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "async-signal" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.59.0", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -301,6 +427,12 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.83" @@ -520,7 +652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be5951c75bdabb58753d140dd5802f12ff3a483cb2e16fb5276e111b94b19e87" dependencies = [ "concurrent-queue 1.2.4", - "event-listener", + "event-listener 2.5.3", "spin", ] @@ -707,6 +839,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" +dependencies = [ + "async-channel 2.3.1", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "brotli" version = "7.0.0" @@ -1676,6 +1821,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endi" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d8a32ae18130a3c84dd492d4215c3d913c3b07c6b63c2eb3eb7ff1101ab7bf" + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -1688,6 +1839,27 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "enumflags2" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d232db7f5956f3f14313dc2f87985c58bd2c695ce124c8cdd984e08e15ac133d" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1742,6 +1914,27 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue 2.5.0", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener 5.3.1", + "pin-project-lite", +] + [[package]] name = "exver" version = "0.2.0" @@ -1956,6 +2149,19 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-lite" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.31" @@ -3029,7 +3235,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06cf485d4867e0714e35c1652e736bcf892d28fceecca01036764575db64ba84" dependencies = [ - "async-channel", + "async-channel 1.9.0", "futures", ] @@ -3233,6 +3439,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -3311,6 +3526,7 @@ dependencies = [ "tracing", "ts-rs", "yasi", + "zbus", ] [[package]] @@ -3381,6 +3597,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset 0.9.1", ] [[package]] @@ -3656,6 +3873,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + [[package]] name = "overload" version = "0.1.1" @@ -3706,6 +3933,12 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.3" @@ -3913,6 +4146,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -3940,6 +4184,21 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "polling" +version = "3.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" +dependencies = [ + "cfg-if", + "concurrent-queue 2.5.0", + "hermit-abi 0.4.0", + "pin-project-lite", + "rustix", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "portable-atomic" version = "1.9.0" @@ -4836,6 +5095,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -5120,7 +5390,7 @@ dependencies = [ "crc", "crossbeam-queue", "either", - "event-listener", + "event-listener 2.5.3", "futures-channel", "futures-core", "futures-intrusive", @@ -5502,6 +5772,7 @@ dependencies = [ "url", "urlencoding", "uuid", + "zbus", "zeroize", ] @@ -6358,6 +6629,17 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "uds_windows" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89daebc3e6fd160ac4aa9fc8b3bf71e1f74fbf92367ae71fb83a037e8bf164b9" +dependencies = [ + "memoffset 0.9.1", + "tempfile", + "winapi", +] + [[package]] name = "unarray" version = "0.1.4" @@ -7001,6 +7283,16 @@ dependencies = [ "rustix", ] +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "xz2" version = "0.1.7" @@ -7067,6 +7359,69 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zbus" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1162094dc63b1629fcc44150bcceeaa80798cd28bcbe7fa987b65a034c258608" +dependencies = [ + "async-broadcast", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener 5.3.1", + "futures-core", + "futures-util", + "hex", + "nix 0.29.0", + "ordered-stream", + "serde", + "serde_repr", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.59.0", + "winnow 0.6.20", + "xdg-home", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd2dcdce3e2727f7d74b7e33b5a89539b3cc31049562137faf7ae4eb86cd16d" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.87", + "zbus_names", + "zvariant", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "856b7a38811f71846fd47856ceee8bccaec8399ff53fb370247e66081ace647b" +dependencies = [ + "serde", + "static_assertions", + "winnow 0.6.20", + "zvariant", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -7178,3 +7533,45 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zvariant" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1200ee6ac32f1e5a312e455a949a4794855515d34f9909f4a3e082d14e1a56f" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "winnow 0.6.20", + "zvariant_derive", + "zvariant_utils", +] + +[[package]] +name = "zvariant_derive" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687e3b97fae6c9104fbbd36c73d27d149abf04fb874e2efbd84838763daa8916" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.87", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20d1d011a38f12360e5fcccceeff5e2c42a8eb7f27f0dcba97a0862ede05c9c6" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "static_assertions", + "syn 2.0.87", + "winnow 0.6.20", +] diff --git a/core/models/Cargo.toml b/core/models/Cargo.toml index 44295745d..b402cb219 100644 --- a/core/models/Cargo.toml +++ b/core/models/Cargo.toml @@ -39,3 +39,4 @@ tokio = { version = "1", features = ["full"] } torut = { git = "https://github.com/Start9Labs/torut.git", branch = "update/dependencies" } tracing = "0.1.39" yasi = "0.1.5" +zbus = "5" diff --git a/core/models/src/errors.rs b/core/models/src/errors.rs index 2077a8bbd..eba2a377f 100644 --- a/core/models/src/errors.rs +++ b/core/models/src/errors.rs @@ -90,6 +90,7 @@ pub enum ErrorKind { Lxc = 72, Cancelled = 73, Git = 74, + DBus = 75, } impl ErrorKind { pub fn as_str(&self) -> &'static str { @@ -169,6 +170,7 @@ impl ErrorKind { Lxc => "LXC Error", Cancelled => "Cancelled", Git => "Git Error", + DBus => "DBus Error", } } } @@ -327,6 +329,11 @@ impl From for Error { Error::new(e, ErrorKind::Tor) } } +impl From for Error { + fn from(e: zbus::Error) -> Self { + Error::new(e, ErrorKind::DBus) + } +} impl From for Error { fn from(value: patch_db::value::Error) -> Self { match value.kind { diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index e2a20a0a1..cfe1e3347 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -216,6 +216,7 @@ unix-named-pipe = "0.2.0" url = { version = "2.4.1", features = ["serde"] } urlencoding = "2.1.3" uuid = { version = "1.4.1", features = ["v4"] } +zbus = "5.1.1" zeroize = "1.6.0" [profile.test] diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 85978134d..380d88ac5 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -54,7 +54,7 @@ impl Public { tor_address: format!("https://{}", account.tor_key.public().get_onion_address()) .parse() .unwrap(), - ip_info: BTreeMap::new(), + network_interfaces: BTreeMap::new(), acme: None, status_info: ServerStatus { backup_progress: None, @@ -130,7 +130,8 @@ pub struct ServerInfo { /// for backwards compatibility #[ts(type = "string")] pub tor_address: Url, - pub ip_info: BTreeMap, + #[ts(as = "BTreeMap::")] + pub network_interfaces: BTreeMap, pub acme: Option, #[serde(default)] pub status_info: ServerStatus, @@ -155,6 +156,15 @@ pub struct ServerInfo { #[serde(rename_all = "camelCase")] #[model = "Model"] #[ts(export)] +pub struct NetworkInterfaceInfo { + pub public: bool, + pub ip_info: IpInfo, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] pub struct IpInfo { #[ts(type = "string | null")] pub ipv4_range: Option, diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index a81e7e336..8bdff7ff2 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -509,7 +509,6 @@ pub async fn init( enable_zram.complete(); update_server_info.start(); - server_info.ip_info = crate::net::dhcp::init_ips().await?; server_info.ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024; server_info.devices = lshw().await?; server_info.status_info = ServerStatus { diff --git a/core/startos/src/lxc/dev.rs b/core/startos/src/lxc/dev.rs index 248546d88..a918672da 100644 --- a/core/startos/src/lxc/dev.rs +++ b/core/startos/src/lxc/dev.rs @@ -8,13 +8,11 @@ use rpc_toolkit::{ use serde::{Deserialize, Serialize}; use ts_rs::TS; +use crate::context::{CliContext, RpcContext}; use crate::lxc::{ContainerId, LxcConfig}; use crate::prelude::*; use crate::rpc_continuations::Guid; -use crate::{ - context::{CliContext, RpcContext}, - service::ServiceStats, -}; +use crate::service::ServiceStats; pub fn lxc() -> ParentHandler { ParentHandler::new() diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index c0fb6eaba..60f9f4301 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -1,8 +1,9 @@ +use std::collections::BTreeSet; +use std::ffi::OsString; use std::net::Ipv4Addr; use std::path::Path; use std::sync::{Arc, Weak}; use std::time::Duration; -use std::{collections::BTreeSet, ffi::OsString}; use clap::builder::ValueParserFactory; use futures::{AsyncWriteExt, StreamExt}; diff --git a/core/startos/src/net/dhcp.rs b/core/startos/src/net/dhcp.rs deleted file mode 100644 index e323ba371..000000000 --- a/core/startos/src/net/dhcp.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::net::IpAddr; - -use clap::Parser; -use futures::TryStreamExt; -use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; -use serde::{Deserialize, Serialize}; -use tokio::sync::RwLock; -use ts_rs::TS; - -use crate::context::{CliContext, RpcContext}; -use crate::db::model::public::IpInfo; -use crate::net::utils::{iface_is_physical, list_interfaces}; -use crate::prelude::*; -use crate::Error; - -lazy_static::lazy_static! { - static ref CACHED_IPS: RwLock> = RwLock::new(BTreeSet::new()); -} - -async fn _ips() -> Result, Error> { - Ok(init_ips() - .await? - .values() - .flat_map(|i| { - std::iter::empty() - .chain(i.ipv4.map(IpAddr::from)) - .chain(i.ipv6.map(IpAddr::from)) - }) - .collect()) -} - -pub async fn ips() -> Result, Error> { - let ips = CACHED_IPS.read().await.clone(); - if !ips.is_empty() { - return Ok(ips); - } - let ips = _ips().await?; - *CACHED_IPS.write().await = ips.clone(); - Ok(ips) -} - -pub async fn init_ips() -> Result, Error> { - let mut res = BTreeMap::new(); - let mut ifaces = list_interfaces(); - while let Some(iface) = ifaces.try_next().await? { - if iface_is_physical(&iface).await { - let ip_info = IpInfo::for_interface(&iface).await?; - res.insert(iface, ip_info); - } - } - Ok(res) -} - -// #[command(subcommands(update))] -pub fn dhcp() -> ParentHandler { - ParentHandler::new().subcommand( - "update", - from_fn_async::<_, _, (), Error, (RpcContext, UpdateParams)>(update) - .no_display() - .with_about("Update IP assigned by dhcp") - .with_call_remote::(), - ) -} -#[derive(Deserialize, Serialize, Parser, TS)] -#[serde(rename_all = "camelCase")] -#[command(rename_all = "kebab-case")] -pub struct UpdateParams { - interface: String, -} - -pub async fn update( - ctx: RpcContext, - UpdateParams { interface }: UpdateParams, -) -> Result<(), Error> { - if iface_is_physical(&interface).await { - let ip_info = IpInfo::for_interface(&interface).await?; - ctx.db - .mutate(|db| { - db.as_public_mut() - .as_server_info_mut() - .as_ip_info_mut() - .insert(&interface, &ip_info) - }) - .await?; - - let mut cached = CACHED_IPS.write().await; - if cached.is_empty() { - *cached = _ips().await?; - } else { - cached.extend( - std::iter::empty() - .chain(ip_info.ipv4.map(IpAddr::from)) - .chain(ip_info.ipv6.map(IpAddr::from)), - ); - } - } - Ok(()) -} diff --git a/core/startos/src/net/host/binding.rs b/core/startos/src/net/host/binding.rs index 174f0330f..41261b7c6 100644 --- a/core/startos/src/net/host/binding.rs +++ b/core/startos/src/net/host/binding.rs @@ -41,12 +41,14 @@ impl FromStr for BindId { pub struct BindInfo { pub enabled: bool, pub options: BindOptions, - pub lan: LanInfo, + pub net: NetInfo, } + #[derive(Clone, Copy, Debug, Deserialize, Serialize, TS, PartialEq, Eq, PartialOrd, Ord)] #[serde(rename_all = "camelCase")] #[ts(export)] -pub struct LanInfo { +pub struct NetInfo { + pub public: bool, pub assigned_port: Option, pub assigned_ssl_port: Option, } @@ -63,7 +65,8 @@ impl BindInfo { Ok(Self { enabled: true, options, - lan: LanInfo { + net: NetInfo { + public: false, assigned_port, assigned_ssl_port, }, @@ -74,7 +77,7 @@ impl BindInfo { available_ports: &mut AvailablePorts, options: BindOptions, ) -> Result { - let Self { mut lan, .. } = self; + let Self { net: mut lan, .. } = self; if options .secure .map_or(false, |s| !(s.ssl && options.add_ssl.is_some())) @@ -104,7 +107,7 @@ impl BindInfo { Ok(Self { enabled: true, options, - lan, + net: lan, }) } pub fn disable(&mut self) { diff --git a/core/startos/src/net/mod.rs b/core/startos/src/net/mod.rs index 53b94454d..9cf72c30b 100644 --- a/core/startos/src/net/mod.rs +++ b/core/startos/src/net/mod.rs @@ -1,13 +1,13 @@ use rpc_toolkit::{Context, HandlerExt, ParentHandler}; pub mod acme; -pub mod dhcp; pub mod dns; pub mod forward; pub mod host; pub mod keys; pub mod mdns; pub mod net_controller; +pub mod network_interface; pub mod service_interface; pub mod ssl; pub mod static_server; @@ -25,10 +25,10 @@ pub fn net() -> ParentHandler { "tor", tor::tor::().with_about("Tor commands such as list-services, logs, and reset"), ) - .subcommand( - "dhcp", - dhcp::dhcp::().with_about("Command to update IP assigned from dhcp"), - ) + // .subcommand( + // "dhcp", + // network_interface::dhcp::().with_about("Command to update IP assigned from dhcp"), + // ) .subcommand( "acme", acme::acme::().with_about("Setup automatic clearnet certificate acquisition"), diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index a8beaf55f..63b909a79 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -15,8 +15,9 @@ use crate::hostname::Hostname; use crate::net::dns::DnsController; use crate::net::forward::LanPortForwardController; use crate::net::host::address::HostAddress; -use crate::net::host::binding::{AddSslOptions, BindId, BindOptions, LanInfo}; +use crate::net::host::binding::{AddSslOptions, BindId, BindOptions, NetInfo}; use crate::net::host::{host_for, Host, HostKind, Hosts}; +use crate::net::network_interface::NetworkInterfaceController; use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname}; use crate::net::tor::TorController; use crate::net::vhost::{AlpnInfo, VHostController}; @@ -28,6 +29,7 @@ pub struct PreInitNetController { pub db: TypedPatchDb, tor: TorController, vhost: VHostController, + net_iface: Arc, os_bindings: Vec>, server_hostnames: Vec>, } @@ -40,10 +42,12 @@ impl PreInitNetController { hostname: &Hostname, os_tor_key: TorSecretKeyV3, ) -> Result { + let net_iface = Arc::new(NetworkInterfaceController::new(db.clone())); let mut res = Self { db: db.clone(), tor: TorController::new(tor_control, tor_socks), - vhost: VHostController::new(db), + vhost: VHostController::new(db, net_iface.clone()), + net_iface, os_bindings: Vec::new(), server_hostnames: Vec::new(), }; @@ -75,26 +79,25 @@ impl PreInitNetController { ]; for hostname in self.server_hostnames.iter().cloned() { - self.os_bindings.push( - self.vhost - .add(hostname, 443, ([127, 0, 0, 1], 80).into(), alpn.clone()) - .await?, - ); + self.os_bindings.push(self.vhost.add( + hostname, + 443, + false, + ([127, 0, 0, 1], 80).into(), + alpn.clone(), + )?); } // Tor - self.os_bindings.push( - self.vhost - .add( - Some(InternedString::from_display( - &tor_key.public().get_onion_address(), - )), - 443, - ([127, 0, 0, 1], 80).into(), - alpn.clone(), - ) - .await?, - ); + self.os_bindings.push(self.vhost.add( + Some(InternedString::from_display( + &tor_key.public().get_onion_address(), + )), + 443, + false, + ([127, 0, 0, 1], 80).into(), + alpn.clone(), + )?); self.os_bindings.extend( self.tor .add( @@ -115,6 +118,7 @@ pub struct NetController { db: TypedPatchDb, pub(super) tor: TorController, pub(super) vhost: VHostController, + pub(super) net_iface: Arc, pub(super) dns: DnsController, pub(super) forward: LanPortForwardController, pub(super) os_bindings: Vec>, @@ -127,6 +131,7 @@ impl NetController { db, tor, vhost, + net_iface, os_bindings, server_hostnames, }: PreInitNetController, @@ -136,6 +141,7 @@ impl NetController { db, tor, vhost, + net_iface, dns: DnsController::init(dns_bind).await?, forward: LanPortForwardController::new(), os_bindings, @@ -172,7 +178,7 @@ struct HostBinds { lan: BTreeMap< u16, ( - LanInfo, + NetInfo, Option, BTreeSet, Vec>, @@ -270,7 +276,7 @@ impl NetService { // LAN let server_info = peek.as_public().as_server_info(); - let ip_info = server_info.as_ip_info().de()?; + let net_ifaces = server_info.as_network_interfaces().de()?; let hostname = server_info.as_hostname().de()?; for (port, bind) in &host.bindings { if !bind.enabled { @@ -280,10 +286,10 @@ impl NetService { let lan_bind = old_lan_bind .as_ref() .filter(|(external, ssl, _, _)| { - ssl == &bind.options.add_ssl && bind.lan == *external + ssl == &bind.options.add_ssl && bind.net == *external }) .cloned(); // only keep existing binding if relevant details match - if bind.lan.assigned_port.is_some() || bind.lan.assigned_ssl_port.is_some() { + if bind.net.assigned_port.is_some() || bind.net.assigned_ssl_port.is_some() { let new_lan_bind = if let Some(b) = lan_bind { b } else { @@ -291,7 +297,7 @@ impl NetService { let mut hostnames = BTreeSet::new(); if let Some(ssl) = &bind.options.add_ssl { let external = bind - .lan + .net .assigned_ssl_port .or_not_found("assigned ssl port")?; let target = (self.ip, *port).into(); @@ -305,53 +311,53 @@ impl NetService { } }; for hostname in ctrl.server_hostnames.iter().cloned() { - rcs.push( - ctrl.vhost - .add(hostname, external, target, connect_ssl.clone()) - .await?, - ); + rcs.push(ctrl.vhost.add( + hostname, + external, + bind.net.public, + target, + connect_ssl.clone(), + )?); } for address in host.addresses() { match address { HostAddress::Onion { address } => { let hostname = InternedString::from_display(address); if hostnames.insert(hostname.clone()) { - rcs.push( - ctrl.vhost - .add( - Some(hostname), - external, - target, - connect_ssl.clone(), - ) - .await?, - ); + rcs.push(ctrl.vhost.add( + Some(hostname), + external, + false, + target, + connect_ssl.clone(), + )?); } } HostAddress::Domain { address } => { if hostnames.insert(address.clone()) { let address = Some(address.clone()); - rcs.push( - ctrl.vhost - .add( - address.clone(), - external, - target, - connect_ssl.clone(), - ) - .await?, - ); + rcs.push(ctrl.vhost.add( + address.clone(), + external, + bind.net.public, + target, + connect_ssl.clone(), + )?); if ssl.preferred_external_port == 443 { - rcs.push( - ctrl.vhost - .add( - address.clone(), - 5443, - target, - connect_ssl.clone(), - ) - .await?, - ); + rcs.push(ctrl.vhost.add( + address.clone(), + 5443, + false, + target, + connect_ssl.clone(), + )?); + rcs.push(ctrl.vhost.add( + address.clone(), + 443, + true, + target, + connect_ssl.clone(), + )?); } } } @@ -363,66 +369,83 @@ impl NetService { // doesn't make sense to have 2 listening ports, both with ssl } else { let external = - bind.lan.assigned_port.or_not_found("assigned lan port")?; + bind.net.assigned_port.or_not_found("assigned lan port")?; rcs.push(ctrl.forward.add(external, (self.ip, *port).into()).await?); } } - (bind.lan, bind.options.add_ssl.clone(), hostnames, rcs) + (bind.net, bind.options.add_ssl.clone(), hostnames, rcs) }; let mut bind_hostname_info: Vec = hostname_info.remove(port).unwrap_or_default(); - for (interface, ip_info) in &ip_info { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public: false, - hostname: IpHostname::Local { - value: InternedString::from_display(&{ - let hostname = &hostname; - lazy_format!("{hostname}.local") - }), - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, - }, - }); - for address in host.addresses() { - if let HostAddress::Domain { address } = address { - if let Some(ssl) = &new_lan_bind.1 { - if ssl.preferred_external_port == 443 { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public: false, - hostname: IpHostname::Domain { - domain: address.clone(), - subdomain: None, - port: None, - ssl_port: Some(443), - }, - }); - } - } - } - } - if let Some(ipv4) = ip_info.ipv4 { + for (interface, iface_info) in &net_ifaces { + if !iface_info.public { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), public: false, - hostname: IpHostname::Ipv4 { - value: ipv4, + hostname: IpHostname::Local { + value: InternedString::from_display(&{ + let hostname = &hostname; + lazy_format!("{hostname}.local") + }), port: new_lan_bind.0.assigned_port, ssl_port: new_lan_bind.0.assigned_ssl_port, }, }); } - if let Some(ipv6) = ip_info.ipv6 { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public: false, - hostname: IpHostname::Ipv6 { - value: ipv6, - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, - }, - }); + for address in host.addresses() { + if let HostAddress::Domain { address } = address { + if new_lan_bind + .1 + .as_ref() + .map_or(false, |ssl| ssl.preferred_external_port == 443) + { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: iface_info.public, + hostname: IpHostname::Domain { + domain: address.clone(), + subdomain: None, + port: None, + ssl_port: Some(443), + }, + }); + } else if iface_info.public && new_lan_bind.0.public { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: iface_info.public, + hostname: IpHostname::Domain { + domain: address.clone(), + subdomain: None, + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } + } + } + if !iface_info.public || new_lan_bind.0.public { + if let Some(ipv4) = iface_info.ip_info.ipv4 { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: iface_info.public, + hostname: IpHostname::Ipv4 { + value: ipv4, + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } + if let Some(ipv6) = iface_info.ip_info.ipv6 { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: iface_info.public, + hostname: IpHostname::Ipv6 { + value: ipv6, + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } } } hostname_info.insert(*port, bind_hostname_info); @@ -431,10 +454,10 @@ impl NetService { if let Some((lan, _, hostnames, _)) = old_lan_bind { if let Some(external) = lan.assigned_ssl_port { for hostname in ctrl.server_hostnames.iter().cloned() { - ctrl.vhost.gc(hostname, external).await?; + ctrl.vhost.gc(hostname, external)?; } for hostname in hostnames { - ctrl.vhost.gc(Some(hostname), external).await?; + ctrl.vhost.gc(Some(hostname), external)?; } } if let Some(external) = lan.assigned_port { @@ -455,10 +478,10 @@ impl NetService { for (lan, hostnames) in removed { if let Some(external) = lan.assigned_ssl_port { for hostname in ctrl.server_hostnames.iter().cloned() { - ctrl.vhost.gc(hostname, external).await?; + ctrl.vhost.gc(hostname, external)?; } for hostname in hostnames { - ctrl.vhost.gc(Some(hostname), external).await?; + ctrl.vhost.gc(Some(hostname), external)?; } } if let Some(external) = lan.assigned_port { @@ -481,7 +504,7 @@ impl NetService { SocketAddr::from((self.ip, *internal)), ); if let (Some(ssl), Some(ssl_internal)) = - (&info.options.add_ssl, info.lan.assigned_ssl_port) + (&info.options.add_ssl, info.net.assigned_ssl_port) { tor_binds.insert( ssl.preferred_external_port, @@ -580,7 +603,7 @@ impl NetService { self.ip } - pub fn get_lan_port(&self, host_id: HostId, internal_port: u16) -> Result { + pub fn get_lan_port(&self, host_id: HostId, internal_port: u16) -> Result { let host_id_binds = self.binds.get_key_value(&host_id); match host_id_binds { Some((_, binds)) => { diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs new file mode 100644 index 000000000..c657b83d1 --- /dev/null +++ b/core/startos/src/net/network_interface.rs @@ -0,0 +1,154 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::net::{IpAddr, SocketAddr}; +use std::sync::{Arc, Weak}; + +use clap::Parser; +use futures::TryStreamExt; +use imbl_value::InternedString; +use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; +use serde::{Deserialize, Serialize}; +use tokio::net::TcpStream; +use tokio::sync::RwLock; +use ts_rs::TS; +use zbus::zvariant::OwnedObjectPath; +use zbus::{proxy, Connection}; + +use crate::context::{CliContext, RpcContext}; +use crate::db::model::public::IpInfo; +use crate::db::model::Database; +use crate::net::utils::{iface_is_physical, list_interfaces}; +use crate::prelude::*; +use crate::util::sync::SyncMutex; + +#[proxy( + interface = "org.freedesktop.NetworkManager", + default_service = "org.freedesktop.NetworkManager", + default_path = "/org/freedesktop/NetworkManager" +)] +trait NetworkManager { + async fn get_all_devices(&self) -> Result, Error>; +} + +#[tokio::test] +async fn test() -> Result<(), Error> { + let connection = Connection::system().await?; + + let proxy = NetworkManagerProxy::new(&connection).await?; + let reply = proxy.get_all_devices().await?; + println!("{reply:?}"); + + Ok(()) +} + +pub struct NetworkInterfaceController { + db: TypedPatchDb, + listeners: SyncMutex>>, +} +impl NetworkInterfaceController { + pub fn new(db: TypedPatchDb) -> Self { + Self { + db, + listeners: SyncMutex::new(BTreeMap::new()), + } + } + + pub fn bind(&self, port: u16) -> Result { + todo!() + } +} + +pub struct NetworkInterfaceListener { + ctrl: Arc, +} +impl NetworkInterfaceListener { + pub async fn accept(&mut self, public: bool) -> Result { + todo!() + } +} + +pub struct Accepted { + pub stream: TcpStream, + pub peer: SocketAddr, + pub is_public: bool, + pub bind: SocketAddr, +} + +// async fn _ips() -> Result, Error> { +// Ok(init_ips() +// .await? +// .values() +// .flat_map(|i| { +// std::iter::empty() +// .chain(i.ipv4.map(IpAddr::from)) +// .chain(i.ipv6.map(IpAddr::from)) +// }) +// .collect()) +// } + +// pub async fn ips() -> Result, Error> { +// let ips = CACHED_IPS.read().await.clone(); +// if !ips.is_empty() { +// return Ok(ips); +// } +// let ips = _ips().await?; +// *CACHED_IPS.write().await = ips.clone(); +// Ok(ips) +// } + +// pub async fn init_ips() -> Result, Error> { +// let mut res = BTreeMap::new(); +// let mut ifaces = list_interfaces(); +// while let Some(iface) = ifaces.try_next().await? { +// if iface_is_physical(&iface).await { +// let ip_info = IpInfo::for_interface(&iface).await?; +// res.insert(iface, ip_info); +// } +// } +// Ok(res) +// } + +// // #[command(subcommands(update))] +// pub fn dhcp() -> ParentHandler { +// ParentHandler::new().subcommand( +// "update", +// from_fn_async::<_, _, (), Error, (RpcContext, UpdateParams)>(update) +// .no_display() +// .with_about("Update IP assigned by dhcp") +// .with_call_remote::(), +// ) +// } +// #[derive(Deserialize, Serialize, Parser, TS)] +// #[serde(rename_all = "camelCase")] +// #[command(rename_all = "kebab-case")] +// pub struct UpdateParams { +// interface: String, +// } + +// pub async fn update( +// ctx: RpcContext, +// UpdateParams { interface }: UpdateParams, +// ) -> Result<(), Error> { +// if iface_is_physical(&interface).await { +// let ip_info = IpInfo::for_interface(&interface).await?; +// ctx.db +// .mutate(|db| { +// db.as_public_mut() +// .as_server_info_mut() +// .as_ip_info_mut() +// .insert(&interface, &ip_info) +// }) +// .await?; + +// let mut cached = CACHED_IPS.write().await; +// if cached.is_empty() { +// *cached = _ips().await?; +// } else { +// cached.extend( +// std::iter::empty() +// .chain(ip_info.ipv4.map(IpAddr::from)) +// .chain(ip_info.ipv6.map(IpAddr::from)), +// ); +// } +// } +// Ok(()) +// } diff --git a/core/startos/src/net/service_interface.rs b/core/startos/src/net/service_interface.rs index ade10d959..b0a8a7676 100644 --- a/core/startos/src/net/service_interface.rs +++ b/core/startos/src/net/service_interface.rs @@ -12,7 +12,8 @@ use ts_rs::TS; #[serde(tag = "kind")] pub enum HostnameInfo { Ip { - network_interface_id: String, + #[ts(type = "string")] + network_interface_id: InternedString, public: bool, hostname: IpHostname, }, diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs index 7d48b1469..63c13e474 100644 --- a/core/startos/src/net/vhost.rs +++ b/core/startos/src/net/vhost.rs @@ -32,11 +32,14 @@ use ts_rs::TS; use crate::db::model::Database; use crate::net::acme::AcmeCertCache; +use crate::net::network_interface::{ + Accepted, NetworkInterfaceController, NetworkInterfaceListener, +}; use crate::net::static_server::server_error; use crate::prelude::*; use crate::util::io::BackTrackingIO; -use crate::util::sync::SyncMutex; use crate::util::serde::MaybeUtf8String; +use crate::util::sync::SyncMutex; #[derive(Debug)] struct SingleCertResolver(Arc); @@ -49,60 +52,69 @@ impl ResolvesServerCert for SingleCertResolver { // not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353 pub struct VHostController { - crypto_provider: Arc, db: TypedPatchDb, - servers: Mutex>, + interfaces: Arc, + crypto_provider: Arc, + servers: SyncMutex>, } impl VHostController { - pub fn new(db: TypedPatchDb) -> Self { + pub fn new(db: TypedPatchDb, interfaces: Arc) -> Self { Self { - crypto_provider: Arc::new(tokio_rustls::rustls::crypto::ring::default_provider()), db, - servers: Mutex::new(BTreeMap::new()), + interfaces, + crypto_provider: Arc::new(tokio_rustls::rustls::crypto::ring::default_provider()), + servers: SyncMutex::new(BTreeMap::new()), } } #[instrument(skip_all)] - pub async fn add( + pub fn add( &self, hostname: Option, external: u16, + public: bool, target: SocketAddr, connect_ssl: Result<(), AlpnInfo>, // Ok: yes, connect using ssl, pass through alpn; Err: connect tcp, use provided strategy for alpn ) -> Result, Error> { - let mut writable = self.servers.lock().await; - let server = if let Some(server) = writable.remove(&external) { - server - } else { - tracing::info!("Listening on {external}"); - VHostServer::new(external, self.db.clone(), self.crypto_provider.clone()).await? - }; - let rc = server - .add( + self.servers.mutate(|writable| { + let server = if let Some(server) = writable.remove(&external) { + server + } else { + VHostServer::new( + external, + self.db.clone(), + self.interfaces.clone(), + self.crypto_provider.clone(), + )? + }; + let rc = server.add( hostname, TargetInfo { + public, addr: target, connect_ssl, }, - ) - .await; - writable.insert(external, server); - Ok(rc?) + ); + writable.insert(external, server); + Ok(rc?) + }) } #[instrument(skip_all)] - pub async fn gc(&self, hostname: Option, external: u16) -> Result<(), Error> { - let mut writable = self.servers.lock().await; - if let Some(server) = writable.remove(&external) { - server.gc(hostname).await?; - if !server.is_empty().await? { - writable.insert(external, server); + pub fn gc(&self, hostname: Option, external: u16) -> Result<(), Error> { + self.servers.mutate(|writable| { + if let Some(server) = writable.remove(&external) { + server.gc(hostname)?; + if !server.is_empty()? { + writable.insert(external, server); + } } - } - Ok(()) + Ok(()) + }) } } #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] struct TargetInfo { + public: bool, addr: SocketAddr, connect_ssl: Result<(), AlpnInfo>, } @@ -120,421 +132,442 @@ impl Default for AlpnInfo { } } +type AcmeTlsAlpnCache = + Arc>>>>>; +type Mapping = SyncMutex, BTreeMap>>>; + struct VHostServer { - mapping: Weak, BTreeMap>>>>, + mapping: Weak, _thread: NonDetachingJoinHandle<()>, } + impl VHostServer { - #[instrument(skip_all)] - async fn new(port: u16, db: TypedPatchDb, crypto_provider: Arc) -> Result { - let acme_tls_alpn_cache = Arc::new(SyncMutex::new(BTreeMap::< - InternedString, - watch::Receiver>>, - >::new())); - // check if port allowed - let listener = TcpListener::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port)) - .await - .with_kind(crate::ErrorKind::Network)?; - let mapping = Arc::new(RwLock::new(BTreeMap::new())); - Ok(Self { - mapping: Arc::downgrade(&mapping), - _thread: tokio::spawn(async move { - loop { - match listener.accept().await { - Ok((stream, _)) => { - if let Err(e) = socket2::SockRef::from(&stream).set_tcp_keepalive( - &socket2::TcpKeepalive::new() - .with_time(Duration::from_secs(900)) - .with_interval(Duration::from_secs(60)) - .with_retries(5), - ) { - tracing::error!("Failed to set tcp keepalive: {e}"); - tracing::debug!("{e:?}"); - } + async fn accept( + listener: &mut NetworkInterfaceListener, + mapping: Arc, + db: TypedPatchDb, + acme_tls_alpn_cache: AcmeTlsAlpnCache, + crypto_provider: Arc, + ) -> Result<(), Error> { + let any_public = mapping.peek(|m| { + m.iter() + .any(|(_, targets)| targets.keys().any(|target| target.public)) + }); + let accepted = listener.accept(any_public).await?; + if let Err(e) = socket2::SockRef::from(&accepted.stream).set_tcp_keepalive( + &socket2::TcpKeepalive::new() + .with_time(Duration::from_secs(900)) + .with_interval(Duration::from_secs(60)) + .with_retries(5), + ) { + tracing::error!("Failed to set tcp keepalive: {e}"); + tracing::debug!("{e:?}"); + } - let mut stream = BackTrackingIO::new(stream); - let mapping = mapping.clone(); - let db = db.clone(); - let acme_tls_alpn_cache = acme_tls_alpn_cache.clone(); - let crypto_provider = crypto_provider.clone(); - tokio::spawn(async move { - if let Err(e) = async { - let mid: tokio_rustls::StartHandshake<&mut BackTrackingIO> = match LazyConfigAcceptor::new( - Acceptor::default(), - &mut stream, - ) + tokio::spawn(async move { + let bind = accepted.bind; + if let Err(e) = + Self::handle_stream(accepted, mapping, db, acme_tls_alpn_cache, crypto_provider) + .await + { + tracing::error!("Error in VHostController on {bind}: {e}"); + tracing::debug!("{e:?}") + } + }); + Ok(()) + } + + async fn handle_stream( + Accepted { + stream, + is_public, + bind, + .. + }: Accepted, + mapping: Arc, + db: TypedPatchDb, + acme_tls_alpn_cache: AcmeTlsAlpnCache, + crypto_provider: Arc, + ) -> Result<(), Error> { + let mut stream = BackTrackingIO::new(stream); + let mid: tokio_rustls::StartHandshake<&mut BackTrackingIO> = + match LazyConfigAcceptor::new(Acceptor::default(), &mut stream).await { + Ok(a) => a, + Err(_) => { + stream.rewind(); + return hyper_util::server::conn::auto::Builder::new( + hyper_util::rt::TokioExecutor::new(), + ) + .serve_connection( + hyper_util::rt::TokioIo::new(stream), + hyper_util::service::TowerToHyperService::new( + axum::Router::new().fallback(axum::routing::method_routing::any( + move |req: Request| async move { + match async move { + let host = req + .headers() + .get(http::header::HOST) + .and_then(|host| host.to_str().ok()); + let uri = Uri::from_parts({ + let mut parts = req.uri().to_owned().into_parts(); + parts.scheme = Some("https".parse()?); + parts.authority = + host.map(FromStr::from_str).transpose()?; + parts + })?; + Response::builder() + .status(http::StatusCode::TEMPORARY_REDIRECT) + .header(http::header::LOCATION, uri.to_string()) + .body(Body::default()) + } .await { Ok(a) => a, - Err(_) => { - stream.rewind(); - return hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()) - .serve_connection( - hyper_util::rt::TokioIo::new(stream), - hyper_util::service::TowerToHyperService::new(axum::Router::new().fallback( - axum::routing::method_routing::any(move |req: Request| async move { - match async move { - let host = req - .headers() - .get(http::header::HOST) - .and_then(|host| host.to_str().ok()); - let uri = Uri::from_parts({ - let mut parts = req.uri().to_owned().into_parts(); - parts.scheme = Some("https".parse()?); - parts.authority = host.map(FromStr::from_str).transpose()?; - parts - })?; - Response::builder() - .status(http::StatusCode::TEMPORARY_REDIRECT) - .header(http::header::LOCATION, uri.to_string()) - .body(Body::default()) - }.await { - Ok(a) => a, - Err(e) => { - tracing::warn!("Error redirecting http request on ssl port: {e}"); - tracing::error!("{e:?}"); - server_error(Error::new(e, ErrorKind::Network)) - } - } - }), - )), - ) - .await - .map_err(|e| Error::new(color_eyre::eyre::Report::msg(e), ErrorKind::Network)); + Err(e) => { + tracing::warn!( + "Error redirecting http request on ssl port: {e}" + ); + tracing::error!("{e:?}"); + server_error(Error::new(e, ErrorKind::Network)) } - }; - let target_name = - mid.client_hello().server_name().map(|s| s.into()); - let target = { - let mapping = mapping.read().await; - mapping - .get(&target_name) - .into_iter() - .flatten() - .find(|(_, rc)| rc.strong_count() > 0) - .or_else(|| { - if target_name - .as_ref() - .map(|s| s.parse::().is_ok()) - .unwrap_or(true) - { - mapping - .get(&None) - .into_iter() - .flatten() - .find(|(_, rc)| rc.strong_count() > 0) - } else { - None - } - }) - .map(|(target, _)| target.clone()) - }; - if let Some(target) = target { - let peek = db.peek().await; - let root = peek.as_private().as_key_store().as_local_certs().as_root_cert().de()?; - let mut cfg = match async { - if let Some(acme_settings) = peek.as_public().as_server_info().as_acme().de()? { - if let Some(domain) = target_name.as_ref().filter(|target_name| acme_settings.domains.contains(*target_name)) { - if mid - .client_hello() - .alpn() - .into_iter() - .flatten() - .any(|alpn| alpn == ACME_TLS_ALPN_NAME) - { - let cert = WatchStream::new( - acme_tls_alpn_cache.peek(|c| c.get(&**domain).cloned()) - .ok_or_else(|| { - Error::new( - eyre!("No challenge recv available for {domain}"), - ErrorKind::OpenSsl - ) - })?, - ); - tracing::info!("Waiting for verification cert for {domain}"); - let cert = cert - .filter(|c| c.is_some()) - .next() - .await - .flatten() - .ok_or_else(|| { - Error::new(eyre!("No challenge available for {domain}"), ErrorKind::OpenSsl) - })?; - tracing::info!("Verification cert received for {domain}"); - let mut cfg = ServerConfig::builder_with_provider(crypto_provider.clone()) - .with_safe_default_protocol_versions() - .with_kind(crate::ErrorKind::OpenSsl)? - .with_no_client_auth() - .with_cert_resolver(Arc::new(SingleCertResolver(cert))); - - cfg.alpn_protocols = vec![ACME_TLS_ALPN_NAME.to_vec()]; - return Ok(Err(cfg)); - } else { - let domains = [domain.to_string()]; - let (send, recv) = watch::channel(None); - acme_tls_alpn_cache.mutate(|c| c.insert(domain.clone(), recv)); - let cert = - async_acme::rustls_helper::order( - |_, cert| { - send.send_replace(Some(Arc::new(cert))); - Ok(()) - }, - acme_settings.provider.as_str(), - &domains, - Some(&AcmeCertCache(&db)), - &acme_settings.contact, - ) - .await - .with_kind(ErrorKind::OpenSsl)?; - return Ok(Ok( - ServerConfig::builder_with_provider(crypto_provider.clone()) - .with_safe_default_protocol_versions() - .with_kind(crate::ErrorKind::OpenSsl)? - .with_no_client_auth() - .with_cert_resolver(Arc::new(SingleCertResolver(Arc::new(cert)))) - )); - } - } - } - let hostnames = target_name - .into_iter() - .chain( - peek - .as_public() - .as_server_info() - .as_ip_info() - .as_entries()? - .into_iter() - .flat_map(|(_, ips)| [ - ips.as_ipv4().de().map(|ip| ip.map(IpAddr::V4)), - ips.as_ipv6().de().map(|ip| ip.map(IpAddr::V6)) - ]) - .filter_map(|a| a.transpose()) - .map(|a| a.map(|ip| InternedString::from_display(&ip))) - .collect::, _>>()?, - ) - .collect(); - let key = db - .mutate(|v| { - v.as_private_mut() - .as_key_store_mut() - .as_local_certs_mut() - .cert_for(&hostnames) - }) - .await?; - let cfg = ServerConfig::builder_with_provider(crypto_provider.clone()) - .with_safe_default_protocol_versions() - .with_kind(crate::ErrorKind::OpenSsl)? - .with_no_client_auth(); - if mid.client_hello().signature_schemes().contains( - &tokio_rustls::rustls::SignatureScheme::ED25519, - ) { - cfg.with_single_cert( - key.fullchain_ed25519() - .into_iter() - .map(|c| { - Ok(tokio_rustls::rustls::pki_types::CertificateDer::from( - c.to_der()?, - )) - }) - .collect::>()?, - PrivateKeyDer::from(PrivatePkcs8KeyDer::from( - key.leaf - .keys - .ed25519 - .private_key_to_pkcs8()?, - )), - ) - } else { - cfg.with_single_cert( - key.fullchain_nistp256() - .into_iter() - .map(|c| { - Ok(tokio_rustls::rustls::pki_types::CertificateDer::from( - c.to_der()?, - )) - }) - .collect::>()?, - PrivateKeyDer::from(PrivatePkcs8KeyDer::from( - key.leaf - .keys - .nistp256 - .private_key_to_pkcs8()?, - )), - ) - } - .with_kind(crate::ErrorKind::OpenSsl) - .map(Ok) - }.await? { - Ok(a) => a, - Err(cfg) => { - tracing::info!("performing ACME auth challenge"); - let mut accept = mid.into_stream(Arc::new(cfg)); - let io = accept.get_mut().unwrap(); - let buffered = io.stop_buffering(); - io.write_all(&buffered).await?; - accept.await?; - tracing::info!("ACME auth challenge completed"); - return Ok(()); - } - }; - let mut tcp_stream = - TcpStream::connect(target.addr).await?; - match target.connect_ssl { - Ok(()) => { - let mut client_cfg = - tokio_rustls::rustls::ClientConfig::builder_with_provider(crypto_provider) - .with_safe_default_protocol_versions() - .with_kind(crate::ErrorKind::OpenSsl)? - .with_root_certificates({ - let mut store = RootCertStore::empty(); - store.add( - CertificateDer::from( - root.to_der()?, - ), - ).with_kind(crate::ErrorKind::OpenSsl)?; - store - }) - .with_no_client_auth(); - client_cfg.alpn_protocols = mid - .client_hello() - .alpn() - .into_iter() - .flatten() - .map(|x| x.to_vec()) - .collect(); - let mut target_stream = - TlsConnector::from(Arc::new(client_cfg)) - .connect_with( - ServerName::IpAddress( - target.addr.ip().into(), - ), - tcp_stream, - |conn| { - cfg.alpn_protocols.extend( - conn.alpn_protocol() - .into_iter() - .map(|p| p.to_vec()), - ) - }, - ) - .await - .with_kind(crate::ErrorKind::OpenSsl)?; - let mut accept = mid.into_stream(Arc::new(cfg)); - let io = accept.get_mut().unwrap(); - let buffered = io.stop_buffering(); - io.write_all(&buffered).await?; - let mut tls_stream = - match accept.await { - Ok(a) => a, - Err(e) => { - tracing::trace!( "VHostController: failed to accept TLS connection on port {port}: {e}"); - tracing::trace!("{e:?}"); - return Ok(()) - } - }; - tokio::io::copy_bidirectional( - &mut tls_stream, - &mut target_stream, - ) - .await - } - Err(AlpnInfo::Reflect) => { - for proto in - mid.client_hello().alpn().into_iter().flatten() - { - cfg.alpn_protocols.push(proto.into()); - } - let mut accept = mid.into_stream(Arc::new(cfg)); - let io = accept.get_mut().unwrap(); - let buffered = io.stop_buffering(); - io.write_all(&buffered).await?; - let mut tls_stream = - match accept.await { - Ok(a) => a, - Err(e) => { - tracing::trace!( "VHostController: failed to accept TLS connection on port {port}: {e}"); - tracing::trace!("{e:?}"); - return Ok(()) - } - }; - tokio::io::copy_bidirectional( - &mut tls_stream, - &mut tcp_stream, - ) - .await - } - Err(AlpnInfo::Specified(alpn)) => { - cfg.alpn_protocols = alpn.into_iter().map(|a| a.0).collect(); - let mut accept = mid.into_stream(Arc::new(cfg)); - let io = accept.get_mut().unwrap(); - let buffered = io.stop_buffering(); - io.write_all(&buffered).await?; - let mut tls_stream = - match accept.await { - Ok(a) => a, - Err(e) => { - tracing::trace!( "VHostController: failed to accept TLS connection on port {port}: {e}"); - tracing::trace!("{e:?}"); - return Ok(()) - } - }; - tokio::io::copy_bidirectional( - &mut tls_stream, - &mut tcp_stream, - ) - .await - } - } - .map_or_else( - |e| { - use std::io::ErrorKind as E; - match e.kind() { - E::UnexpectedEof | E::BrokenPipe | E::ConnectionAborted | E::ConnectionReset | E::ConnectionRefused | E::TimedOut | E::Interrupted | E::NotConnected => Ok(()), - _ => Err(e), - }}, - |_| Ok(()), - )?; - } else { - // 503 } - Ok::<_, Error>(()) - } + }, + )), + ), + ) + .await + .map_err(|e| Error::new(color_eyre::eyre::Report::msg(e), ErrorKind::Network)); + } + }; + let target_name = mid.client_hello().server_name().map(|s| s.into()); + let target = mapping.peek(|m| { + m.get(&target_name) + .into_iter() + .flatten() + .find(|(_, rc)| rc.strong_count() > 0) + .or_else(|| { + if target_name + .as_ref() + .map(|s| s.parse::().is_ok()) + .unwrap_or(true) + { + m.get(&None) + .into_iter() + .flatten() + .find(|(_, rc)| rc.strong_count() > 0) + } else { + None + } + }) + .map(|(target, _)| target.clone()) + }); + if let Some(target) = target { + if is_public && !target.public { + log::warn!("Rejecting connection from public interface to private bind"); + return Ok(()); + } + let peek = db.peek().await; + let root = peek + .as_private() + .as_key_store() + .as_local_certs() + .as_root_cert() + .de()?; + let mut cfg = match async { + if let Some(acme_settings) = peek.as_public().as_server_info().as_acme().de()? { + if let Some(domain) = target_name + .as_ref() + .filter(|target_name| acme_settings.domains.contains(*target_name)) + { + if mid + .client_hello() + .alpn() + .into_iter() + .flatten() + .any(|alpn| alpn == ACME_TLS_ALPN_NAME) + { + let cert = WatchStream::new( + acme_tls_alpn_cache + .peek(|c| c.get(&**domain).cloned()) + .ok_or_else(|| { + Error::new( + eyre!("No challenge recv available for {domain}"), + ErrorKind::OpenSsl, + ) + })?, + ); + tracing::info!("Waiting for verification cert for {domain}"); + let cert = cert + .filter(|c| c.is_some()) + .next() .await - { - tracing::error!("Error in VHostController on port {port}: {e}"); - tracing::debug!("{e:?}") - } - }); + .flatten() + .ok_or_else(|| { + Error::new( + eyre!("No challenge available for {domain}"), + ErrorKind::OpenSsl, + ) + })?; + tracing::info!("Verification cert received for {domain}"); + let mut cfg = + ServerConfig::builder_with_provider(crypto_provider.clone()) + .with_safe_default_protocol_versions() + .with_kind(crate::ErrorKind::OpenSsl)? + .with_no_client_auth() + .with_cert_resolver(Arc::new(SingleCertResolver(cert))); + + cfg.alpn_protocols = vec![ACME_TLS_ALPN_NAME.to_vec()]; + return Ok(Err(cfg)); + } else { + let domains = [domain.to_string()]; + let (send, recv) = watch::channel(None); + acme_tls_alpn_cache.mutate(|c| c.insert(domain.clone(), recv)); + let cert = async_acme::rustls_helper::order( + |_, cert| { + send.send_replace(Some(Arc::new(cert))); + Ok(()) + }, + acme_settings.provider.as_str(), + &domains, + Some(&AcmeCertCache(&db)), + &acme_settings.contact, + ) + .await + .with_kind(ErrorKind::OpenSsl)?; + return Ok(Ok(ServerConfig::builder_with_provider( + crypto_provider.clone(), + ) + .with_safe_default_protocol_versions() + .with_kind(crate::ErrorKind::OpenSsl)? + .with_no_client_auth() + .with_cert_resolver(Arc::new(SingleCertResolver(Arc::new(cert)))))); } + } + } + let hostnames = target_name + .into_iter() + .chain([InternedString::from_display(&bind.ip())]) + .collect(); + let key = db + .mutate(|v| { + v.as_private_mut() + .as_key_store_mut() + .as_local_certs_mut() + .cert_for(&hostnames) + }) + .await?; + let cfg = ServerConfig::builder_with_provider(crypto_provider.clone()) + .with_safe_default_protocol_versions() + .with_kind(crate::ErrorKind::OpenSsl)? + .with_no_client_auth(); + if mid + .client_hello() + .signature_schemes() + .contains(&tokio_rustls::rustls::SignatureScheme::ED25519) + { + cfg.with_single_cert( + key.fullchain_ed25519() + .into_iter() + .map(|c| { + Ok(tokio_rustls::rustls::pki_types::CertificateDer::from( + c.to_der()?, + )) + }) + .collect::>()?, + PrivateKeyDer::from(PrivatePkcs8KeyDer::from( + key.leaf.keys.ed25519.private_key_to_pkcs8()?, + )), + ) + } else { + cfg.with_single_cert( + key.fullchain_nistp256() + .into_iter() + .map(|c| { + Ok(tokio_rustls::rustls::pki_types::CertificateDer::from( + c.to_der()?, + )) + }) + .collect::>()?, + PrivateKeyDer::from(PrivatePkcs8KeyDer::from( + key.leaf.keys.nistp256.private_key_to_pkcs8()?, + )), + ) + } + .with_kind(crate::ErrorKind::OpenSsl) + .map(Ok) + } + .await? + { + Ok(a) => a, + Err(cfg) => { + tracing::info!("performing ACME auth challenge"); + let mut accept = mid.into_stream(Arc::new(cfg)); + let io = accept.get_mut().unwrap(); + let buffered = io.stop_buffering(); + io.write_all(&buffered).await?; + accept.await?; + tracing::info!("ACME auth challenge completed"); + return Ok(()); + } + }; + let mut tcp_stream = TcpStream::connect(target.addr).await?; + match target.connect_ssl { + Ok(()) => { + let mut client_cfg = + tokio_rustls::rustls::ClientConfig::builder_with_provider(crypto_provider) + .with_safe_default_protocol_versions() + .with_kind(crate::ErrorKind::OpenSsl)? + .with_root_certificates({ + let mut store = RootCertStore::empty(); + store + .add(CertificateDer::from(root.to_der()?)) + .with_kind(crate::ErrorKind::OpenSsl)?; + store + }) + .with_no_client_auth(); + client_cfg.alpn_protocols = mid + .client_hello() + .alpn() + .into_iter() + .flatten() + .map(|x| x.to_vec()) + .collect(); + let mut target_stream = TlsConnector::from(Arc::new(client_cfg)) + .connect_with( + ServerName::IpAddress(target.addr.ip().into()), + tcp_stream, + |conn| { + cfg.alpn_protocols + .extend(conn.alpn_protocol().into_iter().map(|p| p.to_vec())) + }, + ) + .await + .with_kind(crate::ErrorKind::OpenSsl)?; + let mut accept = mid.into_stream(Arc::new(cfg)); + let io = accept.get_mut().unwrap(); + let buffered = io.stop_buffering(); + io.write_all(&buffered).await?; + let mut tls_stream = match accept.await { + Ok(a) => a, Err(e) => { tracing::trace!( - "VHostController: failed to accept connection on port {port}: {e}" + "VHostController: failed to accept TLS connection on {bind}: {e}" ); tracing::trace!("{e:?}"); + return Ok(()); } + }; + tokio::io::copy_bidirectional(&mut tls_stream, &mut target_stream).await + } + Err(AlpnInfo::Reflect) => { + for proto in mid.client_hello().alpn().into_iter().flatten() { + cfg.alpn_protocols.push(proto.into()); + } + let mut accept = mid.into_stream(Arc::new(cfg)); + let io = accept.get_mut().unwrap(); + let buffered = io.stop_buffering(); + io.write_all(&buffered).await?; + let mut tls_stream = match accept.await { + Ok(a) => a, + Err(e) => { + tracing::trace!( + "VHostController: failed to accept TLS connection on {bind}: {e}" + ); + tracing::trace!("{e:?}"); + return Ok(()); + } + }; + tokio::io::copy_bidirectional(&mut tls_stream, &mut tcp_stream).await + } + Err(AlpnInfo::Specified(alpn)) => { + cfg.alpn_protocols = alpn.into_iter().map(|a| a.0).collect(); + let mut accept = mid.into_stream(Arc::new(cfg)); + let io = accept.get_mut().unwrap(); + let buffered = io.stop_buffering(); + io.write_all(&buffered).await?; + let mut tls_stream = match accept.await { + Ok(a) => a, + Err(e) => { + tracing::trace!( + "VHostController: failed to accept TLS connection on {bind}: {e}" + ); + tracing::trace!("{e:?}"); + return Ok(()); + } + }; + tokio::io::copy_bidirectional(&mut tls_stream, &mut tcp_stream).await + } + } + .map_or_else( + |e| { + use std::io::ErrorKind as E; + match e.kind() { + E::UnexpectedEof + | E::BrokenPipe + | E::ConnectionAborted + | E::ConnectionReset + | E::ConnectionRefused + | E::TimedOut + | E::Interrupted + | E::NotConnected => Ok(()), + _ => Err(e), + } + }, + |_| Ok(()), + )?; + } else { + // 503 + } + Ok::<_, Error>(()) + } + + #[instrument(skip_all)] + fn new( + port: u16, + db: TypedPatchDb, + iface_ctrl: Arc, + crypto_provider: Arc, + ) -> Result { + let acme_tls_alpn_cache = Arc::new(SyncMutex::new(BTreeMap::new())); + let mut listener = iface_ctrl.bind(port).with_kind(crate::ErrorKind::Network)?; + let mapping = Arc::new(SyncMutex::new(BTreeMap::new())); + Ok(Self { + mapping: Arc::downgrade(&mapping), + _thread: tokio::spawn(async move { + loop { + if let Err(e) = Self::accept( + &mut listener, + mapping.clone(), + db.clone(), + acme_tls_alpn_cache.clone(), + crypto_provider.clone(), + ) + .await + { + tracing::trace!( + "VHostController: failed to accept connection on {port}: {e}" + ); + tracing::trace!("{e:?}"); } } }) .into(), }) } - async fn add( - &self, - hostname: Option, - target: TargetInfo, - ) -> Result, Error> { + fn add(&self, hostname: Option, target: TargetInfo) -> Result, Error> { if let Some(mapping) = Weak::upgrade(&self.mapping) { - let mut writable = mapping.write().await; - let mut targets = writable.remove(&hostname).unwrap_or_default(); - let rc = if let Some(rc) = Weak::upgrade(&targets.remove(&target).unwrap_or_default()) { - rc - } else { - Arc::new(()) - }; - targets.insert(target, Arc::downgrade(&rc)); - writable.insert(hostname, targets); - Ok(rc) + mapping.mutate(|writable| { + let mut targets = writable.remove(&hostname).unwrap_or_default(); + let rc = + if let Some(rc) = Weak::upgrade(&targets.remove(&target).unwrap_or_default()) { + rc + } else { + Arc::new(()) + }; + targets.insert(target, Arc::downgrade(&rc)); + writable.insert(hostname, targets); + Ok(rc) + }) } else { Err(Error::new( eyre!("VHost Service Thread has exited"), @@ -542,18 +575,19 @@ impl VHostServer { )) } } - async fn gc(&self, hostname: Option) -> Result<(), Error> { + fn gc(&self, hostname: Option) -> Result<(), Error> { if let Some(mapping) = Weak::upgrade(&self.mapping) { - let mut writable = mapping.write().await; - let mut targets = writable.remove(&hostname).unwrap_or_default(); - targets = targets - .into_iter() - .filter(|(_, rc)| rc.strong_count() > 0) - .collect(); - if !targets.is_empty() { - writable.insert(hostname, targets); - } - Ok(()) + mapping.mutate(|writable| { + let mut targets = writable.remove(&hostname).unwrap_or_default(); + targets = targets + .into_iter() + .filter(|(_, rc)| rc.strong_count() > 0) + .collect(); + if !targets.is_empty() { + writable.insert(hostname, targets); + } + Ok(()) + }) } else { Err(Error::new( eyre!("VHost Service Thread has exited"), @@ -561,9 +595,9 @@ impl VHostServer { )) } } - async fn is_empty(&self) -> Result { + fn is_empty(&self) -> Result { if let Some(mapping) = Weak::upgrade(&self.mapping) { - Ok(mapping.read().await.is_empty()) + Ok(mapping.peek(|m| m.is_empty())) } else { Err(Error::new( eyre!("VHost Service Thread has exited"), diff --git a/core/startos/src/service/effects/net/bind.rs b/core/startos/src/service/effects/net/bind.rs index 5619375eb..d30b45f72 100644 --- a/core/startos/src/service/effects/net/bind.rs +++ b/core/startos/src/service/effects/net/bind.rs @@ -1,6 +1,6 @@ use models::{HostId, PackageId}; -use crate::net::host::binding::{BindId, BindOptions, LanInfo}; +use crate::net::host::binding::{BindId, BindOptions, NetInfo}; use crate::net::host::HostKind; use crate::service::effects::prelude::*; @@ -58,7 +58,7 @@ pub struct GetServicePortForwardParams { pub async fn get_service_port_forward( context: EffectContext, data: GetServicePortForwardParams, -) -> Result { +) -> Result { let internal_port = data.internal_port as u16; let context = context.deref()?; diff --git a/core/startos/src/service/effects/subcontainer/mod.rs b/core/startos/src/service/effects/subcontainer/mod.rs index 65fcbd387..943c70dbf 100644 --- a/core/startos/src/service/effects/subcontainer/mod.rs +++ b/core/startos/src/service/effects/subcontainer/mod.rs @@ -4,12 +4,11 @@ use imbl_value::InternedString; use models::ImageId; use tokio::process::Command; +use crate::disk::mount::filesystem::overlayfs::OverlayGuard; use crate::rpc_continuations::Guid; use crate::service::effects::prelude::*; +use crate::service::persistent_container::Subcontainer; use crate::util::Invoke; -use crate::{ - disk::mount::filesystem::overlayfs::OverlayGuard, service::persistent_container::Subcontainer, -}; #[cfg(feature = "container-runtime")] mod sync; diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index d73c51beb..d2e293909 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -149,10 +149,10 @@ impl ServiceRef { .values() .flat_map(|h| h.bindings.values()) .flat_map(|b| { - b.lan + b.net .assigned_port .into_iter() - .chain(b.lan.assigned_ssl_port) + .chain(b.net.assigned_ssl_port) }), ); Ok(()) diff --git a/core/startos/src/util/mod.rs b/core/startos/src/util/mod.rs index 0f3563018..8a13f0d3b 100644 --- a/core/startos/src/util/mod.rs +++ b/core/startos/src/util/mod.rs @@ -122,7 +122,8 @@ impl<'a> std::ops::DerefMut for ExtendedCommand<'a> { } impl<'a> Invoke<'a> for tokio::process::Command { - type Extended<'ext> = ExtendedCommand<'ext> + type Extended<'ext> + = ExtendedCommand<'ext> where Self: 'ext, 'ext: 'a; @@ -162,7 +163,8 @@ impl<'a> Invoke<'a> for tokio::process::Command { } impl<'a> Invoke<'a> for ExtendedCommand<'a> { - type Extended<'ext> = &'ext mut ExtendedCommand<'ext> + type Extended<'ext> + = &'ext mut ExtendedCommand<'ext> where Self: 'ext, 'ext: 'a; From e950c5ecdc4b16a5d74e782a0e1f7ba8d9f78f72 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 25 Nov 2024 12:20:43 -0700 Subject: [PATCH 03/29] wip --- core/startos/src/net/network_interface.rs | 10 +++++++++- sdk/base/lib/osBindings/BindInfo.ts | 4 ++-- sdk/base/lib/osBindings/{LanInfo.ts => NetInfo.ts} | 3 ++- sdk/base/lib/osBindings/NetworkInterfaceInfo.ts | 4 ++++ sdk/base/lib/osBindings/ServerInfo.ts | 4 ++-- sdk/base/lib/osBindings/index.ts | 3 ++- 6 files changed, 21 insertions(+), 7 deletions(-) rename sdk/base/lib/osBindings/{LanInfo.ts => NetInfo.ts} (80%) create mode 100644 sdk/base/lib/osBindings/NetworkInterfaceInfo.ts diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index c657b83d1..39fb42f15 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -9,6 +9,7 @@ use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; use tokio::sync::RwLock; +use tokio_stream::StreamExt; use ts_rs::TS; use zbus::zvariant::OwnedObjectPath; use zbus::{proxy, Connection}; @@ -27,6 +28,8 @@ use crate::util::sync::SyncMutex; )] trait NetworkManager { async fn get_all_devices(&self) -> Result, Error>; + #[zbus(property)] + fn active_connections(&self) -> Result, Error>; } #[tokio::test] @@ -34,9 +37,14 @@ async fn test() -> Result<(), Error> { let connection = Connection::system().await?; let proxy = NetworkManagerProxy::new(&connection).await?; - let reply = proxy.get_all_devices().await?; + let reply = proxy.active_connections().await?; println!("{reply:?}"); + let mut stream = proxy.receive_active_connections_changed().await; + while let Some(conn) = stream.next().await { + println!("{:?}", conn.get().await?); + } + Ok(()) } diff --git a/sdk/base/lib/osBindings/BindInfo.ts b/sdk/base/lib/osBindings/BindInfo.ts index 85fc38e94..b03dbe6b2 100644 --- a/sdk/base/lib/osBindings/BindInfo.ts +++ b/sdk/base/lib/osBindings/BindInfo.ts @@ -1,5 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { BindOptions } from "./BindOptions" -import type { LanInfo } from "./LanInfo" +import type { NetInfo } from "./NetInfo" -export type BindInfo = { enabled: boolean; options: BindOptions; lan: LanInfo } +export type BindInfo = { enabled: boolean; options: BindOptions; net: NetInfo } diff --git a/sdk/base/lib/osBindings/LanInfo.ts b/sdk/base/lib/osBindings/NetInfo.ts similarity index 80% rename from sdk/base/lib/osBindings/LanInfo.ts rename to sdk/base/lib/osBindings/NetInfo.ts index 59b8a5519..e790cadaa 100644 --- a/sdk/base/lib/osBindings/LanInfo.ts +++ b/sdk/base/lib/osBindings/NetInfo.ts @@ -1,6 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type LanInfo = { +export type NetInfo = { + public: boolean assignedPort: number | null assignedSslPort: number | null } diff --git a/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts new file mode 100644 index 000000000..f389786aa --- /dev/null +++ b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { IpInfo } from "./IpInfo" + +export type NetworkInterfaceInfo = { public: boolean; ipInfo: IpInfo } diff --git a/sdk/base/lib/osBindings/ServerInfo.ts b/sdk/base/lib/osBindings/ServerInfo.ts index 89d7fc1b0..b5d5be293 100644 --- a/sdk/base/lib/osBindings/ServerInfo.ts +++ b/sdk/base/lib/osBindings/ServerInfo.ts @@ -1,8 +1,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AcmeSettings } from "./AcmeSettings" import type { Governor } from "./Governor" -import type { IpInfo } from "./IpInfo" import type { LshwDevice } from "./LshwDevice" +import type { NetworkInterfaceInfo } from "./NetworkInterfaceInfo" import type { ServerStatus } from "./ServerStatus" import type { SmtpValue } from "./SmtpValue" import type { WifiInfo } from "./WifiInfo" @@ -22,7 +22,7 @@ export type ServerInfo = { * for backwards compatibility */ torAddress: string - ipInfo: { [key: string]: IpInfo } + networkInterfaces: { [key: string]: NetworkInterfaceInfo } acme: AcmeSettings | null statusInfo: ServerStatus wifi: WifiInfo diff --git a/sdk/base/lib/osBindings/index.ts b/sdk/base/lib/osBindings/index.ts index f76f595c9..5f0506e39 100644 --- a/sdk/base/lib/osBindings/index.ts +++ b/sdk/base/lib/osBindings/index.ts @@ -112,7 +112,6 @@ export { InstallingState } from "./InstallingState" export { InstallParams } from "./InstallParams" export { IpHostname } from "./IpHostname" export { IpInfo } from "./IpInfo" -export { LanInfo } from "./LanInfo" export { ListPackageSignersParams } from "./ListPackageSignersParams" export { ListServiceInterfacesParams } from "./ListServiceInterfacesParams" export { ListVersionSignersParams } from "./ListVersionSignersParams" @@ -128,6 +127,8 @@ export { MountParams } from "./MountParams" export { MountTarget } from "./MountTarget" export { NamedHealthCheckResult } from "./NamedHealthCheckResult" export { NamedProgress } from "./NamedProgress" +export { NetInfo } from "./NetInfo" +export { NetworkInterfaceInfo } from "./NetworkInterfaceInfo" export { OnionHostname } from "./OnionHostname" export { OsIndex } from "./OsIndex" export { OsVersionInfoMap } from "./OsVersionInfoMap" From 9735a3221a0593f8839537b67e15aa04f7f58c70 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 26 Nov 2024 13:52:18 -0700 Subject: [PATCH 04/29] wip --- core/startos/src/net/network_interface.rs | 134 ++++++++++++++++++++-- 1 file changed, 124 insertions(+), 10 deletions(-) diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 39fb42f15..531b1b09f 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -1,17 +1,25 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::future::Future; use std::net::{IpAddr, SocketAddr}; use std::sync::{Arc, Weak}; +use std::task::Poll; use clap::Parser; -use futures::TryStreamExt; +use futures::future::BoxFuture; +use futures::{FutureExt, TryStreamExt}; use imbl_value::InternedString; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; -use tokio::sync::RwLock; +use tokio::sync::{watch, RwLock}; use tokio_stream::StreamExt; use ts_rs::TS; -use zbus::zvariant::OwnedObjectPath; +use zbus::fdo::PropertiesChangedStream; +use zbus::proxy::PropertyStream; +use zbus::zvariant::{ + DeserializeDict, DynamicDeserialize, OwnedObjectPath, OwnedValue, SerializeDict, Type as ZType, + Value as ZValue, +}; use zbus::{proxy, Connection}; use crate::context::{CliContext, RpcContext}; @@ -27,27 +35,133 @@ use crate::util::sync::SyncMutex; default_path = "/org/freedesktop/NetworkManager" )] trait NetworkManager { - async fn get_all_devices(&self) -> Result, Error>; #[zbus(property)] fn active_connections(&self) -> Result, Error>; } +#[proxy( + interface = "org.freedesktop.NetworkManager.Connection.Active", + default_service = "org.freedesktop.NetworkManager" +)] +trait ActiveConnection { + #[zbus(property)] + fn ip4_config(&self) -> Result; + #[zbus(property)] + fn devices(&self) -> Result, Error>; +} + +#[proxy( + interface = "org.freedesktop.NetworkManager.IP4Config", + default_service = "org.freedesktop.NetworkManager" +)] +trait Ip4Config { + #[zbus(property)] + fn address_data(&self) -> Result, Error>; +} + +#[derive(Debug, DeserializeDict, ZValue, ZType)] +#[zvariant(signature = "dict")] +struct AddressData { + address: String, + prefix: u32, +} + +#[proxy( + interface = "org.freedesktop.NetworkManager.Device", + default_service = "org.freedesktop.NetworkManager" +)] +trait Device { + #[zbus(property)] + fn ip_interface(&self) -> Result; +} + #[tokio::test] async fn test() -> Result<(), Error> { let connection = Connection::system().await?; let proxy = NetworkManagerProxy::new(&connection).await?; - let reply = proxy.active_connections().await?; - println!("{reply:?}"); - - let mut stream = proxy.receive_active_connections_changed().await; - while let Some(conn) = stream.next().await { - println!("{:?}", conn.get().await?); + let active = proxy + .receive_active_connections_changed() + .await + .next() + .await + .unwrap() + .get() + .await?; + eprintln!("{active:?}"); + for active in active { + let proxy = ActiveConnectionProxy::new(&connection, active).await?; + let ip4 = proxy.ip4_config().await?; + eprintln!("{ip4:?}"); + let ip_proxy = Ip4ConfigProxy::new(&connection, ip4).await?; + let addresses = ip_proxy.address_data().await?; + eprintln!("{addresses:?}"); + let devices = proxy.devices().await?; + eprintln!("{devices:?}"); + for device in devices { + let proxy = DeviceProxy::new(&connection, device).await?; + let ifaces = proxy.ip_interface().await?; + eprintln!("{ifaces:?}"); + } } Ok(()) } +struct WatchPropertyStream<'a, T> { + stream: PropertyStream<'a, T>, + last: T, +} +impl<'a, T> WatchPropertyStream<'a, T> +where + T: Unpin + TryFrom, + T::Error: Into, +{ + async fn new(mut stream: PropertyStream<'a, T>) -> Result { + let last = stream + .next() + .await + .ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))? + .get() + .await?; + Ok(Self { stream, last }) + } + async fn unless_changed>>( + &mut self, + fut: Fut, + ) -> Result, Error> { + tokio::select! { + changed = self.stream.next() => { + self.last = changed.ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))?.get().await?; + Ok(None) + }, + res = fut => { + res.map(Some) + } + } + } +} + +async fn watcher(write_to: watch::Sender>) { + loop { + if let Err(e) = async { + let connection = Connection::system().await?; + let netman_proxy = NetworkManagerProxy::new(&connection).await?; + + let mut active_sub = netman_proxy.receive_active_connections_changed().await; + + 'conn: while let Some(active) = active_sub.next().await {} + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("{e}"); + tracing::debug!("{e:?}"); + } + } +} + pub struct NetworkInterfaceController { db: TypedPatchDb, listeners: SyncMutex>>, From 1079ca2c4db1a0f1cc3c1821333429423b9c38e1 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 26 Nov 2024 14:49:48 -0700 Subject: [PATCH 05/29] wip --- core/startos/src/db/model/public.rs | 2 +- core/startos/src/net/network_interface.rs | 109 +++++++++++++++++++--- 2 files changed, 99 insertions(+), 12 deletions(-) diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 380d88ac5..214ee25a8 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -161,7 +161,7 @@ pub struct NetworkInterfaceInfo { pub ip_info: IpInfo, } -#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel, TS)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] #[model = "Model"] #[ts(export)] diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 531b1b09f..3ec681be4 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -5,8 +5,8 @@ use std::sync::{Arc, Weak}; use std::task::Poll; use clap::Parser; -use futures::future::BoxFuture; -use futures::{FutureExt, TryStreamExt}; +use futures::future::{pending, BoxFuture}; +use futures::{FutureExt, TryFutureExt, TryStreamExt}; use imbl_value::InternedString; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; @@ -27,6 +27,7 @@ use crate::db::model::public::IpInfo; use crate::db::model::Database; use crate::net::utils::{iface_is_physical, list_interfaces}; use crate::prelude::*; +use crate::util::actor::background::{BackgroundJobQueue, BackgroundJobRunner}; use crate::util::sync::SyncMutex; #[proxy( @@ -59,12 +60,15 @@ trait Ip4Config { fn address_data(&self) -> Result, Error>; } -#[derive(Debug, DeserializeDict, ZValue, ZType)] +#[derive(Clone, Debug, DeserializeDict, ZValue, ZType)] #[zvariant(signature = "dict")] struct AddressData { address: String, prefix: u32, } +impl TryFrom> for IpInfo { + fn try_from(value: Vec) -> Result {} +} #[proxy( interface = "org.freedesktop.NetworkManager.Device", @@ -126,31 +130,113 @@ where .await?; Ok(Self { stream, last }) } - async fn unless_changed>>( + async fn until_changed>>( &mut self, fut: Fut, - ) -> Result, Error> { + ) -> Result<(), Error> { + let mut next = self.stream.next(); tokio::select! { - changed = self.stream.next() => { + changed = next => { self.last = changed.ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))?.get().await?; - Ok(None) + Ok(()) }, - res = fut => { - res.map(Some) + res = fut.and_then(|_| pending()) => { + res } } } } async fn watcher(write_to: watch::Sender>) { + let (q, run) = BackgroundJobQueue::new(); loop { if let Err(e) = async { + let mut jobs = BackgroundJobQueue::new(); let connection = Connection::system().await?; let netman_proxy = NetworkManagerProxy::new(&connection).await?; - let mut active_sub = netman_proxy.receive_active_connections_changed().await; + let mut active_sub = + WatchPropertyStream::new(netman_proxy.receive_active_connections_changed().await) + .await?; + + loop { + let active = active_sub.last.clone(); + active_sub + .until_changed(async { + let mut ifaces = BTreeSet::new(); + let mut jobs = Vec::new(); + for active in active { + let ac_proxy = ActiveConnectionProxy::new(&connection, active).await?; + let mut devices = ac_proxy.devices().await?; + if devices.len() == 1 { + let dev_proxy = + DeviceProxy::new(&connection, devices.swap_remove(0)).await?; + let iface = InternedString::intern(dev_proxy.ip_interface().await?); + ifaces.insert(iface.clone()); + jobs.push(async { + let ac_proxy = ac_proxy; + let mut ip_config_sub = WatchPropertyStream::new( + ac_proxy.receive_ip4_config_changed().await, + ) + .await?; + + loop { + let ip_config = ip_config_sub.last.clone(); + ip_config_sub + .until_changed(async { + let ip_proxy = + Ip4ConfigProxy::new(&connection, ip_config) + .await?; + let mut address_sub = WatchPropertyStream::new( + ip_proxy.receive_address_data_changed().await, + ) + .await?; + + loop { + let addresses = address_sub.last.clone(); + address_sub + .until_changed(async { + let ip_info: IpInfo = + addresses.try_into()?; + + write_to.send_if_modified(|m| { + m.insert(iface, ip_info.clone()) + .filter(|old| old == &ip_info) + .is_none() + }); - 'conn: while let Some(active) = active_sub.next().await {} + Ok::<_, Error>(()) + }) + .await?; + } + + Ok::<_, Error>(()) + }) + .await?; + } + + Ok::<_, Error>(()) + }); + } + } + write_to.send_if_modified(|m| { + let mut changed = false; + m.retain(|i, _| { + if ifaces.contains(i) { + true + } else { + changed |= true; + false + } + }); + changed + }); + futures::future::try_join_all(jobs).await?; + + Ok::<_, Error>(()) + }) + .await?; + } Ok::<_, Error>(()) } @@ -160,6 +246,7 @@ async fn watcher(write_to: watch::Sender>) { tracing::debug!("{e:?}"); } } + run.await; } pub struct NetworkInterfaceController { From fb768773612d76ef90f2aba9668878d24f9d590a Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 27 Nov 2024 11:31:07 -0700 Subject: [PATCH 06/29] wip --- core/startos/src/db/model/public.rs | 28 ++------- core/startos/src/net/net_controller.rs | 47 ++++++++------- core/startos/src/net/network_interface.rs | 69 ++++++++--------------- 3 files changed, 55 insertions(+), 89 deletions(-) diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 214ee25a8..68be84cf5 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -4,7 +4,7 @@ use std::net::{Ipv4Addr, Ipv6Addr}; use chrono::{DateTime, Utc}; use exver::{Version, VersionRange}; use imbl_value::InternedString; -use ipnet::{Ipv4Net, Ipv6Net}; +use ipnet::{IpNet, Ipv4Net, Ipv6Net}; use isocountry::CountryCode; use itertools::Itertools; use models::PackageId; @@ -161,30 +161,10 @@ pub struct NetworkInterfaceInfo { pub ip_info: IpInfo, } -#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, HasModel, TS)] -#[serde(rename_all = "camelCase")] -#[model = "Model"] +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, TS)] #[ts(export)] -pub struct IpInfo { - #[ts(type = "string | null")] - pub ipv4_range: Option, - pub ipv4: Option, - #[ts(type = "string | null")] - pub ipv6_range: Option, - pub ipv6: Option, -} -impl IpInfo { - pub async fn for_interface(iface: &str) -> Result { - let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip(); - let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip(); - Ok(Self { - ipv4_range, - ipv4, - ipv6_range, - ipv6, - }) - } -} +#[ts(type = "string[]")] +pub struct IpInfo(pub BTreeSet); #[derive(Debug, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 63b909a79..fadaddfc6 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -5,6 +5,7 @@ use std::sync::{Arc, Weak}; use color_eyre::eyre::eyre; use imbl::OrdMap; use imbl_value::InternedString; +use ipnet::IpNet; use models::{HostId, OptionExt, PackageId}; use torut::onion::{OnionAddressV3, TorSecretKeyV3}; use tracing::instrument; @@ -424,27 +425,31 @@ impl NetService { } } if !iface_info.public || new_lan_bind.0.public { - if let Some(ipv4) = iface_info.ip_info.ipv4 { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public: iface_info.public, - hostname: IpHostname::Ipv4 { - value: ipv4, - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, - }, - }); - } - if let Some(ipv6) = iface_info.ip_info.ipv6 { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public: iface_info.public, - hostname: IpHostname::Ipv6 { - value: ipv6, - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, - }, - }); + for ipnet in &iface_info.ip_info.0 { + match ipnet { + IpNet::V4(net) => { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: iface_info.public, + hostname: IpHostname::Ipv4 { + value: net.addr(), + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } + IpNet::V6(net) => { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: iface_info.public, + hostname: IpHostname::Ipv6 { + value: net.addr(), + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } + } } } } diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 3ec681be4..f3e123345 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -5,29 +5,24 @@ use std::sync::{Arc, Weak}; use std::task::Poll; use clap::Parser; -use futures::future::{pending, BoxFuture}; +use futures::future::pending; use futures::{FutureExt, TryFutureExt, TryStreamExt}; use imbl_value::InternedString; -use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; +use ipnet::IpNet; use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; -use tokio::sync::{watch, RwLock}; +use tokio::sync::watch; use tokio_stream::StreamExt; -use ts_rs::TS; -use zbus::fdo::PropertiesChangedStream; use zbus::proxy::PropertyStream; use zbus::zvariant::{ - DeserializeDict, DynamicDeserialize, OwnedObjectPath, OwnedValue, SerializeDict, Type as ZType, - Value as ZValue, + DeserializeDict, OwnedObjectPath, OwnedValue, Type as ZType, Value as ZValue, }; use zbus::{proxy, Connection}; -use crate::context::{CliContext, RpcContext}; use crate::db::model::public::IpInfo; use crate::db::model::Database; -use crate::net::utils::{iface_is_physical, list_interfaces}; use crate::prelude::*; -use crate::util::actor::background::{BackgroundJobQueue, BackgroundJobRunner}; +use crate::util::actor::background::BackgroundJobQueue; use crate::util::sync::SyncMutex; #[proxy( @@ -67,7 +62,16 @@ struct AddressData { prefix: u32, } impl TryFrom> for IpInfo { - fn try_from(value: Vec) -> Result {} + type Error = Error; + fn try_from(value: Vec) -> Result { + value + .into_iter() + .map(|a| { + IpNet::new(a.address.parse()?, a.prefix as u8).with_kind(ErrorKind::ParseNetAddress) + }) + .collect::>() + .map(Self) + } } #[proxy( @@ -81,32 +85,11 @@ trait Device { #[tokio::test] async fn test() -> Result<(), Error> { - let connection = Connection::system().await?; - - let proxy = NetworkManagerProxy::new(&connection).await?; - let active = proxy - .receive_active_connections_changed() - .await - .next() - .await - .unwrap() - .get() - .await?; - eprintln!("{active:?}"); - for active in active { - let proxy = ActiveConnectionProxy::new(&connection, active).await?; - let ip4 = proxy.ip4_config().await?; - eprintln!("{ip4:?}"); - let ip_proxy = Ip4ConfigProxy::new(&connection, ip4).await?; - let addresses = ip_proxy.address_data().await?; - eprintln!("{addresses:?}"); - let devices = proxy.devices().await?; - eprintln!("{devices:?}"); - for device in devices { - let proxy = DeviceProxy::new(&connection, device).await?; - let ifaces = proxy.ip_interface().await?; - eprintln!("{ifaces:?}"); - } + let (write_to, mut read_from) = watch::channel(BTreeMap::new()); + tokio::task::spawn(watcher(write_to)); + loop { + eprintln!("{:?}", &*read_from.borrow()); + read_from.changed().await; } Ok(()) @@ -134,7 +117,7 @@ where &mut self, fut: Fut, ) -> Result<(), Error> { - let mut next = self.stream.next(); + let next = self.stream.next(); tokio::select! { changed = next => { self.last = changed.ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))?.get().await?; @@ -148,10 +131,8 @@ where } async fn watcher(write_to: watch::Sender>) { - let (q, run) = BackgroundJobQueue::new(); loop { if let Err(e) = async { - let mut jobs = BackgroundJobQueue::new(); let connection = Connection::system().await?; let netman_proxy = NetworkManagerProxy::new(&connection).await?; @@ -175,6 +156,7 @@ async fn watcher(write_to: watch::Sender>) { ifaces.insert(iface.clone()); jobs.push(async { let ac_proxy = ac_proxy; + let iface = iface; let mut ip_config_sub = WatchPropertyStream::new( ac_proxy.receive_ip4_config_changed().await, ) @@ -200,7 +182,7 @@ async fn watcher(write_to: watch::Sender>) { addresses.try_into()?; write_to.send_if_modified(|m| { - m.insert(iface, ip_info.clone()) + m.insert(iface.clone(), ip_info.clone()) .filter(|old| old == &ip_info) .is_none() }); @@ -209,14 +191,14 @@ async fn watcher(write_to: watch::Sender>) { }) .await?; } - - Ok::<_, Error>(()) }) .await?; } Ok::<_, Error>(()) }); + } else { + tracing::warn!("devices.len ({}) is not exactly 1. We're not sure what this means, but it shouldn't happen", devices.len()); } } write_to.send_if_modified(|m| { @@ -246,7 +228,6 @@ async fn watcher(write_to: watch::Sender>) { tracing::debug!("{e:?}"); } } - run.await; } pub struct NetworkInterfaceController { From 8af5f6d5915af2265140b36d472486bd4a85b9cd Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 27 Nov 2024 13:57:00 -0700 Subject: [PATCH 07/29] implement some things --- core/startos/src/db/model/public.rs | 2 +- core/startos/src/net/network_interface.rs | 210 ++++++++++++++++++---- 2 files changed, 180 insertions(+), 32 deletions(-) diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 68be84cf5..410dfa6d1 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -152,7 +152,7 @@ pub struct ServerInfo { pub devices: Vec, } -#[derive(Debug, Deserialize, Serialize, HasModel, TS)] +#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] #[model = "Model"] #[ts(export)] diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index f3e123345..849f9d6d8 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -1,16 +1,20 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::net::{IpAddr, SocketAddr}; +use std::ops::Deref; +use std::pin::Pin; use std::sync::{Arc, Weak}; use std::task::Poll; use clap::Parser; use futures::future::pending; use futures::{FutureExt, TryFutureExt, TryStreamExt}; +use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use ipnet::IpNet; use serde::{Deserialize, Serialize}; -use tokio::net::TcpStream; +use serde_json::de; +use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; use tokio_stream::StreamExt; use zbus::proxy::PropertyStream; @@ -19,10 +23,11 @@ use zbus::zvariant::{ }; use zbus::{proxy, Connection}; -use crate::db::model::public::IpInfo; +use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; use crate::db::model::Database; use crate::prelude::*; use crate::util::actor::background::BackgroundJobQueue; +use crate::util::logger::EmbassyLogger; use crate::util::sync::SyncMutex; #[proxy( @@ -44,6 +49,8 @@ trait ActiveConnection { fn ip4_config(&self) -> Result; #[zbus(property)] fn devices(&self) -> Result, Error>; + #[zbus(property, name = "Type")] + fn connection_type(&self) -> Result; } #[proxy( @@ -85,6 +92,7 @@ trait Device { #[tokio::test] async fn test() -> Result<(), Error> { + EmbassyLogger::init(); let (write_to, mut read_from) = watch::channel(BTreeMap::new()); tokio::task::spawn(watcher(write_to)); loop { @@ -104,14 +112,11 @@ where T: Unpin + TryFrom, T::Error: Into, { - async fn new(mut stream: PropertyStream<'a, T>) -> Result { - let last = stream - .next() - .await - .ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))? - .get() - .await?; - Ok(Self { stream, last }) + fn new(stream: PropertyStream<'a, T>, first: T) -> Self { + Self { + stream, + last: first, + } } async fn until_changed>>( &mut self, @@ -130,15 +135,16 @@ where } } -async fn watcher(write_to: watch::Sender>) { +async fn watcher(write_to: watch::Sender>) { loop { if let Err(e) = async { let connection = Connection::system().await?; let netman_proxy = NetworkManagerProxy::new(&connection).await?; - let mut active_sub = - WatchPropertyStream::new(netman_proxy.receive_active_connections_changed().await) - .await?; + let mut active_sub = WatchPropertyStream::new( + netman_proxy.receive_active_connections_changed().await, + netman_proxy.active_connections().await?, + ); loop { let active = active_sub.last.clone(); @@ -148,19 +154,23 @@ async fn watcher(write_to: watch::Sender>) { let mut jobs = Vec::new(); for active in active { let ac_proxy = ActiveConnectionProxy::new(&connection, active).await?; - let mut devices = ac_proxy.devices().await?; - if devices.len() == 1 { - let dev_proxy = - DeviceProxy::new(&connection, devices.swap_remove(0)).await?; + // dbg!(ac_proxy.connection_type().await?); + let devices = ac_proxy.devices().await?; + for device in devices { + let dev_proxy = DeviceProxy::new(&connection, device).await?; let iface = InternedString::intern(dev_proxy.ip_interface().await?); + if iface.is_empty() { + continue; + } ifaces.insert(iface.clone()); + let ac_proxy = ac_proxy.clone(); jobs.push(async { let ac_proxy = ac_proxy; let iface = iface; let mut ip_config_sub = WatchPropertyStream::new( ac_proxy.receive_ip4_config_changed().await, - ) - .await?; + ac_proxy.ip4_config().await?, + ); loop { let ip_config = ip_config_sub.last.clone(); @@ -171,8 +181,8 @@ async fn watcher(write_to: watch::Sender>) { .await?; let mut address_sub = WatchPropertyStream::new( ip_proxy.receive_address_data_changed().await, - ) - .await?; + ip_proxy.address_data().await?, + ); loop { let addresses = address_sub.last.clone(); @@ -182,9 +192,20 @@ async fn watcher(write_to: watch::Sender>) { addresses.try_into()?; write_to.send_if_modified(|m| { - m.insert(iface.clone(), ip_info.clone()) - .filter(|old| old == &ip_info) - .is_none() + let public = m + .get(&iface) + .map_or(false, |i| i.public); + m.insert( + iface.clone(), + NetworkInterfaceInfo { + public, + ip_info: ip_info.clone(), + }, + ) + .filter(|old| { + &old.ip_info == &ip_info + }) + .is_none() }); Ok::<_, Error>(()) @@ -197,8 +218,6 @@ async fn watcher(write_to: watch::Sender>) { Ok::<_, Error>(()) }); - } else { - tracing::warn!("devices.len ({}) is not exactly 1. We're not sure what this means, but it shouldn't happen", devices.len()); } } write_to.send_if_modified(|m| { @@ -232,27 +251,156 @@ async fn watcher(write_to: watch::Sender>) { pub struct NetworkInterfaceController { db: TypedPatchDb, + ip_info: watch::Sender>, + watcher: NonDetachingJoinHandle<()>, listeners: SyncMutex>>, } impl NetworkInterfaceController { + async fn sync( + db: &TypedPatchDb, + ip_info: &BTreeMap, + ) -> Result<(), Error> { + db.mutate(|db| { + let ifaces_model = db + .as_public_mut() + .as_server_info_mut() + .as_network_interfaces_mut(); + for (iface, ip_info) in ip_info { + ifaces_model + .upsert(&iface, || Ok(NetworkInterfaceInfo::default()))? + .ser(&ip_info)?; + } + Ok(()) + }) + .await?; + Ok(()) + } pub fn new(db: TypedPatchDb) -> Self { + let (write_to, mut read_from) = watch::channel(BTreeMap::new()); Self { - db, + db: db.clone(), + ip_info: write_to.clone(), + watcher: tokio::spawn(async move { + tokio::join!(watcher(write_to), async { + loop { + if let Err(e) = async { + let ip_info = read_from.borrow().clone(); + Self::sync(&db, &ip_info).await?; + + read_from.changed().await; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error syncing ip info to db: {e}"); + tracing::debug!("{e:?}"); + } + } + }); + }) + .into(), listeners: SyncMutex::new(BTreeMap::new()), } } pub fn bind(&self, port: u16) -> Result { - todo!() + let arc = Arc::new(()); + self.listeners.mutate(|l| { + if l.get(&port).filter(|w| w.strong_count() > 0).is_some() { + return Err(Error::new( + std::io::Error::from_raw_os_error(libc::EADDRINUSE), + ErrorKind::Network, + )); + } + l.insert(port, Arc::downgrade(&arc)); + Ok(()) + })?; + Ok(NetworkInterfaceListener { + arc, + ip_info: self.ip_info.subscribe(), + listeners: ListenerMap::new(), + port, + }) + } +} + +struct ListenerMap(BTreeMap); +impl ListenerMap { + fn new() -> Self { + Self(BTreeMap::new()) + } + async fn update( + &mut self, + ip_info: &BTreeMap, + port: u16, + public: bool, + ) -> Result<(), Error> { + let mut keep = BTreeSet::::new(); + for info in ip_info.values() { + if public || !info.public { + for ipnet in &info.ip_info.0 { + if let Some((_, is_public)) = self.0.get_mut(&ipnet.addr()) { + *is_public = info.public; + continue; + } + self.0.insert( + ipnet.addr(), + ( + TcpListener::bind(SocketAddr::new(ipnet.addr(), port)).await?, + info.public, + ), + ); + keep.insert(ipnet.addr()); + } + } + } + self.0.retain(|ip, _| keep.contains(ip)); + Ok(()) + } + fn accept(&mut self) -> ListenerMapFut { + ListenerMapFut(&mut self.0) + } +} +#[pin_project::pin_project] +struct ListenerMapFut<'a>(&'a mut BTreeMap); +impl<'a> Future for ListenerMapFut<'a> { + type Output = Result<(IpAddr, bool, TcpStream, SocketAddr), Error>; + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + let this = self.project(); + for (ip, listener) in this.0.iter() { + if let Poll::Ready((stream, addr)) = listener.0.poll_accept(cx)? { + return Poll::Ready(Ok((*ip, listener.1, stream, addr))); + } + } + Poll::Pending } } pub struct NetworkInterfaceListener { - ctrl: Arc, + ip_info: watch::Receiver>, + listeners: ListenerMap, + port: u16, + arc: Arc<()>, } impl NetworkInterfaceListener { pub async fn accept(&mut self, public: bool) -> Result { - todo!() + loop { + let ip_info = self.ip_info.borrow().clone(); + self.listeners.update(&ip_info, self.port, public).await?; + tokio::select! { + accepted = self.listeners.accept() => { + let (ip, is_public, stream, peer) = accepted?; + return Ok(Accepted { + stream, + peer, + is_public, + bind: (ip, self.port).into(), + }) + }, + _ = self.ip_info.changed() => {} + } + } } } From def669966314edf519b6769593b14b16e6945fa3 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 28 Nov 2024 22:00:18 -0700 Subject: [PATCH 08/29] fix warning --- core/startos/src/net/network_interface.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 849f9d6d8..ae139e7a9 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -317,7 +317,7 @@ impl NetworkInterfaceController { Ok(()) })?; Ok(NetworkInterfaceListener { - arc, + _arc: arc, ip_info: self.ip_info.subscribe(), listeners: ListenerMap::new(), port, @@ -381,7 +381,7 @@ pub struct NetworkInterfaceListener { ip_info: watch::Receiver>, listeners: ListenerMap, port: u16, - arc: Arc<()>, + _arc: Arc<()>, } impl NetworkInterfaceListener { pub async fn accept(&mut self, public: bool) -> Result { From 821460ecfb751d4c2b4e02704cc0aa577a850df5 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 28 Nov 2024 23:24:29 -0700 Subject: [PATCH 09/29] wip --- core/startos/src/db/model/public.rs | 1 + core/startos/src/registry/package/index.rs | 1 - core/startos/src/s9pk/git_hash.rs | 29 ++++++++++++++++- core/startos/src/s9pk/v2/manifest.rs | 4 +-- sdk/base/lib/Effects.ts | 5 ++- sdk/base/lib/osBindings/GitHash.ts | 3 ++ sdk/base/lib/osBindings/IpInfo.ts | 7 +---- sdk/base/lib/osBindings/Manifest.ts | 3 +- sdk/base/lib/osBindings/PackageVersionInfo.ts | 3 +- sdk/base/lib/osBindings/index.ts | 1 + sdk/package/lib/manifest/setupManifest.ts | 11 ------- .../server-specs/server-specs.page.html | 31 +++++++------------ .../ui/src/app/services/api/mock-patch.ts | 17 +++++----- 13 files changed, 62 insertions(+), 54 deletions(-) create mode 100644 sdk/base/lib/osBindings/GitHash.ts diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 410dfa6d1..2b4bfe945 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -131,6 +131,7 @@ pub struct ServerInfo { #[ts(type = "string")] pub tor_address: Url, #[ts(as = "BTreeMap::")] + #[serde(default)] pub network_interfaces: BTreeMap, pub acme: Option, #[serde(default)] diff --git a/core/startos/src/registry/package/index.rs b/core/startos/src/registry/package/index.rs index 428200165..9973bae7e 100644 --- a/core/startos/src/registry/package/index.rs +++ b/core/startos/src/registry/package/index.rs @@ -72,7 +72,6 @@ pub struct PackageVersionInfo { pub icon: DataUrl<'static>, pub description: Description, pub release_notes: String, - #[ts(type = "string")] pub git_hash: GitHash, #[ts(type = "string")] pub license: InternedString, diff --git a/core/startos/src/s9pk/git_hash.rs b/core/startos/src/s9pk/git_hash.rs index 02f83bf4a..762ef8704 100644 --- a/core/startos/src/s9pk/git_hash.rs +++ b/core/startos/src/s9pk/git_hash.rs @@ -1,11 +1,13 @@ use std::path::Path; use tokio::process::Command; +use ts_rs::TS; use crate::prelude::*; use crate::util::Invoke; -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, TS)] +#[ts(type = "string")] pub struct GitHash(String); impl GitHash { @@ -31,6 +33,31 @@ impl GitHash { } Ok(GitHash(hash)) } + pub fn load_sync() -> Option { + let mut hash = String::from_utf8( + std::process::Command::new("git") + .arg("rev-parse") + .arg("HEAD") + .output() + .ok()? + .stdout, + ) + .ok()?; + if !std::process::Command::new("git") + .arg("diff-index") + .arg("--quiet") + .arg("HEAD") + .arg("--") + .output() + .ok()? + .status + .success() + { + hash += "-modified"; + } + + Some(GitHash(hash)) + } } impl AsRef for GitHash { diff --git a/core/startos/src/s9pk/v2/manifest.rs b/core/startos/src/s9pk/v2/manifest.rs index 85f3cd796..187b2dede 100644 --- a/core/startos/src/s9pk/v2/manifest.rs +++ b/core/startos/src/s9pk/v2/manifest.rs @@ -62,8 +62,8 @@ pub struct Manifest { pub dependencies: Dependencies, #[serde(default)] pub hardware_requirements: HardwareRequirements, - #[serde(default)] - #[ts(type = "string | null")] + #[ts(optional)] + #[serde(default = "GitHash::load_sync")] pub git_hash: Option, #[serde(default = "current_version")] #[ts(type = "string")] diff --git a/sdk/base/lib/Effects.ts b/sdk/base/lib/Effects.ts index 00d56cfba..a0eb08d67 100644 --- a/sdk/base/lib/Effects.ts +++ b/sdk/base/lib/Effects.ts @@ -8,11 +8,10 @@ import { SetHealth, BindParams, HostId, - LanInfo, + NetInfo, Host, ExportServiceInterfaceParams, ServiceInterface, - ActionRequest, RequestActionParams, MainStatus, } from "./osBindings" @@ -118,7 +117,7 @@ export type Effects = { packageId?: PackageId hostId: HostId internalPort: number - }): Promise + }): Promise /** Removes all network bindings, called in the setupInputSpec */ clearBindings(options: { except: { id: HostId; internalPort: number }[] diff --git a/sdk/base/lib/osBindings/GitHash.ts b/sdk/base/lib/osBindings/GitHash.ts new file mode 100644 index 000000000..43f6adde3 --- /dev/null +++ b/sdk/base/lib/osBindings/GitHash.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type GitHash = string diff --git a/sdk/base/lib/osBindings/IpInfo.ts b/sdk/base/lib/osBindings/IpInfo.ts index ae8c88d1b..184e72ddf 100644 --- a/sdk/base/lib/osBindings/IpInfo.ts +++ b/sdk/base/lib/osBindings/IpInfo.ts @@ -1,8 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type IpInfo = { - ipv4Range: string | null - ipv4: string | null - ipv6Range: string | null - ipv6: string | null -} +export type IpInfo = string[] diff --git a/sdk/base/lib/osBindings/Manifest.ts b/sdk/base/lib/osBindings/Manifest.ts index 8007b565b..2c9a2457e 100644 --- a/sdk/base/lib/osBindings/Manifest.ts +++ b/sdk/base/lib/osBindings/Manifest.ts @@ -2,6 +2,7 @@ import type { Alerts } from "./Alerts" import type { Dependencies } from "./Dependencies" import type { Description } from "./Description" +import type { GitHash } from "./GitHash" import type { HardwareRequirements } from "./HardwareRequirements" import type { ImageConfig } from "./ImageConfig" import type { ImageId } from "./ImageId" @@ -30,6 +31,6 @@ export type Manifest = { alerts: Alerts dependencies: Dependencies hardwareRequirements: HardwareRequirements - gitHash: string | null + gitHash?: GitHash osVersion: string } diff --git a/sdk/base/lib/osBindings/PackageVersionInfo.ts b/sdk/base/lib/osBindings/PackageVersionInfo.ts index 80481acb3..c71fd5921 100644 --- a/sdk/base/lib/osBindings/PackageVersionInfo.ts +++ b/sdk/base/lib/osBindings/PackageVersionInfo.ts @@ -3,6 +3,7 @@ import type { Alerts } from "./Alerts" import type { DataUrl } from "./DataUrl" import type { DependencyMetadata } from "./DependencyMetadata" import type { Description } from "./Description" +import type { GitHash } from "./GitHash" import type { HardwareRequirements } from "./HardwareRequirements" import type { MerkleArchiveCommitment } from "./MerkleArchiveCommitment" import type { PackageId } from "./PackageId" @@ -13,7 +14,7 @@ export type PackageVersionInfo = { icon: DataUrl description: Description releaseNotes: string - gitHash: string + gitHash: GitHash license: string wrapperRepo: string upstreamRepo: string diff --git a/sdk/base/lib/osBindings/index.ts b/sdk/base/lib/osBindings/index.ts index 5f0506e39..9ca2b51b8 100644 --- a/sdk/base/lib/osBindings/index.ts +++ b/sdk/base/lib/osBindings/index.ts @@ -90,6 +90,7 @@ export { GetSslKeyParams } from "./GetSslKeyParams" export { GetStatusParams } from "./GetStatusParams" export { GetStoreParams } from "./GetStoreParams" export { GetSystemSmtpParams } from "./GetSystemSmtpParams" +export { GitHash } from "./GitHash" export { Governor } from "./Governor" export { Guid } from "./Guid" export { HardwareRequirements } from "./HardwareRequirements" diff --git a/sdk/package/lib/manifest/setupManifest.ts b/sdk/package/lib/manifest/setupManifest.ts index c529f1ab7..2db2cc839 100644 --- a/sdk/package/lib/manifest/setupManifest.ts +++ b/sdk/package/lib/manifest/setupManifest.ts @@ -27,16 +27,6 @@ export function setupManifest< return manifest } -function gitHash(): string { - const hash = execSync("git rev-parse HEAD").toString().trim() - try { - execSync("git diff-index --quiet HEAD --") - return hash - } catch (e) { - return hash + "-modified" - } -} - export function buildManifest< Id extends string, Version extends string, @@ -67,7 +57,6 @@ export function buildManifest< ) return { ...manifest, - gitHash: gitHash(), osVersion: SDKVersion, version: versions.current.options.version, releaseNotes: versions.current.options.releaseNotes, diff --git a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html index 03d7ef3d7..1ea8f8a79 100644 --- a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html +++ b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html @@ -58,25 +58,18 @@

LAN

- - - -

{{ iface.key }} (IPv4)

-

{{ ipv4 || 'n/a' }}

- - - - - - - -

{{ iface.key }} (IPv6)

-

{{ ipv6 || 'n/a' }}

- - - - - + + + + +

{{ iface.key }} ({{ ipAddr.includes("::") ? "IPv6" : "IPv4" }})

+

{{ ipAddr }}

+ + + + + + Device Credentials diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index aea6b5828..a6a86e0fc 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -42,18 +42,17 @@ export const mockPatchData: DataModel = { lastBackup: new Date(new Date().valueOf() - 604800001).toISOString(), lanAddress: 'https://adjective-noun.local', torAddress: 'https://myveryownspecialtoraddress.onion', - ipInfo: { + networkInterfaces: { eth0: { - ipv4: '10.0.0.1', - ipv4Range: '10.0.0.1/24', - ipv6: null, - ipv6Range: null, + public: false, + ipInfo: ['10.0.0.1/24'], }, wlan0: { - ipv4: '10.0.90.12', - ipv4Range: '10.0.90.12/24', - ipv6: 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD', - ipv6Range: 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64', + public: false, + ipInfo: [ + '10.0.90.12/24', + 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64', + ], }, }, acme: null, From 17d5d1c687889fb521d37bda86b41b45b2648170 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 28 Nov 2024 23:32:48 -0700 Subject: [PATCH 10/29] alpha.23 --- sdk/base/package-lock.json | 8 ++++---- sdk/base/package.json | 2 +- sdk/package/package-lock.json | 12 ++++++------ sdk/package/package.json | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sdk/base/package-lock.json b/sdk/base/package-lock.json index d7b491303..4d5625489 100644 --- a/sdk/base/package-lock.json +++ b/sdk/base/package-lock.json @@ -14,7 +14,7 @@ "isomorphic-fetch": "^3.0.0", "lodash.merge": "^4.6.2", "mime-types": "^2.1.35", - "ts-matches": "^6.1.0", + "ts-matches": "^6.2.1", "yaml": "^2.2.2" }, "devDependencies": { @@ -3897,9 +3897,9 @@ "dev": true }, "node_modules/ts-matches": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.1.0.tgz", - "integrity": "sha512-01qvbIpOiKdbzzXDH84JeHunvCwBGFdZw94jS6kOGLSN5ms+1nBZtfe8WSuYMIPb1xPA+qyAiVgznFi2VCQ6UQ==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.2.1.tgz", + "integrity": "sha512-qdnMgTHsGCEGGK6QiaNMY2vD9eQtRp2Q+pAxcOAzxHJKDKTBYsc1ISTg1zp8H2+EmtCB0eko/1TwYUA5/mUGug==", "license": "MIT" }, "node_modules/ts-morph": { diff --git a/sdk/base/package.json b/sdk/base/package.json index 4cc2fc7ca..6eae719a7 100644 --- a/sdk/base/package.json +++ b/sdk/base/package.json @@ -27,7 +27,7 @@ "isomorphic-fetch": "^3.0.0", "lodash.merge": "^4.6.2", "mime-types": "^2.1.35", - "ts-matches": "^6.1.0", + "ts-matches": "^6.2.1", "yaml": "^2.2.2" }, "prettier": { diff --git a/sdk/package/package-lock.json b/sdk/package/package-lock.json index 1f5715258..3befe3fbf 100644 --- a/sdk/package/package-lock.json +++ b/sdk/package/package-lock.json @@ -1,12 +1,12 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-alpha.21", + "version": "0.3.6-alpha.23", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.3.6-alpha.21", + "version": "0.3.6-alpha.23", "license": "MIT", "dependencies": { "@iarna/toml": "^2.2.5", @@ -15,7 +15,7 @@ "isomorphic-fetch": "^3.0.0", "lodash.merge": "^4.6.2", "mime-types": "^2.1.35", - "ts-matches": "^6.1.0", + "ts-matches": "^6.2.1", "yaml": "^2.2.2" }, "devDependencies": { @@ -3918,9 +3918,9 @@ "dev": true }, "node_modules/ts-matches": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.1.0.tgz", - "integrity": "sha512-01qvbIpOiKdbzzXDH84JeHunvCwBGFdZw94jS6kOGLSN5ms+1nBZtfe8WSuYMIPb1xPA+qyAiVgznFi2VCQ6UQ==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.2.1.tgz", + "integrity": "sha512-qdnMgTHsGCEGGK6QiaNMY2vD9eQtRp2Q+pAxcOAzxHJKDKTBYsc1ISTg1zp8H2+EmtCB0eko/1TwYUA5/mUGug==", "license": "MIT" }, "node_modules/ts-morph": { diff --git a/sdk/package/package.json b/sdk/package/package.json index 2bf4b71f5..b720350a4 100644 --- a/sdk/package/package.json +++ b/sdk/package/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-alpha.21", + "version": "0.3.6-alpha.23", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./package/lib/index.js", "types": "./package/lib/index.d.ts", @@ -33,7 +33,7 @@ "isomorphic-fetch": "^3.0.0", "lodash.merge": "^4.6.2", "mime-types": "^2.1.35", - "ts-matches": "^6.1.0", + "ts-matches": "^6.2.1", "yaml": "^2.2.2", "@iarna/toml": "^2.2.5", "@noble/curves": "^1.4.0", From cf1eefd2e989df8bedad6b41bd762bf46ac9d089 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 2 Dec 2024 18:13:13 -0700 Subject: [PATCH 11/29] misc fixes --- build/lib/scripts/enable-kiosk | 5 +- core/Cargo.lock | 12 + core/startos/Cargo.toml | 1 + core/startos/src/db/model/public.rs | 1 + core/startos/src/net/network_interface.rs | 334 +++++++++++------- core/startos/src/net/vhost.rs | 105 +++--- core/startos/src/util/io.rs | 20 +- .../lib/osBindings/NetworkInterfaceInfo.ts | 6 +- .../server-specs/server-specs.page.html | 22 +- .../ui/src/app/services/api/mock-patch.ts | 2 + 10 files changed, 324 insertions(+), 184 deletions(-) diff --git a/build/lib/scripts/enable-kiosk b/build/lib/scripts/enable-kiosk index 45bed5fe9..40753af40 100755 --- a/build/lib/scripts/enable-kiosk +++ b/build/lib/scripts/enable-kiosk @@ -4,7 +4,7 @@ set -e # install dependencies /usr/bin/apt update -/usr/bin/apt install --no-install-recommends -y xserver-xorg x11-xserver-utils xinit firefox-esr matchbox-window-manager libnss3-tools +/usr/bin/apt install --no-install-recommends -y xserver-xorg x11-xserver-utils xinit firefox-esr matchbox-window-manager libnss3-tools p11-kit-modules #Change a default preference set by stock debian firefox-esr sed -i 's|^pref("extensions.update.enabled", true);$|pref("extensions.update.enabled", false);|' /etc/firefox-esr/firefox-esr.js @@ -83,6 +83,9 @@ user_pref("toolkit.telemetry.updatePing.enabled", false); user_pref("toolkit.telemetry.cachedClientID", ""); EOF +cp /usr/lib/firefox-esr/libnssckbi.so /usr/lib/firefox-esr/libnssckbi.so.bak +ln -sf /usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-trust.so /usr/lib/firefox-esr/libnssckbi.so + # create kiosk script cat > /home/kiosk/kiosk.sh << 'EOF' #!/bin/sh diff --git a/core/Cargo.lock b/core/Cargo.lock index db5055e87..a51bf95b9 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2244,6 +2244,17 @@ dependencies = [ "webpki-roots 0.26.6", ] +[[package]] +name = "getifaddrs" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba121d81ab5ea05b0cd5858516266800bf965531a794f7ac58e3eeb804f364f" +dependencies = [ + "bitflags 2.6.0", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -5676,6 +5687,7 @@ dependencies = [ "fd-lock-rs", "form_urlencoded", "futures", + "getifaddrs", "gpt", "helpers", "hex", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index cfe1e3347..43ea2c19b 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -96,6 +96,7 @@ exver = { version = "0.2.0", git = "https://github.com/Start9Labs/exver-rs.git", fd-lock-rs = "0.1.4" form_urlencoded = "1.2.1" futures = "0.3.28" +getifaddrs = "0.1.5" gpt = "3.1.0" helpers = { path = "../helpers" } hex = "0.4.3" diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 2b4bfe945..7a9b28b5d 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -159,6 +159,7 @@ pub struct ServerInfo { #[ts(export)] pub struct NetworkInterfaceInfo { pub public: bool, + pub scope_id: Option, pub ip_info: IpInfo, } diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index ae139e7a9..1554a6c91 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -1,19 +1,17 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; -use std::net::{IpAddr, SocketAddr}; -use std::ops::Deref; +use std::net::{IpAddr, SocketAddr, SocketAddrV6}; use std::pin::Pin; use std::sync::{Arc, Weak}; use std::task::Poll; -use clap::Parser; use futures::future::pending; -use futures::{FutureExt, TryFutureExt, TryStreamExt}; +use futures::TryFutureExt; +use getifaddrs::if_nametoindex; use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use ipnet::IpNet; -use serde::{Deserialize, Serialize}; -use serde_json::de; +use itertools::Itertools; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; use tokio_stream::StreamExt; @@ -26,8 +24,6 @@ use zbus::{proxy, Connection}; use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; use crate::db::model::Database; use crate::prelude::*; -use crate::util::actor::background::BackgroundJobQueue; -use crate::util::logger::EmbassyLogger; use crate::util::sync::SyncMutex; #[proxy( @@ -37,7 +33,7 @@ use crate::util::sync::SyncMutex; )] trait NetworkManager { #[zbus(property)] - fn active_connections(&self) -> Result, Error>; + fn devices(&self) -> Result, Error>; } #[proxy( @@ -46,9 +42,7 @@ trait NetworkManager { )] trait ActiveConnection { #[zbus(property)] - fn ip4_config(&self) -> Result; - #[zbus(property)] - fn devices(&self) -> Result, Error>; + fn state_flags(&self) -> Result; #[zbus(property, name = "Type")] fn connection_type(&self) -> Result; } @@ -62,6 +56,15 @@ trait Ip4Config { fn address_data(&self) -> Result, Error>; } +#[proxy( + interface = "org.freedesktop.NetworkManager.IP6Config", + default_service = "org.freedesktop.NetworkManager" +)] +trait Ip6Config { + #[zbus(property)] + fn address_data(&self) -> Result, Error>; +} + #[derive(Clone, Debug, DeserializeDict, ZValue, ZType)] #[zvariant(signature = "dict")] struct AddressData { @@ -76,6 +79,7 @@ impl TryFrom> for IpInfo { .map(|a| { IpNet::new(a.address.parse()?, a.prefix as u8).with_kind(ErrorKind::ParseNetAddress) }) + .filter_ok(|ipnet| !ipnet.addr().is_unspecified() && !ipnet.addr().is_multicast()) .collect::>() .map(Self) } @@ -88,19 +92,18 @@ impl TryFrom> for IpInfo { trait Device { #[zbus(property)] fn ip_interface(&self) -> Result; -} -#[tokio::test] -async fn test() -> Result<(), Error> { - EmbassyLogger::init(); - let (write_to, mut read_from) = watch::channel(BTreeMap::new()); - tokio::task::spawn(watcher(write_to)); - loop { - eprintln!("{:?}", &*read_from.borrow()); - read_from.changed().await; - } + #[zbus(property)] + fn managed(&self) -> Result; + + #[zbus(property)] + fn active_connection(&self) -> Result; + + #[zbus(property)] + fn ip4_config(&self) -> Result; - Ok(()) + #[zbus(property)] + fn ip6_config(&self) -> Result; } struct WatchPropertyStream<'a, T> { @@ -135,91 +138,53 @@ where } } +#[instrument(skip_all)] async fn watcher(write_to: watch::Sender>) { loop { - if let Err(e) = async { + let res: Result<(), Error> = async { let connection = Connection::system().await?; let netman_proxy = NetworkManagerProxy::new(&connection).await?; - let mut active_sub = WatchPropertyStream::new( - netman_proxy.receive_active_connections_changed().await, - netman_proxy.active_connections().await?, + let mut devices_sub = WatchPropertyStream::new( + netman_proxy.receive_devices_changed().await, + netman_proxy.devices().await?, ); loop { - let active = active_sub.last.clone(); - active_sub + let devices = devices_sub.last.clone(); + devices_sub .until_changed(async { let mut ifaces = BTreeSet::new(); let mut jobs = Vec::new(); - for active in active { - let ac_proxy = ActiveConnectionProxy::new(&connection, active).await?; - // dbg!(ac_proxy.connection_type().await?); - let devices = ac_proxy.devices().await?; - for device in devices { - let dev_proxy = DeviceProxy::new(&connection, device).await?; - let iface = InternedString::intern(dev_proxy.ip_interface().await?); - if iface.is_empty() { - continue; - } - ifaces.insert(iface.clone()); - let ac_proxy = ac_proxy.clone(); - jobs.push(async { - let ac_proxy = ac_proxy; - let iface = iface; - let mut ip_config_sub = WatchPropertyStream::new( - ac_proxy.receive_ip4_config_changed().await, - ac_proxy.ip4_config().await?, - ); - - loop { - let ip_config = ip_config_sub.last.clone(); - ip_config_sub - .until_changed(async { - let ip_proxy = - Ip4ConfigProxy::new(&connection, ip_config) - .await?; - let mut address_sub = WatchPropertyStream::new( - ip_proxy.receive_address_data_changed().await, - ip_proxy.address_data().await?, - ); - - loop { - let addresses = address_sub.last.clone(); - address_sub - .until_changed(async { - let ip_info: IpInfo = - addresses.try_into()?; - - write_to.send_if_modified(|m| { - let public = m - .get(&iface) - .map_or(false, |i| i.public); - m.insert( - iface.clone(), - NetworkInterfaceInfo { - public, - ip_info: ip_info.clone(), - }, - ) - .filter(|old| { - &old.ip_info == &ip_info - }) - .is_none() - }); - - Ok::<_, Error>(()) - }) - .await?; - } - }) - .await?; - } - - Ok::<_, Error>(()) - }); + for device in devices { + let device_proxy = + DeviceProxy::new(&connection, device.clone()).await?; + let iface = InternedString::intern(device_proxy.ip_interface().await?); + if iface.is_empty() { + continue; + } + let managed = device_proxy.managed().await?; + if !managed { + continue; } + let dac = device_proxy.active_connection().await?; + if &*dac == "/" { + continue; + } + let ac_proxy = ActiveConnectionProxy::new(&connection, dac).await?; + let external = ac_proxy.state_flags().await? & 0x80 != 0; + if external && iface != "lo" { + continue; + } + jobs.push(watch_ip( + &connection, + device_proxy.clone(), + iface.clone(), + &write_to, + )); + ifaces.insert(iface); } + write_to.send_if_modified(|m| { let mut changed = false; m.retain(|i, _| { @@ -238,21 +203,87 @@ async fn watcher(write_to: watch::Sender(()) } - .await - { + .await; + if let Err(e) = res { tracing::error!("{e}"); tracing::debug!("{e:?}"); } } } +#[instrument(skip_all)] +async fn watch_ip( + connection: &Connection, + device_proxy: DeviceProxy<'_>, + iface: InternedString, + write_to: &watch::Sender>, +) -> Result<(), Error> { + let mut ip4_config_sub = WatchPropertyStream::new( + device_proxy.receive_ip4_config_changed().await, + device_proxy.ip4_config().await?, + ); + let mut ip6_config_sub = WatchPropertyStream::new( + device_proxy.receive_ip6_config_changed().await, + device_proxy.ip6_config().await?, + ); + + loop { + let ip4_config = ip4_config_sub.last.clone(); + let ip6_config = ip6_config_sub.last.clone(); + ip4_config_sub + .until_changed(ip6_config_sub.until_changed(async { + let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config).await?; + let mut address4_sub = WatchPropertyStream::new( + ip4_proxy.receive_address_data_changed().await, + ip4_proxy.address_data().await?, + ); + let ip6_proxy = Ip6ConfigProxy::new(&connection, ip6_config).await?; + let mut address6_sub = WatchPropertyStream::new( + ip6_proxy.receive_address_data_changed().await, + ip6_proxy.address_data().await?, + ); + + loop { + let addresses = address4_sub + .last + .iter() + .cloned() + .chain(address6_sub.last.iter().cloned()) + .collect_vec(); + address4_sub + .until_changed(address6_sub.until_changed(async { + let ip_info: IpInfo = addresses.try_into()?; + let scope_id = + Some(if_nametoindex(&*iface).with_kind(ErrorKind::Network)?); + + write_to.send_if_modified(|m| { + let public = m.get(&iface).map_or(false, |i| i.public); + m.insert( + iface.clone(), + NetworkInterfaceInfo { + public, + scope_id, + ip_info: ip_info.clone(), + }, + ) + .filter(|old| &old.ip_info == &ip_info && old.scope_id == scope_id) + .is_none() + }); + + Ok::<_, Error>(()) + })) + .await?; + } + })) + .await?; + } +} + pub struct NetworkInterfaceController { db: TypedPatchDb, ip_info: watch::Sender>, - watcher: NonDetachingJoinHandle<()>, + _watcher: NonDetachingJoinHandle<()>, listeners: SyncMutex>>, } impl NetworkInterfaceController { @@ -265,11 +296,20 @@ impl NetworkInterfaceController { .as_public_mut() .as_server_info_mut() .as_network_interfaces_mut(); - for (iface, ip_info) in ip_info { + let mut keep = BTreeSet::new(); + for (iface, ip_info) in dbg!(ip_info) { + keep.insert(iface); ifaces_model .upsert(&iface, || Ok(NetworkInterfaceInfo::default()))? .ser(&ip_info)?; } + for iface in ifaces_model.keys()? { + if !keep.contains(&&iface) { + if let Some(info) = ifaces_model.as_idx_mut(&iface) { + info.as_ip_info_mut().ser(&IpInfo::default())?; + } + } + } Ok(()) }) .await?; @@ -280,14 +320,35 @@ impl NetworkInterfaceController { Self { db: db.clone(), ip_info: write_to.clone(), - watcher: tokio::spawn(async move { + _watcher: tokio::spawn(async move { + match db + .peek() + .await + .as_public() + .as_server_info() + .as_network_interfaces() + .de() + { + Ok(info) => { + write_to.send_replace(info); + } + Err(e) => { + tracing::error!("Error loading network interface info: {e}"); + tracing::debug!("{e:?}"); + } + }; tokio::join!(watcher(write_to), async { loop { if let Err(e) = async { let ip_info = read_from.borrow().clone(); - Self::sync(&db, &ip_info).await?; + Self::sync(&db, dbg!(&ip_info)).await?; - read_from.changed().await; + read_from.changed().await.map_err(|_| { + Error::new( + eyre!("NetworkManager watch thread exited"), + ErrorKind::Network, + ) + })?; Ok::<_, Error>(()) } @@ -319,56 +380,70 @@ impl NetworkInterfaceController { Ok(NetworkInterfaceListener { _arc: arc, ip_info: self.ip_info.subscribe(), - listeners: ListenerMap::new(), - port, + listeners: ListenerMap::new(port), + needs_update: true, }) } } -struct ListenerMap(BTreeMap); +struct ListenerMap { + port: u16, + listeners: BTreeMap<(IpAddr, u32), (TcpListener, bool)>, +} impl ListenerMap { - fn new() -> Self { - Self(BTreeMap::new()) + fn new(port: u16) -> Self { + Self { + port, + listeners: BTreeMap::new(), + } } async fn update( &mut self, ip_info: &BTreeMap, - port: u16, public: bool, ) -> Result<(), Error> { - let mut keep = BTreeSet::::new(); + let mut keep = BTreeSet::<(IpAddr, u32)>::new(); for info in ip_info.values() { if public || !info.public { for ipnet in &info.ip_info.0 { - if let Some((_, is_public)) = self.0.get_mut(&ipnet.addr()) { + let scope_id = info.scope_id.unwrap_or(0); + let key = (ipnet.addr(), scope_id); + keep.insert(key); + if let Some((_, is_public)) = self.listeners.get_mut(&key) { *is_public = info.public; continue; } - self.0.insert( - ipnet.addr(), + self.listeners.insert( + key, ( - TcpListener::bind(SocketAddr::new(ipnet.addr(), port)).await?, + TcpListener::bind(match ipnet.addr() { + IpAddr::V6(ip6) => { + SocketAddrV6::new(ip6, self.port, 0, scope_id).into() + } + ip => SocketAddr::new(ip, self.port), + }) + .await?, info.public, ), ); - keep.insert(ipnet.addr()); } } } - self.0.retain(|ip, _| keep.contains(ip)); + self.listeners.retain(|key, _| keep.contains(key)); Ok(()) } fn accept(&mut self) -> ListenerMapFut { - ListenerMapFut(&mut self.0) + ListenerMapFut(&mut self.listeners) } } #[pin_project::pin_project] -struct ListenerMapFut<'a>(&'a mut BTreeMap); +struct ListenerMapFut<'a>(&'a mut BTreeMap<(IpAddr, u32), (TcpListener, bool)>); impl<'a> Future for ListenerMapFut<'a> { type Output = Result<(IpAddr, bool, TcpStream, SocketAddr), Error>; fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { let this = self.project(); - for (ip, listener) in this.0.iter() { + for ((ip, _), listener) in this.0.iter() { + dbg!(ip, listener); if let Poll::Ready((stream, addr)) = listener.0.poll_accept(cx)? { return Poll::Ready(Ok((*ip, listener.1, stream, addr))); } @@ -378,16 +453,19 @@ impl<'a> Future for ListenerMapFut<'a> { } pub struct NetworkInterfaceListener { + needs_update: bool, ip_info: watch::Receiver>, listeners: ListenerMap, - port: u16, _arc: Arc<()>, } impl NetworkInterfaceListener { pub async fn accept(&mut self, public: bool) -> Result { loop { - let ip_info = self.ip_info.borrow().clone(); - self.listeners.update(&ip_info, self.port, public).await?; + if self.needs_update { + let ip_info = self.ip_info.borrow().clone(); + self.listeners.update(dbg!(&ip_info), public).await?; + self.needs_update = false; + } tokio::select! { accepted = self.listeners.accept() => { let (ip, is_public, stream, peer) = accepted?; @@ -395,10 +473,16 @@ impl NetworkInterfaceListener { stream, peer, is_public, - bind: (ip, self.port).into(), + bind: (ip, self.listeners.port).into(), }) }, - _ = self.ip_info.changed() => {} + res = self.ip_info.changed() => { + res.map_err(|_| Error::new( + eyre!("NetworkManager watch thread exited"), + ErrorKind::Network, + ))?; + self.needs_update = true; + } } } } diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs index 63c13e474..9f662f790 100644 --- a/core/startos/src/net/vhost.rs +++ b/core/startos/src/net/vhost.rs @@ -193,50 +193,69 @@ impl VHostServer { let mid: tokio_rustls::StartHandshake<&mut BackTrackingIO> = match LazyConfigAcceptor::new(Acceptor::default(), &mut stream).await { Ok(a) => a, - Err(_) => { - stream.rewind(); - return hyper_util::server::conn::auto::Builder::new( - hyper_util::rt::TokioExecutor::new(), - ) - .serve_connection( - hyper_util::rt::TokioIo::new(stream), - hyper_util::service::TowerToHyperService::new( - axum::Router::new().fallback(axum::routing::method_routing::any( - move |req: Request| async move { - match async move { - let host = req - .headers() - .get(http::header::HOST) - .and_then(|host| host.to_str().ok()); - let uri = Uri::from_parts({ - let mut parts = req.uri().to_owned().into_parts(); - parts.scheme = Some("https".parse()?); - parts.authority = - host.map(FromStr::from_str).transpose()?; - parts - })?; - Response::builder() - .status(http::StatusCode::TEMPORARY_REDIRECT) - .header(http::header::LOCATION, uri.to_string()) - .body(Body::default()) - } - .await - { - Ok(a) => a, - Err(e) => { - tracing::warn!( + Err(e) => { + let (_, buf) = stream.rewind(); + if std::str::from_utf8(buf) + .ok() + .and_then(|buf| { + buf.lines() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .next() + }) + .map_or(false, |buf| { + regex::Regex::new("[A-Z]+ (.+) HTTP/1") + .unwrap() + .is_match(buf) + }) + { + return hyper_util::server::conn::auto::Builder::new( + hyper_util::rt::TokioExecutor::new(), + ) + .serve_connection( + hyper_util::rt::TokioIo::new(stream), + hyper_util::service::TowerToHyperService::new( + axum::Router::new().fallback(axum::routing::method_routing::any( + move |req: Request| async move { + match async move { + let host = req + .headers() + .get(http::header::HOST) + .and_then(|host| host.to_str().ok()); + let uri = Uri::from_parts({ + let mut parts = req.uri().to_owned().into_parts(); + parts.scheme = Some("https".parse()?); + parts.authority = + host.map(FromStr::from_str).transpose()?; + parts + })?; + Response::builder() + .status(http::StatusCode::TEMPORARY_REDIRECT) + .header(http::header::LOCATION, uri.to_string()) + .body(Body::default()) + } + .await + { + Ok(a) => a, + Err(e) => { + tracing::warn!( "Error redirecting http request on ssl port: {e}" ); - tracing::error!("{e:?}"); - server_error(Error::new(e, ErrorKind::Network)) + tracing::error!("{e:?}"); + server_error(Error::new(e, ErrorKind::Network)) + } } - } - }, - )), - ), - ) - .await - .map_err(|e| Error::new(color_eyre::eyre::Report::msg(e), ErrorKind::Network)); + }, + )), + ), + ) + .await + .map_err(|e| { + Error::new(color_eyre::eyre::Report::msg(e), ErrorKind::Network) + }); + } else { + return Err(e).with_kind(ErrorKind::Network); + } } }; let target_name = mid.client_hello().server_name().map(|s| s.into()); @@ -544,10 +563,10 @@ impl VHostServer { ) .await { - tracing::trace!( + tracing::error!( "VHostController: failed to accept connection on {port}: {e}" ); - tracing::trace!("{e:?}"); + tracing::debug!("{e:?}"); } } }) diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs index 0e7aada54..8e270c912 100644 --- a/core/startos/src/util/io.rs +++ b/core/startos/src/util/io.rs @@ -460,18 +460,30 @@ impl BackTrackingIO { } } } - pub fn rewind(&mut self) -> Vec { + pub fn rewind<'a>(&'a mut self) -> (Vec, &'a [u8]) { match std::mem::take(&mut self.buffer) { BTBuffer::Buffering { read, write } => { self.buffer = BTBuffer::Rewound { read: Cursor::new(read), }; - write + ( + write, + match &self.buffer { + BTBuffer::Rewound { read } => read.get_ref(), + _ => unreachable!(), + }, + ) } - BTBuffer::NotBuffering => Vec::new(), + BTBuffer::NotBuffering => (Vec::new(), &[]), BTBuffer::Rewound { read } => { self.buffer = BTBuffer::Rewound { read }; - Vec::new() + ( + Vec::new(), + match &self.buffer { + BTBuffer::Rewound { read } => read.get_ref(), + _ => unreachable!(), + }, + ) } } } diff --git a/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts index f389786aa..ecb8bcea7 100644 --- a/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts +++ b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts @@ -1,4 +1,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { IpInfo } from "./IpInfo" -export type NetworkInterfaceInfo = { public: boolean; ipInfo: IpInfo } +export type NetworkInterfaceInfo = { + public: boolean + scopeId: number | null + ipInfo: IpInfo +} diff --git a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html index 1ea8f8a79..f685b3b32 100644 --- a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html +++ b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html @@ -59,16 +59,18 @@

LAN

- - - -

{{ iface.key }} ({{ ipAddr.includes("::") ? "IPv6" : "IPv4" }})

-

{{ ipAddr }}

- - - - - + + + + +

{{ iface.key }} ({{ ipAddr.includes("::") ? "IPv6" : "IPv4" }})

+

{{ ipAddr.includes("::") ? "[" + ipAddr + (iface.value.scopeId ? "%" + iface.value.scopeId : "") + "]" : ipAddr }}

+ + + + + + diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index a6a86e0fc..f66f2add8 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -45,10 +45,12 @@ export const mockPatchData: DataModel = { networkInterfaces: { eth0: { public: false, + scopeId: 1, ipInfo: ['10.0.0.1/24'], }, wlan0: { public: false, + scopeId: 2, ipInfo: [ '10.0.90.12/24', 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64', From eb5de045a7c00ed1255f8d70cd06bb06aaaa5a11 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 2 Dec 2024 18:16:02 -0700 Subject: [PATCH 12/29] remove ufw since no longer required --- build/dpkg-deps/depends | 1 - 1 file changed, 1 deletion(-) diff --git a/build/dpkg-deps/depends b/build/dpkg-deps/depends index 0f0f091b6..4c2dbc557 100644 --- a/build/dpkg-deps/depends +++ b/build/dpkg-deps/depends @@ -56,7 +56,6 @@ systemd-resolved systemd-sysv systemd-timesyncd tor -ufw util-linux vim wireguard-tools From d5fe5278f686e4752b29bee879244984e431a189 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 2 Dec 2024 18:25:57 -0700 Subject: [PATCH 13/29] remove debug info --- core/startos/src/net/network_interface.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 1554a6c91..79aae67bc 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -297,7 +297,7 @@ impl NetworkInterfaceController { .as_server_info_mut() .as_network_interfaces_mut(); let mut keep = BTreeSet::new(); - for (iface, ip_info) in dbg!(ip_info) { + for (iface, ip_info) in ip_info { keep.insert(iface); ifaces_model .upsert(&iface, || Ok(NetworkInterfaceInfo::default()))? @@ -341,7 +341,7 @@ impl NetworkInterfaceController { loop { if let Err(e) = async { let ip_info = read_from.borrow().clone(); - Self::sync(&db, dbg!(&ip_info)).await?; + Self::sync(&db, &ip_info).await?; read_from.changed().await.map_err(|_| { Error::new( @@ -443,7 +443,6 @@ impl<'a> Future for ListenerMapFut<'a> { fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { let this = self.project(); for ((ip, _), listener) in this.0.iter() { - dbg!(ip, listener); if let Poll::Ready((stream, addr)) = listener.0.poll_accept(cx)? { return Poll::Ready(Ok((*ip, listener.1, stream, addr))); } @@ -463,7 +462,7 @@ impl NetworkInterfaceListener { loop { if self.needs_update { let ip_info = self.ip_info.borrow().clone(); - self.listeners.update(dbg!(&ip_info), public).await?; + self.listeners.update(&ip_info, public).await?; self.needs_update = false; } tokio::select! { From 309798b722279cdc1747763d59c3354b7a8f934a Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 3 Dec 2024 15:55:01 -0700 Subject: [PATCH 14/29] add cli bindings --- core/startos/src/db/model/public.rs | 17 +- core/startos/src/net/mod.rs | 5 + core/startos/src/net/net_controller.rs | 14 +- core/startos/src/net/network_interface.rs | 151 +++++++++++++++++- .../lib/osBindings/NetworkInterfaceInfo.ts | 2 +- sdk/base/lib/osBindings/SetPublicParams.ts | 3 + sdk/base/lib/osBindings/UnsetPublicParams.ts | 3 + sdk/base/lib/osBindings/index.ts | 2 + 8 files changed, 183 insertions(+), 14 deletions(-) create mode 100644 sdk/base/lib/osBindings/SetPublicParams.ts create mode 100644 sdk/base/lib/osBindings/UnsetPublicParams.ts diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 7a9b28b5d..6cc81722e 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -1,5 +1,5 @@ use std::collections::{BTreeMap, BTreeSet}; -use std::net::{Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use chrono::{DateTime, Utc}; use exver::{Version, VersionRange}; @@ -158,10 +158,23 @@ pub struct ServerInfo { #[model = "Model"] #[ts(export)] pub struct NetworkInterfaceInfo { - pub public: bool, + pub public: Option, pub scope_id: Option, pub ip_info: IpInfo, } +impl NetworkInterfaceInfo { + pub fn public(&self) -> bool { + self.public.unwrap_or_else(|| { + !self.ip_info.0.iter().all(|ipnet| { + if let IpAddr::V4(ip4) = ipnet.addr() { + ip4.is_loopback() || ip4.is_private() || ip4.is_link_local() + } else { + true + } + }) + }) + } +} #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, TS)] #[ts(export)] diff --git a/core/startos/src/net/mod.rs b/core/startos/src/net/mod.rs index 9cf72c30b..53ba39887 100644 --- a/core/startos/src/net/mod.rs +++ b/core/startos/src/net/mod.rs @@ -33,4 +33,9 @@ pub fn net() -> ParentHandler { "acme", acme::acme::().with_about("Setup automatic clearnet certificate acquisition"), ) + .subcommand( + "network-interface", + network_interface::network_interface_api::() + .with_about("View and edit network interface configurations"), + ) } diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index fadaddfc6..2d425cdfd 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -379,7 +379,7 @@ impl NetService { let mut bind_hostname_info: Vec = hostname_info.remove(port).unwrap_or_default(); for (interface, iface_info) in &net_ifaces { - if !iface_info.public { + if !iface_info.public() { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), public: false, @@ -402,7 +402,7 @@ impl NetService { { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), - public: iface_info.public, + public: true, // TODO: check if port forward is active hostname: IpHostname::Domain { domain: address.clone(), subdomain: None, @@ -410,10 +410,10 @@ impl NetService { ssl_port: Some(443), }, }); - } else if iface_info.public && new_lan_bind.0.public { + } else if iface_info.public() && new_lan_bind.0.public { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), - public: iface_info.public, + public: iface_info.public(), hostname: IpHostname::Domain { domain: address.clone(), subdomain: None, @@ -424,13 +424,13 @@ impl NetService { } } } - if !iface_info.public || new_lan_bind.0.public { + if !iface_info.public() || new_lan_bind.0.public { for ipnet in &iface_info.ip_info.0 { match ipnet { IpNet::V4(net) => { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), - public: iface_info.public, + public: iface_info.public(), hostname: IpHostname::Ipv4 { value: net.addr(), port: new_lan_bind.0.assigned_port, @@ -441,7 +441,7 @@ impl NetService { IpNet::V6(net) => { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), - public: iface_info.public, + public: iface_info.public(), hostname: IpHostname::Ipv6 { value: net.addr(), port: new_lan_bind.0.assigned_port, diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 79aae67bc..f4624bcfa 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -5,6 +5,7 @@ use std::pin::Pin; use std::sync::{Arc, Weak}; use std::task::Poll; +use clap::Parser; use futures::future::pending; use futures::TryFutureExt; use getifaddrs::if_nametoindex; @@ -12,20 +13,125 @@ use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use ipnet::IpNet; use itertools::Itertools; +use patch_db::json_ptr::JsonPointer; +use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; +use serde::{Deserialize, Serialize}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; use tokio_stream::StreamExt; +use ts_rs::TS; use zbus::proxy::PropertyStream; use zbus::zvariant::{ DeserializeDict, OwnedObjectPath, OwnedValue, Type as ZType, Value as ZValue, }; use zbus::{proxy, Connection}; +use crate::context::{CliContext, RpcContext}; use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; use crate::db::model::Database; use crate::prelude::*; +use crate::util::serde::{display_serializable, HandlerExtSerde}; use crate::util::sync::SyncMutex; +pub fn network_interface_api() -> ParentHandler { + ParentHandler::new() + .subcommand( + "list", + from_fn_async(list_interfaces) + .with_display_serializable() + .with_custom_display_fn(|HandlerArgs { params, .. }, res| { + use prettytable::*; + + if let Some(format) = params.format { + return Ok(display_serializable(format, res)); + } + + let mut table = Table::new(); + table.add_row(row![bc => "INTERFACE", "PUBLIC", "ADDRESSES"]); + for (iface, info) in res { + table.add_row(row![ + iface, + info.public(), + info.ip_info + .0 + .into_iter() + .map(|ipnet| match ipnet.addr() { + IpAddr::V4(ip) => format!("{ip}/{}", ipnet.prefix_len()), + IpAddr::V6(ip) => format!( + "[{ip}{}]/{}", + info.scope_id.map(|s| format!("%{s}")).unwrap_or_default(), + ipnet.prefix_len() + ), + }) + .join(", ") + ]); + } + + table.print_tty(false).unwrap(); + + Ok(()) + }) + .with_about("Show network interfaces StartOS can listen on") + .with_call_remote::(), + ) + .subcommand( + "set-public", + from_fn_async(set_public) + .with_metadata("sync_db", Value::Bool(true)) + .no_display() + .with_about("Indicate whether this interface is publicly addressable") + .with_call_remote::(), + ).subcommand( + "unset-public", + from_fn_async(unset_public) + .with_metadata("sync_db", Value::Bool(true)) + .no_display() + .with_about("Allow this interface to infer whether it is publicly addressable based on its IPv4 address") + .with_call_remote::(), + ) +} + +async fn list_interfaces( + ctx: RpcContext, +) -> Result, Error> { + Ok(ctx.net_controller.net_iface.ip_info.borrow().clone()) +} + +#[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)] +#[ts(export)] +struct SetPublicParams { + #[ts(type = "string")] + interface: InternedString, + public: Option, +} + +async fn set_public( + ctx: RpcContext, + SetPublicParams { interface, public }: SetPublicParams, +) -> Result<(), Error> { + ctx.net_controller + .net_iface + .set_public(&interface, Some(public.unwrap_or(true))) + .await +} + +#[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)] +#[ts(export)] +struct UnsetPublicParams { + #[ts(type = "string")] + interface: InternedString, +} + +async fn unset_public( + ctx: RpcContext, + UnsetPublicParams { interface }: UnsetPublicParams, +) -> Result<(), Error> { + ctx.net_controller + .net_iface + .set_public(&interface, None) + .await +} + #[proxy( interface = "org.freedesktop.NetworkManager", default_service = "org.freedesktop.NetworkManager", @@ -258,7 +364,7 @@ async fn watch_ip( Some(if_nametoindex(&*iface).with_kind(ErrorKind::Network)?); write_to.send_if_modified(|m| { - let public = m.get(&iface).map_or(false, |i| i.public); + let public = m.get(&iface).map_or(None, |i| i.public); m.insert( iface.clone(), NetworkInterfaceInfo { @@ -384,6 +490,43 @@ impl NetworkInterfaceController { needs_update: true, }) } + + pub async fn set_public( + &self, + interface: &InternedString, + public: Option, + ) -> Result<(), Error> { + let mut sub = self + .db + .subscribe( + "/public/serverInfo/networkInterfaces" + .parse::>() + .with_kind(ErrorKind::Database)?, + ) + .await; + let mut err = None; + let changed = self.ip_info.send_if_modified(|ip_info| { + let prev = std::mem::replace( + &mut match ip_info.get_mut(interface).or_not_found(interface) { + Ok(a) => a, + Err(e) => { + err = Some(e); + return false; + } + } + .public, + public, + ); + prev == public + }); + if let Some(e) = err { + return Err(e); + } + if changed { + sub.recv().await; + } + Ok(()) + } } struct ListenerMap { @@ -404,13 +547,13 @@ impl ListenerMap { ) -> Result<(), Error> { let mut keep = BTreeSet::<(IpAddr, u32)>::new(); for info in ip_info.values() { - if public || !info.public { + if public || !info.public() { for ipnet in &info.ip_info.0 { let scope_id = info.scope_id.unwrap_or(0); let key = (ipnet.addr(), scope_id); keep.insert(key); if let Some((_, is_public)) = self.listeners.get_mut(&key) { - *is_public = info.public; + *is_public = info.public(); continue; } self.listeners.insert( @@ -423,7 +566,7 @@ impl ListenerMap { ip => SocketAddr::new(ip, self.port), }) .await?, - info.public, + info.public(), ), ); } diff --git a/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts index ecb8bcea7..c9f82005d 100644 --- a/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts +++ b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts @@ -2,7 +2,7 @@ import type { IpInfo } from "./IpInfo" export type NetworkInterfaceInfo = { - public: boolean + public: boolean | null scopeId: number | null ipInfo: IpInfo } diff --git a/sdk/base/lib/osBindings/SetPublicParams.ts b/sdk/base/lib/osBindings/SetPublicParams.ts new file mode 100644 index 000000000..03bc3082b --- /dev/null +++ b/sdk/base/lib/osBindings/SetPublicParams.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SetPublicParams = { interface: string; public: boolean | null } diff --git a/sdk/base/lib/osBindings/UnsetPublicParams.ts b/sdk/base/lib/osBindings/UnsetPublicParams.ts new file mode 100644 index 000000000..db8f730e1 --- /dev/null +++ b/sdk/base/lib/osBindings/UnsetPublicParams.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type UnsetPublicParams = { interface: string } diff --git a/sdk/base/lib/osBindings/index.ts b/sdk/base/lib/osBindings/index.ts index 9ca2b51b8..623ebc23a 100644 --- a/sdk/base/lib/osBindings/index.ts +++ b/sdk/base/lib/osBindings/index.ts @@ -174,6 +174,7 @@ export { SetIconParams } from "./SetIconParams" export { SetMainStatusStatus } from "./SetMainStatusStatus" export { SetMainStatus } from "./SetMainStatus" export { SetNameParams } from "./SetNameParams" +export { SetPublicParams } from "./SetPublicParams" export { SetStoreParams } from "./SetStoreParams" export { SetupExecuteParams } from "./SetupExecuteParams" export { SetupProgress } from "./SetupProgress" @@ -183,6 +184,7 @@ export { SignAssetParams } from "./SignAssetParams" export { SignerInfo } from "./SignerInfo" export { SmtpValue } from "./SmtpValue" export { StartStop } from "./StartStop" +export { UnsetPublicParams } from "./UnsetPublicParams" export { UpdatingState } from "./UpdatingState" export { VerifyCifsParams } from "./VerifyCifsParams" export { VersionSignerParams } from "./VersionSignerParams" From 8f68822517675e1c863a12e232a8c6ee9901b4a0 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 5 Dec 2024 16:15:46 -0700 Subject: [PATCH 15/29] debugging --- core/startos/src/db/model/public.rs | 26 +- core/startos/src/init.rs | 2 +- core/startos/src/net/net_controller.rs | 54 ++- core/startos/src/net/network_interface.rs | 311 +++++++++++------- core/startos/src/net/utils.rs | 4 + core/startos/src/net/vhost.rs | 4 +- core/startos/src/net/wifi.rs | 15 +- sdk/base/lib/osBindings/IpInfo.ts | 6 +- .../lib/osBindings/NetworkInterfaceInfo.ts | 3 +- .../server-specs/server-specs.page.html | 6 +- .../ui/src/app/services/api/mock-patch.ts | 20 +- 11 files changed, 280 insertions(+), 171 deletions(-) diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 6cc81722e..972d0e8e8 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -159,18 +159,19 @@ pub struct ServerInfo { #[ts(export)] pub struct NetworkInterfaceInfo { pub public: Option, - pub scope_id: Option, - pub ip_info: IpInfo, + pub ip_info: Option, } impl NetworkInterfaceInfo { pub fn public(&self) -> bool { self.public.unwrap_or_else(|| { - !self.ip_info.0.iter().all(|ipnet| { - if let IpAddr::V4(ip4) = ipnet.addr() { - ip4.is_loopback() || ip4.is_private() || ip4.is_link_local() - } else { - true - } + !self.ip_info.as_ref().map_or(true, |ip_info| { + ip_info.subnets.iter().all(|ipnet| { + if let IpAddr::V4(ip4) = ipnet.addr() { + ip4.is_loopback() || ip4.is_private() || ip4.is_link_local() + } else { + true + } + }) }) }) } @@ -178,8 +179,13 @@ impl NetworkInterfaceInfo { #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, TS)] #[ts(export)] -#[ts(type = "string[]")] -pub struct IpInfo(pub BTreeSet); +#[serde(rename_all = "camelCase")] +pub struct IpInfo { + pub scope_id: u32, + #[ts(type = "string[]")] + pub subnets: BTreeSet, + pub wan_ip: Option, +} #[derive(Debug, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index 8bdff7ff2..b514ebbe8 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -419,7 +419,7 @@ pub async fn init( load_ca_cert.complete(); load_wifi.start(); - crate::net::wifi::synchronize_wpa_supplicant_conf( + crate::net::wifi::synchronize_network_manager( &cfg.datadir().join("main"), &mut server_info.wifi, ) diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 2d425cdfd..91e3e8e67 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -21,6 +21,7 @@ use crate::net::host::{host_for, Host, HostKind, Hosts}; use crate::net::network_interface::NetworkInterfaceController; use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname}; use crate::net::tor::TorController; +use crate::net::utils::ipv6_is_local; use crate::net::vhost::{AlpnInfo, VHostController}; use crate::prelude::*; use crate::util::serde::MaybeUtf8String; @@ -378,8 +379,16 @@ impl NetService { }; let mut bind_hostname_info: Vec = hostname_info.remove(port).unwrap_or_default(); - for (interface, iface_info) in &net_ifaces { - if !iface_info.public() { + for (interface, public, ip_info) in + net_ifaces.iter().filter_map(|(interface, info)| { + if let Some(ip_info) = &info.ip_info { + Some((interface, info.public(), ip_info)) + } else { + None + } + }) + { + if !public { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), public: false, @@ -410,10 +419,10 @@ impl NetService { ssl_port: Some(443), }, }); - } else if iface_info.public() && new_lan_bind.0.public { + } else if public && new_lan_bind.0.public { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), - public: iface_info.public(), + public, hostname: IpHostname::Domain { domain: address.clone(), subdomain: None, @@ -424,24 +433,37 @@ impl NetService { } } } - if !iface_info.public() || new_lan_bind.0.public { - for ipnet in &iface_info.ip_info.0 { + if !public || new_lan_bind.0.public { + if let Some(wan_ip) = ip_info.wan_ip.filter(|_| public) { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public, + hostname: IpHostname::Ipv4 { + value: wan_ip, + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } + for ipnet in &ip_info.subnets { match ipnet { IpNet::V4(net) => { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public: iface_info.public(), - hostname: IpHostname::Ipv4 { - value: net.addr(), - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, - }, - }); + if !public { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public, + hostname: IpHostname::Ipv4 { + value: net.addr(), + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } } IpNet::V6(net) => { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), - public: iface_info.public(), + public: public && !ipv6_is_local(net.addr()), hostname: IpHostname::Ipv6 { value: net.addr(), port: new_lan_bind.0.assigned_port, diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index f4624bcfa..6071875f3 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -1,13 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; -use std::net::{IpAddr, SocketAddr, SocketAddrV6}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV6}; use std::pin::Pin; use std::sync::{Arc, Weak}; use std::task::Poll; use clap::Parser; use futures::future::pending; -use futures::TryFutureExt; +use futures::{FutureExt, TryFutureExt}; use getifaddrs::if_nametoindex; use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; @@ -20,7 +20,7 @@ use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; use tokio_stream::StreamExt; use ts_rs::TS; -use zbus::proxy::PropertyStream; +use zbus::proxy::{PropertyStream, SignalStream}; use zbus::zvariant::{ DeserializeDict, OwnedObjectPath, OwnedValue, Type as ZType, Value as ZValue, }; @@ -29,6 +29,7 @@ use zbus::{proxy, Connection}; use crate::context::{CliContext, RpcContext}; use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; use crate::db::model::Database; +use crate::net::network_interface::active_connection::ActiveConnectionProxy; use crate::prelude::*; use crate::util::serde::{display_serializable, HandlerExtSerde}; use crate::util::sync::SyncMutex; @@ -52,18 +53,19 @@ pub fn network_interface_api() -> ParentHandler { table.add_row(row![ iface, info.public(), - info.ip_info - .0 - .into_iter() - .map(|ipnet| match ipnet.addr() { - IpAddr::V4(ip) => format!("{ip}/{}", ipnet.prefix_len()), - IpAddr::V6(ip) => format!( - "[{ip}{}]/{}", - info.scope_id.map(|s| format!("%{s}")).unwrap_or_default(), - ipnet.prefix_len() - ), - }) - .join(", ") + info.ip_info.map_or_else( + || "".to_owned(), + |ip_info| ip_info.subnets + .into_iter() + .map(|ipnet| match ipnet.addr() { + IpAddr::V4(ip) => format!("{ip}/{}", ipnet.prefix_len()), + IpAddr::V6(ip) => format!( + "[{ip}%{}]/{}", + ip_info.scope_id, + ipnet.prefix_len() + ), + }) + .join(", ")) ]); } @@ -140,17 +142,33 @@ async fn unset_public( trait NetworkManager { #[zbus(property)] fn devices(&self) -> Result, Error>; + + #[zbus(signal)] + fn device_added(&self) -> Result<(), Error>; + + #[zbus(signal)] + fn device_removed(&self) -> Result<(), Error>; } -#[proxy( - interface = "org.freedesktop.NetworkManager.Connection.Active", - default_service = "org.freedesktop.NetworkManager" -)] -trait ActiveConnection { - #[zbus(property)] - fn state_flags(&self) -> Result; - #[zbus(property, name = "Type")] - fn connection_type(&self) -> Result; +mod active_connection { + use zbus::proxy; + + use crate::prelude::*; + + #[proxy( + interface = "org.freedesktop.NetworkManager.Connection.Active", + default_service = "org.freedesktop.NetworkManager" + )] + pub trait ActiveConnection { + #[zbus(property)] + fn state_flags(&self) -> Result; + + #[zbus(property, name = "Type")] + fn connection_type(&self) -> Result; + + #[zbus(signal)] + fn state_changed(&self) -> Result<(), Error>; + } } #[proxy( @@ -177,17 +195,10 @@ struct AddressData { address: String, prefix: u32, } -impl TryFrom> for IpInfo { +impl TryFrom for IpNet { type Error = Error; - fn try_from(value: Vec) -> Result { - value - .into_iter() - .map(|a| { - IpNet::new(a.address.parse()?, a.prefix as u8).with_kind(ErrorKind::ParseNetAddress) - }) - .filter_ok(|ipnet| !ipnet.addr().is_unspecified() && !ipnet.addr().is_multicast()) - .collect::>() - .map(Self) + fn try_from(value: AddressData) -> Result { + IpNet::new(value.address.parse()?, value.prefix as u8).with_kind(ErrorKind::ParseNetAddress) } } @@ -210,31 +221,48 @@ trait Device { #[zbus(property)] fn ip6_config(&self) -> Result; + + #[zbus(signal)] + fn state_changed(&self) -> Result<(), Error>; } struct WatchPropertyStream<'a, T> { stream: PropertyStream<'a, T>, - last: T, + signals: Vec>, } impl<'a, T> WatchPropertyStream<'a, T> where T: Unpin + TryFrom, T::Error: Into, { - fn new(stream: PropertyStream<'a, T>, first: T) -> Self { + fn new(stream: PropertyStream<'a, T>) -> Self { Self { stream, - last: first, + signals: Vec::new(), } } + + fn with_signal(mut self, stream: SignalStream<'a>) -> Self { + self.signals.push(stream); + self + } + async fn until_changed>>( &mut self, fut: Fut, ) -> Result<(), Error> { let next = self.stream.next(); + let signal = if !self.signals.is_empty() { + futures::future::select_all(self.signals.iter_mut().map(|s| s.next().boxed())).boxed() + } else { + futures::future::pending().boxed() + }; tokio::select! { changed = next => { - self.last = changed.ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))?.get().await?; + changed.ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))?.get().await?; + Ok(()) + }, + _ = signal => { Ok(()) }, res = fut.and_then(|_| pending()) => { @@ -251,13 +279,13 @@ async fn watcher(write_to: watch::Sender Result, Error> { + Ok(reqwest::Client::builder() + .local_address(Some(IpAddr::V4(local_addr))) + .build()? + .get("http://ip4only.me/api/") + .send() + .await? + .error_for_status()? + .text() + .await? + .split(",") + .skip(1) + .next() + .filter(|s| s.is_empty()) + .map(|s| s.parse()) + .transpose()?) +} + #[instrument(skip_all)] async fn watch_ip( connection: &Connection, + active_connection_proxy: ActiveConnectionProxy<'_>, device_proxy: DeviceProxy<'_>, iface: InternedString, write_to: &watch::Sender>, ) -> Result<(), Error> { - let mut ip4_config_sub = WatchPropertyStream::new( - device_proxy.receive_ip4_config_changed().await, - device_proxy.ip4_config().await?, - ); - let mut ip6_config_sub = WatchPropertyStream::new( - device_proxy.receive_ip6_config_changed().await, - device_proxy.ip6_config().await?, - ); + let mut con_sub = + WatchPropertyStream::new(device_proxy.receive_active_connection_changed().await) + .with_signal(device_proxy.receive_state_changed().await?.into_inner()) + .with_signal( + active_connection_proxy + .receive_state_changed() + .await? + .into_inner(), + ); + let mut ip4_config_sub = + WatchPropertyStream::new(device_proxy.receive_ip4_config_changed().await); + let mut ip6_config_sub = + WatchPropertyStream::new(device_proxy.receive_ip6_config_changed().await); loop { - let ip4_config = ip4_config_sub.last.clone(); - let ip6_config = ip6_config_sub.last.clone(); + let ip4_config = device_proxy.ip4_config().await?; + let ip6_config = device_proxy.ip6_config().await?; ip4_config_sub .until_changed(ip6_config_sub.until_changed(async { let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config).await?; - let mut address4_sub = WatchPropertyStream::new( - ip4_proxy.receive_address_data_changed().await, - ip4_proxy.address_data().await?, - ); + let mut address4_sub = + WatchPropertyStream::new(ip4_proxy.receive_address_data_changed().await); let ip6_proxy = Ip6ConfigProxy::new(&connection, ip6_config).await?; - let mut address6_sub = WatchPropertyStream::new( - ip6_proxy.receive_address_data_changed().await, - ip6_proxy.address_data().await?, - ); + let mut address6_sub = + WatchPropertyStream::new(ip6_proxy.receive_address_data_changed().await); loop { - let addresses = address4_sub - .last - .iter() - .cloned() - .chain(address6_sub.last.iter().cloned()) + let addresses = ip4_proxy + .address_data() + .await? + .into_iter() + .chain(ip6_proxy.address_data().await?) .collect_vec(); address4_sub .until_changed(address6_sub.until_changed(async { - let ip_info: IpInfo = addresses.try_into()?; - let scope_id = - Some(if_nametoindex(&*iface).with_kind(ErrorKind::Network)?); + let scope_id = if_nametoindex(&*iface).with_kind(ErrorKind::Network)?; + let subnets: BTreeSet = + addresses.into_iter().map(TryInto::try_into).try_collect()?; + let ip_info = if !subnets.is_empty() { + let wan_ip = if let Some(local_addr) = + subnets.iter().find_map(|s| match s { + IpNet::V4(net) + if !net.addr().is_loopback() + && !net.addr().is_link_local() => + { + Some(net) + } + _ => None, + }) { + match get_wan_ipv4(local_addr.addr()).await { + Ok(a) => a, + Err(e) => { + tracing::error!( + "Failed to determine WAN IP for {iface}: {e}" + ); + tracing::debug!("{e:?}"); + None + } + } + } else { + None + }; + Some(IpInfo { + scope_id, + subnets, + wan_ip, + }) + } else { + None + }; write_to.send_if_modified(|m| { let public = m.get(&iface).map_or(None, |i| i.public); @@ -369,11 +450,10 @@ async fn watch_ip( iface.clone(), NetworkInterfaceInfo { public, - scope_id, ip_info: ip_info.clone(), }, ) - .filter(|old| &old.ip_info == &ip_info && old.scope_id == scope_id) + .filter(|old| &old.ip_info == &ip_info) .is_none() }); @@ -395,28 +475,13 @@ pub struct NetworkInterfaceController { impl NetworkInterfaceController { async fn sync( db: &TypedPatchDb, - ip_info: &BTreeMap, + info: &BTreeMap, ) -> Result<(), Error> { db.mutate(|db| { - let ifaces_model = db - .as_public_mut() + db.as_public_mut() .as_server_info_mut() - .as_network_interfaces_mut(); - let mut keep = BTreeSet::new(); - for (iface, ip_info) in ip_info { - keep.insert(iface); - ifaces_model - .upsert(&iface, || Ok(NetworkInterfaceInfo::default()))? - .ser(&ip_info)?; - } - for iface in ifaces_model.keys()? { - if !keep.contains(&&iface) { - if let Some(info) = ifaces_model.as_idx_mut(&iface) { - info.as_ip_info_mut().ser(&IpInfo::default())?; - } - } - } - Ok(()) + .as_network_interfaces_mut() + .ser(info) }) .await?; Ok(()) @@ -435,7 +500,10 @@ impl NetworkInterfaceController { .as_network_interfaces() .de() { - Ok(info) => { + Ok(mut info) => { + for info in info.values_mut() { + info.ip_info = None; + } write_to.send_replace(info); } Err(e) => { @@ -517,7 +585,7 @@ impl NetworkInterfaceController { .public, public, ); - prev == public + prev != public }); if let Some(e) = err { return Err(e); @@ -548,27 +616,29 @@ impl ListenerMap { let mut keep = BTreeSet::<(IpAddr, u32)>::new(); for info in ip_info.values() { if public || !info.public() { - for ipnet in &info.ip_info.0 { - let scope_id = info.scope_id.unwrap_or(0); - let key = (ipnet.addr(), scope_id); - keep.insert(key); - if let Some((_, is_public)) = self.listeners.get_mut(&key) { - *is_public = info.public(); - continue; + if let Some(ip_info) = &info.ip_info { + for ipnet in &ip_info.subnets { + let key = (ipnet.addr(), ip_info.scope_id); + keep.insert(key); + if let Some((_, is_public)) = self.listeners.get_mut(&key) { + *is_public = info.public(); + continue; + } + self.listeners.insert( + key, + ( + TcpListener::bind(match ipnet.addr() { + IpAddr::V6(ip6) => { + SocketAddrV6::new(ip6, self.port, 0, ip_info.scope_id) + .into() + } + ip => SocketAddr::new(ip, self.port), + }) + .await?, + info.public(), + ), + ); } - self.listeners.insert( - key, - ( - TcpListener::bind(match ipnet.addr() { - IpAddr::V6(ip6) => { - SocketAddrV6::new(ip6, self.port, 0, scope_id).into() - } - ip => SocketAddr::new(ip, self.port), - }) - .await?, - info.public(), - ), - ); } } } @@ -605,6 +675,7 @@ impl NetworkInterfaceListener { loop { if self.needs_update { let ip_info = self.ip_info.borrow().clone(); + dbg!("changed", &ip_info); self.listeners.update(&ip_info, public).await?; self.needs_update = false; } diff --git a/core/startos/src/net/utils.rs b/core/startos/src/net/utils.rs index 9cba8a0cd..2505611da 100644 --- a/core/startos/src/net/utils.rs +++ b/core/startos/src/net/utils.rs @@ -12,6 +12,10 @@ use tokio::process::Command; use crate::util::Invoke; use crate::Error; +pub fn ipv6_is_local(addr: Ipv6Addr) -> bool { + (addr.segments()[0] & 0xfe00) == 0xfc00 || (addr.segments()[0] & 0xffc0) == 0xfe80 +} + fn parse_iface_ip(output: &str) -> Result, Error> { let output = output.trim(); if output.is_empty() { diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs index 9f662f790..798471219 100644 --- a/core/startos/src/net/vhost.rs +++ b/core/startos/src/net/vhost.rs @@ -112,7 +112,7 @@ impl VHostController { } } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] struct TargetInfo { public: bool, addr: SocketAddr, @@ -280,7 +280,7 @@ impl VHostServer { }) .map(|(target, _)| target.clone()) }); - if let Some(target) = target { + if let Some(target) = dbg!(target) { if is_public && !target.public { log::warn!("Rejecting connection from public interface to private bind"); return Ok(()); diff --git a/core/startos/src/net/wifi.rs b/core/startos/src/net/wifi.rs index 056a403de..5c879399c 100644 --- a/core/startos/src/net/wifi.rs +++ b/core/startos/src/net/wifi.rs @@ -897,32 +897,29 @@ impl TypedValueParser for CountryCodeParser { } #[instrument(skip_all)] -pub async fn synchronize_wpa_supplicant_conf>( +pub async fn synchronize_network_manager>( main_datadir: P, wifi: &mut WifiInfo, ) -> Result<(), Error> { wifi.interface = find_wifi_iface().await?; - let Some(wifi_iface) = &wifi.interface else { - return Ok(()); - }; let persistent = main_datadir.as_ref().join("system-connections"); - tracing::debug!("persistent: {:?}", persistent); - // let supplicant = Path::new("/etc/wpa_supplicant.conf"); if tokio::fs::metadata(&persistent).await.is_err() { tokio::fs::create_dir_all(&persistent).await?; } crate::disk::mount::util::bind(&persistent, "/etc/NetworkManager/system-connections", false) .await?; - // if tokio::fs::metadata(&supplicant).await.is_err() { - // tokio::fs::write(&supplicant, include_str!("wpa_supplicant.conf.base")).await?; - // } Command::new("systemctl") .arg("restart") .arg("NetworkManager") .invoke(ErrorKind::Wifi) .await?; + + let Some(wifi_iface) = &wifi.interface else { + return Ok(()); + }; + Command::new("ifconfig") .arg(wifi_iface) .arg("up") diff --git a/sdk/base/lib/osBindings/IpInfo.ts b/sdk/base/lib/osBindings/IpInfo.ts index 184e72ddf..0933b80e6 100644 --- a/sdk/base/lib/osBindings/IpInfo.ts +++ b/sdk/base/lib/osBindings/IpInfo.ts @@ -1,3 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type IpInfo = string[] +export type IpInfo = { + scopeId: number + subnets: string[] + wanIp: string | null +} diff --git a/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts index c9f82005d..796046b93 100644 --- a/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts +++ b/sdk/base/lib/osBindings/NetworkInterfaceInfo.ts @@ -3,6 +3,5 @@ import type { IpInfo } from "./IpInfo" export type NetworkInterfaceInfo = { public: boolean | null - scopeId: number | null - ipInfo: IpInfo + ipInfo: IpInfo | null } diff --git a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html index f685b3b32..855e757ef 100644 --- a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html +++ b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html @@ -59,12 +59,12 @@

LAN

- - + +

{{ iface.key }} ({{ ipAddr.includes("::") ? "IPv6" : "IPv4" }})

-

{{ ipAddr.includes("::") ? "[" + ipAddr + (iface.value.scopeId ? "%" + iface.value.scopeId : "") + "]" : ipAddr }}

+

{{ ipAddr.includes("::") ? "[" + ipAddr + "%" + iface.value.ipInfo.scopeId + "]" : ipAddr }}

 diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index 3779f350a..9fd777da8 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -44,16 +44,22 @@ export const mockPatchData: DataModel = { networkInterfaces: { eth0: { public: false, - scopeId: 1, - ipInfo: ['10.0.0.1/24'], + ipInfo: { + scopeId: 1, + subnets: ['10.0.0.1/24'], + wanIp: null, + }, }, wlan0: { public: false, - scopeId: 2, - ipInfo: [ - '10.0.90.12/24', - 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64', - ], + ipInfo: { + scopeId: 2, + subnets: [ + '10.0.90.12/24', + 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64', + ], + wanIp: null, + }, }, }, acme: null, From 1c90ddb4aaee124a577c9a1d48b0da599dc5b4c5 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 10 Dec 2024 18:23:33 -0700 Subject: [PATCH 16/29] fixes --- core/startos/src/auth.rs | 3 +- core/startos/src/bins/container_cli.rs | 4 +- core/startos/src/bins/registry.rs | 4 +- core/startos/src/bins/start_cli.rs | 5 +- core/startos/src/bins/start_init.rs | 1 + core/startos/src/bins/startd.rs | 10 +- core/startos/src/net/network_interface.rs | 153 +++++++++--------- core/startos/src/prelude.rs | 9 ++ core/startos/src/util/future.rs | 62 ++++++- core/startos/src/util/io.rs | 17 +- core/startos/src/util/logger.rs | 66 +++++++- debian/postinst | 2 + patch-db | 2 +- .../lib/dependencies/setupDependencies.ts | 2 +- 14 files changed, 241 insertions(+), 99 deletions(-) diff --git a/core/startos/src/auth.rs b/core/startos/src/auth.rs index a6b624b70..9085709ab 100644 --- a/core/startos/src/auth.rs +++ b/core/startos/src/auth.rs @@ -187,9 +187,8 @@ pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<( Ok(()) } -#[derive(Deserialize, Serialize, Parser, TS)] +#[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -#[command(rename_all = "kebab-case")] #[ts(export)] pub struct LoginParams { password: Option, diff --git a/core/startos/src/bins/container_cli.rs b/core/startos/src/bins/container_cli.rs index b0da1cb00..118133f55 100644 --- a/core/startos/src/bins/container_cli.rs +++ b/core/startos/src/bins/container_cli.rs @@ -4,7 +4,7 @@ use rpc_toolkit::CliApp; use serde_json::Value; use crate::service::cli::{ContainerCliContext, ContainerClientConfig}; -use crate::util::logger::EmbassyLogger; +use crate::util::logger::LOGGER; use crate::version::{Current, VersionT}; lazy_static::lazy_static! { @@ -12,7 +12,7 @@ lazy_static::lazy_static! { } pub fn main(args: impl IntoIterator) { - EmbassyLogger::init(); + LOGGER.enable(); if let Err(e) = CliApp::new( |cfg: ContainerClientConfig| Ok(ContainerCliContext::init(cfg)), crate::service::effects::handler(), diff --git a/core/startos/src/bins/registry.rs b/core/startos/src/bins/registry.rs index 132e0984a..a455167fc 100644 --- a/core/startos/src/bins/registry.rs +++ b/core/startos/src/bins/registry.rs @@ -8,7 +8,7 @@ use tracing::instrument; use crate::net::web_server::WebServer; use crate::prelude::*; use crate::registry::context::{RegistryConfig, RegistryContext}; -use crate::util::logger::EmbassyLogger; +use crate::util::logger::LOGGER; #[instrument(skip_all)] async fn inner_main(config: &RegistryConfig) -> Result<(), Error> { @@ -63,7 +63,7 @@ async fn inner_main(config: &RegistryConfig) -> Result<(), Error> { } pub fn main(args: impl IntoIterator) { - EmbassyLogger::init(); + LOGGER.enable(); let config = RegistryConfig::parse_from(args).load().unwrap(); diff --git a/core/startos/src/bins/start_cli.rs b/core/startos/src/bins/start_cli.rs index 2e92e0cc0..bda5e00d3 100644 --- a/core/startos/src/bins/start_cli.rs +++ b/core/startos/src/bins/start_cli.rs @@ -5,7 +5,7 @@ use serde_json::Value; use crate::context::config::ClientConfig; use crate::context::CliContext; -use crate::util::logger::EmbassyLogger; +use crate::util::logger::LOGGER; use crate::version::{Current, VersionT}; lazy_static::lazy_static! { @@ -13,7 +13,8 @@ lazy_static::lazy_static! { } pub fn main(args: impl IntoIterator) { - EmbassyLogger::init(); + LOGGER.enable(); + if let Err(e) = CliApp::new( |cfg: ClientConfig| Ok(CliContext::init(cfg.load()?)?), crate::expanded_api(), diff --git a/core/startos/src/bins/start_init.rs b/core/startos/src/bins/start_init.rs index 394d42c8d..fc65feae9 100644 --- a/core/startos/src/bins/start_init.rs +++ b/core/startos/src/bins/start_init.rs @@ -178,6 +178,7 @@ async fn setup_or_init( tracing::info!("Loaded Disk"); if requires_reboot.0 { + tracing::info!("Rebooting..."); let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1)); reboot_phase.start(); return Ok(Err(Shutdown { diff --git a/core/startos/src/bins/startd.rs b/core/startos/src/bins/startd.rs index d383f3091..e93857c38 100644 --- a/core/startos/src/bins/startd.rs +++ b/core/startos/src/bins/startd.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use clap::Parser; use color_eyre::eyre::eyre; use futures::{FutureExt, TryFutureExt}; +use tokio::fs::OpenOptions; use tokio::signal::unix::signal; use tracing::instrument; @@ -15,7 +16,8 @@ use crate::context::{DiagnosticContext, InitContext, RpcContext}; use crate::net::web_server::WebServer; use crate::shutdown::Shutdown; use crate::system::launch_metrics_task; -use crate::util::logger::EmbassyLogger; +use crate::util::io::append_file; +use crate::util::logger::LOGGER; use crate::{Error, ErrorKind, ResultExt}; #[instrument(skip_all)] @@ -27,6 +29,9 @@ async fn inner_main( .await .is_ok() { + LOGGER.set_logfile(Some( + append_file("/run/startos/init.log").await?.into_std().await, + )); let (ctx, handle) = match super::start_init::main(server, &config).await? { Err(s) => return Ok(Some(s)), Ok(ctx) => ctx, @@ -34,6 +39,7 @@ async fn inner_main( tokio::fs::write("/run/startos/initialized", "").await?; server.serve_main(ctx.clone()); + LOGGER.set_logfile(None); handle.complete(); ctx @@ -131,7 +137,7 @@ async fn inner_main( } pub fn main(args: impl IntoIterator) { - EmbassyLogger::init(); + LOGGER.enable(); let config = ServerConfig::parse_from(args).load().unwrap(); diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 6071875f3..dfb1f34b2 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -4,10 +4,10 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV6}; use std::pin::Pin; use std::sync::{Arc, Weak}; use std::task::Poll; +use std::time::Duration; use clap::Parser; -use futures::future::pending; -use futures::{FutureExt, TryFutureExt}; +use futures::{Stream, StreamExt, TryStreamExt}; use getifaddrs::if_nametoindex; use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; @@ -18,9 +18,8 @@ use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler use serde::{Deserialize, Serialize}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; -use tokio_stream::StreamExt; use ts_rs::TS; -use zbus::proxy::{PropertyStream, SignalStream}; +use zbus::proxy::{PropertyChanged, PropertyStream, SignalStream}; use zbus::zvariant::{ DeserializeDict, OwnedObjectPath, OwnedValue, Type as ZType, Value as ZValue, }; @@ -31,6 +30,7 @@ use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; use crate::db::model::Database; use crate::net::network_interface::active_connection::ActiveConnectionProxy; use crate::prelude::*; +use crate::util::future::Until; use crate::util::serde::{display_serializable, HandlerExtSerde}; use crate::util::sync::SyncMutex; @@ -222,53 +222,31 @@ trait Device { #[zbus(property)] fn ip6_config(&self) -> Result; + #[zbus(property, name = "State")] + fn _state(&self) -> Result; + #[zbus(signal)] fn state_changed(&self) -> Result<(), Error>; } -struct WatchPropertyStream<'a, T> { - stream: PropertyStream<'a, T>, - signals: Vec>, +trait StubStream<'a> { + fn stub(self) -> impl Stream> + 'a; } -impl<'a, T> WatchPropertyStream<'a, T> +impl<'a, T> StubStream<'a> for PropertyStream<'a, T> where - T: Unpin + TryFrom, + T: Unpin + TryFrom + std::fmt::Debug + 'a, T::Error: Into, { - fn new(stream: PropertyStream<'a, T>) -> Self { - Self { - stream, - signals: Vec::new(), - } - } - - fn with_signal(mut self, stream: SignalStream<'a>) -> Self { - self.signals.push(stream); - self + fn stub(self) -> impl Stream> + 'a { + StreamExt::then(self, |d| async move { + PropertyChanged::get(&d).await.map(|_| ()) + }) + .map_err(Error::from) } - - async fn until_changed>>( - &mut self, - fut: Fut, - ) -> Result<(), Error> { - let next = self.stream.next(); - let signal = if !self.signals.is_empty() { - futures::future::select_all(self.signals.iter_mut().map(|s| s.next().boxed())).boxed() - } else { - futures::future::pending().boxed() - }; - tokio::select! { - changed = next => { - changed.ok_or_else(|| Error::new(eyre!("stream is empty"), ErrorKind::DBus))?.get().await?; - Ok(()) - }, - _ = signal => { - Ok(()) - }, - res = fut.and_then(|_| pending()) => { - res - } - } +} +impl<'a> StubStream<'a> for SignalStream<'a> { + fn stub(self) -> impl Stream> + 'a { + self.map(|_| Ok(())) } } @@ -277,17 +255,30 @@ async fn watcher(write_to: watch::Sender = async { let connection = Connection::system().await?; + let netman_proxy = NetworkManagerProxy::new(&connection).await?; - let mut devices_sub = - WatchPropertyStream::new(netman_proxy.receive_devices_changed().await) - .with_signal(netman_proxy.receive_device_added().await?.into_inner()) - .with_signal(netman_proxy.receive_device_removed().await?.into_inner()); + let mut until = Until::new() + .with_stream(netman_proxy.receive_devices_changed().await.stub()) + .with_stream( + netman_proxy + .receive_device_added() + .await? + .into_inner() + .stub(), + ) + .with_stream( + netman_proxy + .receive_device_removed() + .await? + .into_inner() + .stub(), + ); loop { let devices = netman_proxy.devices().await?; - devices_sub - .until_changed(async { + until + .run(async { let mut ifaces = BTreeSet::new(); let mut jobs = Vec::new(); for device in devices { @@ -352,6 +343,7 @@ async fn get_wan_ipv4(local_addr: Ipv4Addr) -> Result, Error> { .local_address(Some(IpAddr::V4(local_addr))) .build()? .get("http://ip4only.me/api/") + .timeout(Duration::from_secs(10)) .send() .await? .error_for_status()? @@ -373,31 +365,40 @@ async fn watch_ip( iface: InternedString, write_to: &watch::Sender>, ) -> Result<(), Error> { - let mut con_sub = - WatchPropertyStream::new(device_proxy.receive_active_connection_changed().await) - .with_signal(device_proxy.receive_state_changed().await?.into_inner()) - .with_signal( - active_connection_proxy - .receive_state_changed() - .await? - .into_inner(), - ); - let mut ip4_config_sub = - WatchPropertyStream::new(device_proxy.receive_ip4_config_changed().await); - let mut ip6_config_sub = - WatchPropertyStream::new(device_proxy.receive_ip6_config_changed().await); + let mut until = Until::new() + .with_stream( + device_proxy + .receive_active_connection_changed() + .await + .stub(), + ) + .with_stream( + device_proxy + .receive_state_changed() + .await? + .into_inner() + .stub(), + ) + .with_stream( + active_connection_proxy + .receive_state_changed() + .await? + .into_inner() + .stub(), + ) + .with_stream(device_proxy.receive_ip4_config_changed().await.stub()) + .with_stream(device_proxy.receive_ip6_config_changed().await.stub()); loop { let ip4_config = device_proxy.ip4_config().await?; let ip6_config = device_proxy.ip6_config().await?; - ip4_config_sub - .until_changed(ip6_config_sub.until_changed(async { + until + .run(async { let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config).await?; - let mut address4_sub = - WatchPropertyStream::new(ip4_proxy.receive_address_data_changed().await); let ip6_proxy = Ip6ConfigProxy::new(&connection, ip6_config).await?; - let mut address6_sub = - WatchPropertyStream::new(ip6_proxy.receive_address_data_changed().await); + let mut until = Until::new() + .with_stream(ip4_proxy.receive_address_data_changed().await.stub()) + .with_stream(ip6_proxy.receive_address_data_changed().await.stub()); loop { let addresses = ip4_proxy @@ -406,8 +407,11 @@ async fn watch_ip( .into_iter() .chain(ip6_proxy.address_data().await?) .collect_vec(); - address4_sub - .until_changed(address6_sub.until_changed(async { + if iface == "enp1s0" { + dbg!(&addresses); + } + until + .run(async { let scope_id = if_nametoindex(&*iface).with_kind(ErrorKind::Network)?; let subnets: BTreeSet = addresses.into_iter().map(TryInto::try_into).try_collect()?; @@ -448,20 +452,20 @@ async fn watch_ip( let public = m.get(&iface).map_or(None, |i| i.public); m.insert( iface.clone(), - NetworkInterfaceInfo { + dbg!(NetworkInterfaceInfo { public, ip_info: ip_info.clone(), - }, + }), ) - .filter(|old| &old.ip_info == &ip_info) + .filter(|old| &dbg!(old).ip_info == &ip_info) .is_none() }); Ok::<_, Error>(()) - })) + }) .await?; } - })) + }) .await?; } } @@ -675,7 +679,6 @@ impl NetworkInterfaceListener { loop { if self.needs_update { let ip_info = self.ip_info.borrow().clone(); - dbg!("changed", &ip_info); self.listeners.update(&ip_info, public).await?; self.needs_update = false; } diff --git a/core/startos/src/prelude.rs b/core/startos/src/prelude.rs index dddc1ecda..a6a78a58d 100644 --- a/core/startos/src/prelude.rs +++ b/core/startos/src/prelude.rs @@ -6,3 +6,12 @@ pub use tracing::instrument; pub use crate::db::prelude::*; pub use crate::ensure_code; pub use crate::error::{Error, ErrorCollection, ErrorKind, ResultExt}; + +#[macro_export] +macro_rules! dbg { + ($e:expr) => {{ + let e = $e; + tracing::debug!("[{}:{}:{}] $e = {e:?}", file!(), line!(), column!()); + e + }}; +} diff --git a/core/startos/src/util/future.rs b/core/startos/src/util/future.rs index f40e847bf..c690f9754 100644 --- a/core/startos/src/util/future.rs +++ b/core/startos/src/util/future.rs @@ -1,11 +1,13 @@ use std::pin::Pin; use std::task::{Context, Poll}; -use futures::future::abortable; -use futures::stream::{AbortHandle, Abortable}; -use futures::Future; +use futures::future::{abortable, pending, BoxFuture, FusedFuture}; +use futures::stream::{AbortHandle, Abortable, BoxStream}; +use futures::{Future, FutureExt, Stream, StreamExt}; use tokio::sync::watch; +use crate::prelude::*; + #[pin_project::pin_project(PinnedDrop)] pub struct DropSignaling { #[pin] @@ -102,6 +104,60 @@ impl CancellationHandle { } } +#[derive(Default)] +pub struct Until<'a> { + streams: Vec>>, + async_fns: Vec BoxFuture<'a, Result<(), Error>> + Send + 'a>>, +} +impl<'a> Until<'a> { + pub fn new() -> Self { + Self::default() + } + + pub fn with_stream( + mut self, + stream: impl Stream> + Send + 'a, + ) -> Self { + self.streams.push(stream.boxed()); + self + } + + pub fn with_async_fn(mut self, mut f: F) -> Self + where + F: FnMut() -> Fut + Send + 'a, + Fut: Future> + FusedFuture + Send + 'a, + { + self.async_fns.push(Box::new(move || f().boxed())); + self + } + + pub async fn run> + Send>( + &mut self, + fut: Fut, + ) -> Result<(), Error> { + let (res, _, _) = futures::future::select_all( + self.streams + .iter_mut() + .map(|s| { + async { + s.next().await.transpose()?.ok_or_else(|| { + Error::new(eyre!("stream is empty"), ErrorKind::Cancelled) + }) + } + .boxed() + }) + .chain(self.async_fns.iter_mut().map(|f| f())) + .chain([async { + fut.await?; + pending().await + } + .boxed()]), + ) + .await; + res + } +} + #[tokio::test] async fn test_cancellable() { use std::sync::Arc; diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs index 8e270c912..9018b3344 100644 --- a/core/startos/src/util/io.rs +++ b/core/startos/src/util/io.rs @@ -15,7 +15,7 @@ use futures::future::{BoxFuture, Fuse}; use futures::{AsyncSeek, FutureExt, Stream, TryStreamExt}; use helpers::NonDetachingJoinHandle; use nix::unistd::{Gid, Uid}; -use tokio::fs::File; +use tokio::fs::{File, OpenOptions}; use tokio::io::{ duplex, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf, WriteHalf, }; @@ -935,6 +935,21 @@ pub async fn create_file(path: impl AsRef) -> Result { .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("create {path:?}"))) } +pub async fn append_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent) + .await + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("mkdir -p {parent:?}")))?; + } + OpenOptions::new() + .create(true) + .append(true) + .open(path) + .await + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("create {path:?}"))) +} + pub async fn delete_file(path: impl AsRef) -> Result<(), Error> { let path = path.as_ref(); tokio::fs::remove_file(path) diff --git a/core/startos/src/util/logger.rs b/core/startos/src/util/logger.rs index c464b328d..816721d5b 100644 --- a/core/startos/src/util/logger.rs +++ b/core/startos/src/util/logger.rs @@ -1,13 +1,62 @@ -use std::io; +use std::fs::File; +use std::io::{self, Write}; +use std::sync::{Arc, Mutex, MutexGuard}; +use lazy_static::lazy_static; use tracing::Subscriber; +use tracing_subscriber::fmt::MakeWriter; use tracing_subscriber::util::SubscriberInitExt; +lazy_static! { + pub static ref LOGGER: StartOSLogger = StartOSLogger::init(); +} + #[derive(Clone)] -pub struct EmbassyLogger {} +pub struct StartOSLogger { + logfile: LogFile, +} + +#[derive(Clone, Default)] +struct LogFile(Arc>>); +impl<'a> MakeWriter<'a> for LogFile { + type Writer = Box; + fn make_writer(&'a self) -> Self::Writer { + let f = self.0.lock().unwrap(); + if f.is_some() { + struct TeeWriter<'a>(MutexGuard<'a, Option>); + impl<'a> Write for TeeWriter<'a> { + fn write(&mut self, buf: &[u8]) -> io::Result { + let n = if let Some(f) = &mut *self.0 { + f.write(buf)? + } else { + buf.len() + }; + io::stderr().write_all(&buf[..n])?; + Ok(n) + } + fn flush(&mut self) -> io::Result<()> { + if let Some(f) = &mut *self.0 { + f.flush()?; + } + Ok(()) + } + } + Box::new(TeeWriter(f)) + } else { + drop(f); + Box::new(io::stderr()) + } + } +} + +impl StartOSLogger { + pub fn enable(&self) {} + + pub fn set_logfile(&self, logfile: Option) { + *self.logfile.0.lock().unwrap() = logfile; + } -impl EmbassyLogger { - fn base_subscriber() -> impl Subscriber { + fn base_subscriber(logfile: LogFile) -> impl Subscriber { use tracing_error::ErrorLayer; use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; @@ -24,7 +73,7 @@ impl EmbassyLogger { .add_directive("tokio=trace".parse().unwrap()) .add_directive("runtime=trace".parse().unwrap()); let fmt_layer = fmt::layer() - .with_writer(io::stderr) + .with_writer(logfile) .with_line_number(true) .with_file(true) .with_target(true); @@ -39,11 +88,12 @@ impl EmbassyLogger { sub } - pub fn init() -> Self { - Self::base_subscriber().init(); + fn init() -> Self { + let logfile = LogFile::default(); + Self::base_subscriber(logfile.clone()).init(); color_eyre::install().unwrap_or_else(|_| tracing::warn!("tracing too many times")); - EmbassyLogger {} + StartOSLogger { logfile } } } diff --git a/debian/postinst b/debian/postinst index 3714df8d4..176bdb6b2 100755 --- a/debian/postinst +++ b/debian/postinst @@ -86,6 +86,8 @@ sed -i '/^\s*#\?\s*issue_discards\s*=\s*/c\issue_discards = 1' /etc/lvm/lvm.conf sed -i '/\(^\|#\)\s*unqualified-search-registries\s*=\s*/c\unqualified-search-registries = ["docker.io"]' /etc/containers/registries.conf sed -i 's/\(#\|\^\)\s*\([^=]\+\)=\(suspend\|hibernate\)\s*$/\2=ignore/g' /etc/systemd/logind.conf sed -i '/\(^\|#\)MulticastDNS=/c\MulticastDNS=no' /etc/systemd/resolved.conf +sed -i 's/\[Service\]/[Service]\nEnvironment=SYSTEMD_LOG_LEVEL=debug/' /lib/systemd/system/systemd-timesyncd.service +sed -i '/\(^\|#\)RootDistanceMaxSec=/c\RootDistanceMaxSec=10' /etc/systemd/timesyncd.conf mkdir -p /etc/nginx/ssl diff --git a/patch-db b/patch-db index 99076d349..2600a784a 160000 --- a/patch-db +++ b/patch-db @@ -1 +1 @@ -Subproject commit 99076d349c6768000483ea8d47216d273586552e +Subproject commit 2600a784a9899a6f8e0c9abe0bf4c4ce48cb85a9 diff --git a/sdk/base/lib/dependencies/setupDependencies.ts b/sdk/base/lib/dependencies/setupDependencies.ts index 6b15ef0d1..710ec96ed 100644 --- a/sdk/base/lib/dependencies/setupDependencies.ts +++ b/sdk/base/lib/dependencies/setupDependencies.ts @@ -51,7 +51,7 @@ export function setupDependencies( dependencies: Object.entries(dependencyType).map( ([id, { versionRange, ...x }, ,]) => ({ - // id, + id, ...x, versionRange: versionRange.toString(), }) as T.DependencyRequirement, From a1b4311ab4c4175d32266c02ec2851e77a3a7fc7 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Fri, 13 Dec 2024 15:37:28 -0700 Subject: [PATCH 17/29] individualized acme and privacy settings for domains and bindings --- core/Cargo.lock | 548 +++++++++--------- core/startos/Cargo.toml | 2 +- core/startos/src/db/model/public.rs | 23 +- core/startos/src/net/acme.rs | 174 ++---- core/startos/src/net/host/address.rs | 321 ++++++++-- core/startos/src/net/host/binding.rs | 126 +++- core/startos/src/net/host/mod.rs | 162 ++---- core/startos/src/net/net_controller.rs | 125 ++-- core/startos/src/net/network_interface.rs | 302 ++++++++-- core/startos/src/net/ssl.rs | 1 - core/startos/src/net/static_server.rs | 3 +- core/startos/src/net/vhost.rs | 331 ++++++----- core/startos/src/net/wifi.rs | 6 +- core/startos/src/prelude.rs | 10 +- core/startos/src/registry/context.rs | 1 - core/startos/src/registry/mod.rs | 3 +- .../signer/commitment/merkle_archive.rs | 4 +- core/startos/src/service/effects/callbacks.rs | 2 +- core/startos/src/service/effects/net/ssl.rs | 20 +- core/startos/src/service/effects/store.rs | 2 +- core/startos/src/service/mod.rs | 2 - .../src/service/persistent_container.rs | 2 +- core/startos/src/service/rpc.rs | 2 +- .../startos/src/service/transition/restore.rs | 2 +- core/startos/src/util/rpc_client.rs | 6 +- core/startos/src/version/v0_3_6_alpha_0.rs | 2 - .../{HostAddress.ts => AcmeProvider.ts} | 4 +- sdk/base/lib/osBindings/AcmeSettings.ts | 12 +- sdk/base/lib/osBindings/DomainConfig.ts | 4 + .../lib/osBindings/ForgetInterfaceParams.ts | 3 + sdk/base/lib/osBindings/Host.ts | 5 +- sdk/base/lib/osBindings/IpInfo.ts | 1 + sdk/base/lib/osBindings/ServerInfo.ts | 3 +- sdk/base/lib/osBindings/index.ts | 4 +- .../ui/src/app/services/api/mock-patch.ts | 2 + 35 files changed, 1322 insertions(+), 898 deletions(-) rename sdk/base/lib/osBindings/{HostAddress.ts => AcmeProvider.ts} (51%) create mode 100644 sdk/base/lib/osBindings/DomainConfig.ts create mode 100644 sdk/base/lib/osBindings/ForgetInterfaceParams.ts diff --git a/core/Cargo.lock b/core/Cargo.lock index 16a0a0e7b..4a7914999 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -162,9 +162,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arrayref" @@ -205,7 +205,7 @@ dependencies = [ "nom 7.1.3", "num-traits", "rusticata-macros", - "thiserror", + "thiserror 1.0.69", "time", ] @@ -217,7 +217,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "synstructure", ] @@ -229,13 +229,13 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "async-acme" -version = "0.5.0" -source = "git+https://github.com/dr-bonez/async-acme.git#b9ff31ad900adc9086c0d1437ce51661d30856d2" +version = "0.6.0" +source = "git+https://github.com/dr-bonez/async-acme.git#0ddf25152237b5fc1726d977a7931e44513ce309" dependencies = [ "async-trait", "base64 0.22.1", @@ -245,11 +245,11 @@ dependencies = [ "pem", "rcgen", "ring", - "rustls 0.23.17", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", "tokio", "x509-parser", ] @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ "brotli", "flate2", @@ -384,7 +384,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -424,7 +424,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -441,7 +441,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -467,21 +467,20 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7c2840b66236045acd2607d5866e274380afd87ef99d6226e961e2cb47df45" +checksum = "f47bb8cc16b669d267eeccf585aea077d0882f4777b1c1f740217885d6e6e5a3" dependencies = [ "aws-lc-sys", - "mirai-annotations", "paste", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.23.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad3a619a9de81e1d7de1f1186dcba4506ed661a0e483d84410fdef0ee87b2f96" +checksum = "a2101df3813227bbaaaa0b04cd61c534c7954b22bd68d399b440be937dc63ff7" dependencies = [ "bindgen", "cc", @@ -531,7 +530,7 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -548,10 +547,10 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "sha1", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-tungstenite 0.24.0", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -583,13 +582,13 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -603,7 +602,7 @@ checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -623,7 +622,7 @@ dependencies = [ "deku", "flate2", "rustc-hash", - "thiserror", + "thiserror 1.0.69", "tracing", "xz2", "zstd", @@ -731,7 +730,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.87", + "syn 2.0.90", "which", ] @@ -808,9 +807,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" dependencies = [ "arrayref", "arrayvec 0.7.6", @@ -899,9 +898,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cache-padded" @@ -911,9 +910,9 @@ checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" [[package]] name = "cc" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -943,9 +942,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1024,9 +1023,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -1034,9 +1033,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -1053,20 +1052,20 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" dependencies = [ "cc", ] @@ -1267,9 +1266,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -1462,7 +1461,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1486,7 +1485,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1497,7 +1496,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1528,7 +1527,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1565,7 +1564,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1588,7 +1587,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1641,7 +1640,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1836,7 +1835,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1857,7 +1856,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1879,12 +1878,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1927,9 +1926,9 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ "event-listener 5.3.1", "pin-project-lite", @@ -1965,9 +1964,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fd-lock-rs" @@ -2170,7 +2169,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -2241,7 +2240,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-rustls 0.25.0", - "webpki-roots 0.26.6", + "webpki-roots 0.26.7", ] [[package]] @@ -2324,7 +2323,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.6.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -2342,8 +2341,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", - "indexmap 2.6.0", + "http 1.2.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -2393,9 +2392,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "hashlink" @@ -2515,9 +2514,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -2542,7 +2541,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -2553,7 +2552,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] @@ -2610,7 +2609,7 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "httparse", "httpdate", @@ -2628,13 +2627,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.1.0", + "http 1.2.0", "hyper 1.5.1", "hyper-util", - "rustls 0.23.17", + "rustls 0.23.20", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tower-service", ] @@ -2675,7 +2674,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "hyper 1.5.1", "pin-project-lite", @@ -2823,7 +2822,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -2908,8 +2907,8 @@ dependencies = [ [[package]] name = "imbl-value" -version = "0.1.0" -source = "git+https://github.com/Start9Labs/imbl-value.git#3ce01b17ae5e756fc829ee5e3513a1b19b2a03fc" +version = "0.1.1" +source = "git+https://github.com/Start9Labs/imbl-value.git#1900943e17116def03bf00bff05cf12e54d810bc" dependencies = [ "imbl", "serde", @@ -2956,12 +2955,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", "serde", ] @@ -3041,7 +3040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ea1dc4bf0fb4904ba83ffdb98af3d9c325274e92e6e295e4151e86c96363e04" dependencies = [ "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -3091,9 +3090,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jaq-core" @@ -3156,16 +3155,17 @@ dependencies = [ "regex", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", "time", ] [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -3186,7 +3186,7 @@ dependencies = [ "imbl", "imbl-value", "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -3286,15 +3286,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.164" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -3314,7 +3314,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall 0.5.7", + "redox_syscall 0.5.8", ] [[package]] @@ -3342,9 +3342,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "litrs" @@ -3404,7 +3404,7 @@ dependencies = [ "bitvec 1.0.1", "serde", "serde-big-array", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -3491,23 +3491,16 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi 0.3.9", "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "models" version = "0.1.0" @@ -3531,7 +3524,7 @@ dependencies = [ "serde_json", "sqlx", "ssh-key", - "thiserror", + "thiserror 1.0.69", "tokio", "torut", "tracing", @@ -3778,7 +3771,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3827,7 +3820,7 @@ dependencies = [ "byteorder", "md-5", "sha2 0.10.8", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -3853,7 +3846,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3968,7 +3961,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.7", + "redox_syscall 0.5.8", "smallvec", "windows-targets 0.52.6", ] @@ -3995,7 +3988,7 @@ dependencies = [ "patch-db-macro", "serde", "serde_cbor", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", "tracing-error", @@ -4057,20 +4050,20 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror", + "thiserror 2.0.6", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" dependencies = [ "pest", "pest_generator", @@ -4078,22 +4071,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "pest_meta" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" dependencies = [ "once_cell", "pest", @@ -4107,7 +4100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.6.0", + "indexmap 2.7.0", ] [[package]] @@ -4142,7 +4135,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4212,9 +4205,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" [[package]] name = "powerfmt" @@ -4244,7 +4237,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4281,9 +4274,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -4342,7 +4335,7 @@ checksum = "6ff7ff745a347b87471d859a377a9a404361e7efc2a971d73424a6d183c0fc77" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4365,7 +4358,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4557,9 +4550,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags 2.6.0", ] @@ -4572,7 +4565,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom 0.2.15", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -4633,7 +4626,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -4652,7 +4645,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "system-configuration", "tokio", "tokio-native-tls", @@ -4725,7 +4718,7 @@ dependencies = [ "axum 0.7.9", "clap", "futures", - "http 1.1.0", + "http 1.2.0", "http-body-util", "imbl-value", "itertools 0.12.1", @@ -4736,7 +4729,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", "url", @@ -4745,9 +4738,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +checksum = "47c75d7c5c6b673e58bf54d8544a9f432e3a925b0e80f7cd3602ab5c50c55519" dependencies = [ "const-oid", "digest 0.10.7", @@ -4817,15 +4810,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", - "errno 0.3.9", + "errno 0.3.10", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4855,9 +4848,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.17" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "aws-lc-rs", "log", @@ -4935,18 +4928,18 @@ dependencies = [ [[package]] name = "rustyline-async" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9396d834c31f9fddd716e7c279e7cb70207092a1e59767918610f5c560c6eb" +checksum = "1b8a29112291cda41f18306ed8919c49360e5273328162445ca250aae37c8f89" dependencies = [ "crossterm", "futures-channel", "futures-util", "pin-project", "thingbuf", - "thiserror", + "thiserror 2.0.6", "unicode-segmentation", - "unicode-width 0.1.12", + "unicode-width 0.2.0", ] [[package]] @@ -5037,9 +5030,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] @@ -5063,13 +5056,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5078,7 +5071,7 @@ version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "memchr", "ryu", @@ -5103,7 +5096,7 @@ checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c" dependencies = [ "percent-encoding", "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -5114,7 +5107,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5148,7 +5141,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_derive", "serde_json", @@ -5165,7 +5158,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5174,7 +5167,7 @@ version = "0.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ce6afeda22f0b55dde2c34897bce76a629587348480384231205c14b59a01f" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "libyml", "log", @@ -5337,9 +5330,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -5409,7 +5402,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.6.0", + "indexmap 2.7.0", "log", "memchr", "once_cell", @@ -5422,7 +5415,7 @@ dependencies = [ "sha2 0.10.8", "smallvec", "sqlformat", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", "tracing", @@ -5507,7 +5500,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 1.0.69", "tracing", "whoami", ] @@ -5546,7 +5539,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 1.0.69", "tracing", "whoami", ] @@ -5598,7 +5591,7 @@ dependencies = [ "quote", "regex-syntax 0.6.29", "strsim 0.10.0", - "syn 2.0.87", + "syn 2.0.90", "unicode-width 0.1.12", ] @@ -5692,14 +5685,14 @@ dependencies = [ "helpers", "hex", "hmac", - "http 1.1.0", + "http 1.2.0", "http-body-util", "hyper-util", "id-pool", "imbl", "imbl-value", "include_dir", - "indexmap 2.6.0", + "indexmap 2.7.0", "indicatif", "integer-encoding", "ipnet", @@ -5760,9 +5753,9 @@ dependencies = [ "ssh-key", "tar", "textwrap", - "thiserror", + "thiserror 1.0.69", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-socks", "tokio-stream", "tokio-tar", @@ -5849,9 +5842,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -5866,9 +5859,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -5881,7 +5874,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5982,7 +5975,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +dependencies = [ + "thiserror-impl 2.0.6", ] [[package]] @@ -5993,7 +5995,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -6019,9 +6032,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -6040,9 +6053,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -6084,9 +6097,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -6119,7 +6132,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -6145,12 +6158,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.17", - "rustls-pki-types", + "rustls 0.23.20", "tokio", ] @@ -6162,15 +6174,15 @@ checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" dependencies = [ "either", "futures-util", - "thiserror", + "thiserror 1.0.69", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -6220,9 +6232,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -6270,7 +6282,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -6283,7 +6295,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -6358,14 +6370,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.2", "tokio", "tower-layer", "tower-service", @@ -6386,9 +6398,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -6398,20 +6410,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -6419,9 +6431,9 @@ dependencies = [ [[package]] name = "tracing-error" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" dependencies = [ "tracing", "tracing-subscriber", @@ -6439,9 +6451,9 @@ dependencies = [ [[package]] name = "tracing-journald" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba316a74e8fc3c3896a850dba2375928a9fa171b085ecddfc7c054d39970f3fd" +checksum = "fc0b4143302cf1022dac868d521e36e8b27691f72c84b3311750d5188ebba657" dependencies = [ "libc", "tracing-core", @@ -6461,9 +6473,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -6504,7 +6516,7 @@ dependencies = [ "once_cell", "rand 0.8.5", "smallvec", - "thiserror", + "thiserror 1.0.69", "tinyvec", "tokio", "tracing", @@ -6525,7 +6537,7 @@ dependencies = [ "futures-executor", "futures-util", "serde", - "thiserror", + "thiserror 1.0.69", "time", "tokio", "toml 0.7.8", @@ -6544,7 +6556,7 @@ name = "ts-rs" version = "8.1.0" source = "git+https://github.com/dr-bonez/ts-rs.git?branch=feature%2Ftop-level-as#7ae88ade90b5e724159048a663a0bdb04bed27f7" dependencies = [ - "thiserror", + "thiserror 1.0.69", "ts-rs-macros", ] @@ -6556,7 +6568,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "termcolor", ] @@ -6580,13 +6592,13 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.1.0", + "http 1.2.0", "httparse", "log", "native-tls", "rand 0.8.5", "sha1", - "thiserror", + "thiserror 1.0.69", "url", "utf-8", ] @@ -6600,12 +6612,12 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.1.0", + "http 1.2.0", "httparse", "log", "rand 0.8.5", "sha1", - "thiserror", + "thiserror 1.0.69", "utf-8", ] @@ -6626,7 +6638,7 @@ checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -6672,9 +6684,9 @@ checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-linebreak" @@ -6745,9 +6757,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna 1.0.3", @@ -6860,9 +6872,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -6871,36 +6883,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6908,22 +6920,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" @@ -6940,9 +6952,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -6966,9 +6978,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -6991,7 +7003,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.7", + "redox_syscall 0.5.8", "wasite", ] @@ -7271,7 +7283,7 @@ dependencies = [ "nom 7.1.3", "oid-registry", "rusticata-macros", - "thiserror", + "thiserror 1.0.69", "time", ] @@ -7323,7 +7335,7 @@ dependencies = [ "anyhow", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -7349,9 +7361,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -7361,13 +7373,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "synstructure", ] @@ -7416,7 +7428,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "zbus_names", "zvariant", "zvariant_utils", @@ -7452,27 +7464,27 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "synstructure", ] @@ -7493,7 +7505,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -7515,7 +7527,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -7570,7 +7582,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "zvariant_utils", ] @@ -7584,6 +7596,6 @@ dependencies = [ "quote", "serde", "static_assertions", - "syn 2.0.87", + "syn 2.0.90", "winnow 0.6.20", ] diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index a9d7bb1fc..7ee26dbca 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -50,7 +50,7 @@ test = [] [dependencies] aes = { version = "0.7.5", features = ["ctr"] } -async-acme = { version = "0.5.0", git = "https://github.com/dr-bonez/async-acme.git", features = [ +async-acme = { version = "0.6.0", git = "https://github.com/dr-bonez/async-acme.git", features = [ "use_rustls", "use_tokio", ] } diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 972d0e8e8..3463d4db6 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -1,10 +1,10 @@ use std::collections::{BTreeMap, BTreeSet}; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr}; use chrono::{DateTime, Utc}; use exver::{Version, VersionRange}; use imbl_value::InternedString; -use ipnet::{IpNet, Ipv4Net, Ipv6Net}; +use ipnet::IpNet; use isocountry::CountryCode; use itertools::Itertools; use models::PackageId; @@ -17,7 +17,7 @@ use ts_rs::TS; use crate::account::AccountInfo; use crate::db::model::package::AllPackageData; -use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr}; +use crate::net::acme::AcmeProvider; use crate::prelude::*; use crate::progress::FullProgress; use crate::system::SmtpValue; @@ -55,7 +55,7 @@ impl Public { .parse() .unwrap(), network_interfaces: BTreeMap::new(), - acme: None, + acme: BTreeMap::new(), status_info: ServerStatus { backup_progress: None, updated: false, @@ -133,7 +133,8 @@ pub struct ServerInfo { #[ts(as = "BTreeMap::")] #[serde(default)] pub network_interfaces: BTreeMap, - pub acme: Option, + #[serde(default)] + pub acme: BTreeMap, #[serde(default)] pub status_info: ServerStatus, pub wifi: WifiInfo, @@ -167,7 +168,9 @@ impl NetworkInterfaceInfo { !self.ip_info.as_ref().map_or(true, |ip_info| { ip_info.subnets.iter().all(|ipnet| { if let IpAddr::V4(ip4) = ipnet.addr() { - ip4.is_loopback() || ip4.is_private() || ip4.is_link_local() + ip4.is_loopback() + || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations + || ip4.is_link_local() } else { true } @@ -185,6 +188,8 @@ pub struct IpInfo { #[ts(type = "string[]")] pub subnets: BTreeSet, pub wan_ip: Option, + #[ts(type = "string[]")] + pub ntp_servers: BTreeSet, } #[derive(Debug, Deserialize, Serialize, HasModel, TS)] @@ -192,13 +197,7 @@ pub struct IpInfo { #[model = "Model"] #[ts(export)] pub struct AcmeSettings { - #[ts(type = "string")] - pub provider: Url, - /// email addresses for letsencrypt pub contact: Vec, - #[ts(type = "string[]")] - /// domains to get letsencrypt certs for - pub domains: BTreeSet, } #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] diff --git a/core/startos/src/net/acme.rs b/core/startos/src/net/acme.rs index 95f9d4adb..5d8da41f1 100644 --- a/core/startos/src/net/acme.rs +++ b/core/startos/src/net/acme.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::str::FromStr; +use async_acme::acme::Identifier; use clap::builder::ValueParserFactory; use clap::Parser; use imbl_value::InternedString; @@ -10,6 +11,7 @@ use openssl::pkey::{PKey, Private}; use openssl::x509::X509; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; +use ts_rs::TS; use url::Url; use crate::context::{CliContext, RpcContext}; @@ -78,10 +80,18 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> { async fn read_certificate( &self, - domains: &[String], + identifiers: &[Identifier], directory_url: &str, ) -> Result, Self::Error> { - let domains = JsonKey::new(domains.into_iter().map(InternedString::intern).collect()); + let identifiers = JsonKey::new( + identifiers + .into_iter() + .map(|d| match d { + Identifier::Dns(d) => d.into(), + Identifier::Ip(ip) => InternedString::from_display(ip), + }) + .collect(), + ); let directory_url = directory_url .parse::() .with_kind(ErrorKind::ParseUrl)?; @@ -94,7 +104,7 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> { .into_acme() .into_certs() .into_idx(&directory_url) - .and_then(|a| a.into_idx(&domains)) + .and_then(|a| a.into_idx(&identifiers)) else { return Ok(None); }; @@ -120,13 +130,21 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> { async fn write_certificate( &self, - domains: &[String], + identifiers: &[Identifier], directory_url: &str, key_pem: &str, certificate_pem: &str, ) -> Result<(), Self::Error> { - tracing::info!("Saving new certificate for {domains:?}"); - let domains = JsonKey::new(domains.into_iter().map(InternedString::intern).collect()); + tracing::info!("Saving new certificate for {identifiers:?}"); + let identifiers = JsonKey::new( + identifiers + .into_iter() + .map(|d| match d { + Identifier::Dns(d) => d.into(), + Identifier::Ip(ip) => InternedString::from_display(ip), + }) + .collect(), + ); let directory_url = directory_url .parse::() .with_kind(ErrorKind::ParseUrl)?; @@ -146,7 +164,7 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> { .as_acme_mut() .as_certs_mut() .upsert(&directory_url, || Ok(BTreeMap::new()))? - .insert(&domains, &cert) + .insert(&identifiers, &cert) }) .await?; @@ -155,22 +173,17 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> { } pub fn acme() -> ParentHandler { - ParentHandler::new() - .subcommand( - "init", - from_fn_async(init) - .no_display() - .with_about("Setup ACME certificate acquisition") - .with_call_remote::(), - ) - .subcommand( - "domain", - domain::() - .with_about("Add, remove, or view domains for which to acquire ACME certificates"), - ) + ParentHandler::new().subcommand( + "init", + from_fn_async(init) + .no_display() + .with_about("Setup ACME certificate acquisition") + .with_call_remote::(), + ) } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)] +#[ts(type = "string")] pub struct AcmeProvider(pub Url); impl FromStr for AcmeProvider { type Err = ::Err; @@ -183,6 +196,11 @@ impl FromStr for AcmeProvider { .map(Self) } } +impl AsRef for AcmeProvider { + fn as_ref(&self) -> &str { + self.0.as_str() + } +} impl ValueParserFactory for AcmeProvider { type Parser = FromStrParser; fn value_parser() -> Self::Parser { @@ -200,125 +218,15 @@ pub struct InitAcmeParams { pub async fn init( ctx: RpcContext, - InitAcmeParams { - provider: AcmeProvider(provider), - contact, - }: InitAcmeParams, + InitAcmeParams { provider, contact }: InitAcmeParams, ) -> Result<(), Error> { ctx.db .mutate(|db| { db.as_public_mut() .as_server_info_mut() .as_acme_mut() - .map_mutate(|acme| { - Ok(Some(AcmeSettings { - provider, - contact, - domains: acme.map(|acme| acme.domains).unwrap_or_default(), - })) - }) + .insert(&provider, &AcmeSettings { contact }) }) .await?; Ok(()) } - -pub fn domain() -> ParentHandler { - ParentHandler::new() - .subcommand( - "add", - from_fn_async(add_domain) - .no_display() - .with_about("Add a domain for which to acquire ACME certificates") - .with_call_remote::(), - ) - .subcommand( - "remove", - from_fn_async(remove_domain) - .no_display() - .with_about("Remove a domain for which to acquire ACME certificates") - .with_call_remote::(), - ) - .subcommand( - "list", - from_fn_async(list_domains) - .with_custom_display_fn(|_, res| { - for domain in res { - println!("{domain}") - } - Ok(()) - }) - .with_about("List domains for which to acquire ACME certificates") - .with_call_remote::(), - ) -} - -#[derive(Deserialize, Serialize, Parser)] -pub struct DomainParams { - pub domain: InternedString, -} - -pub async fn add_domain( - ctx: RpcContext, - DomainParams { domain }: DomainParams, -) -> Result<(), Error> { - ctx.db - .mutate(|db| { - db.as_public_mut() - .as_server_info_mut() - .as_acme_mut() - .transpose_mut() - .ok_or_else(|| { - Error::new( - eyre!("Please call `start-cli net acme init` before adding a domain"), - ErrorKind::InvalidRequest, - ) - })? - .as_domains_mut() - .mutate(|domains| { - domains.insert(domain); - Ok(()) - }) - }) - .await?; - Ok(()) -} - -pub async fn remove_domain( - ctx: RpcContext, - DomainParams { domain }: DomainParams, -) -> Result<(), Error> { - ctx.db - .mutate(|db| { - if let Some(acme) = db - .as_public_mut() - .as_server_info_mut() - .as_acme_mut() - .transpose_mut() - { - acme.as_domains_mut().mutate(|domains| { - domains.remove(&domain); - Ok(()) - }) - } else { - Ok(()) - } - }) - .await?; - Ok(()) -} - -pub async fn list_domains(ctx: RpcContext) -> Result, Error> { - if let Some(acme) = ctx - .db - .peek() - .await - .into_public() - .into_server_info() - .into_acme() - .transpose() - { - acme.into_domains().de() - } else { - Ok(BTreeSet::new()) - } -} diff --git a/core/startos/src/net/host/address.rs b/core/startos/src/net/host/address.rs index 05942ffa9..3d639b31e 100644 --- a/core/startos/src/net/host/address.rs +++ b/core/startos/src/net/host/address.rs @@ -1,57 +1,298 @@ -use std::fmt; -use std::str::FromStr; - -use clap::builder::ValueParserFactory; +use clap::Parser; use imbl_value::InternedString; -use models::FromStrParser; +use models::{HostId, PackageId}; +use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use torut::onion::OnionAddressV3; use ts_rs::TS; +use crate::context::{CliContext, RpcContext}; +use crate::net::acme::AcmeProvider; use crate::prelude::*; +use crate::util::serde::{display_serializable, HandlerExtSerde}; -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, TS)] -#[serde(rename_all = "camelCase")] -#[serde(tag = "kind")] -#[ts(export)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub enum HostAddress { Onion { - #[ts(type = "string")] address: OnionAddressV3, }, Domain { - #[ts(type = "string")] address: InternedString, + public: bool, + acme: Option, }, } -impl FromStr for HostAddress { - type Err = Error; - fn from_str(s: &str) -> Result { - if let Some(addr) = s.strip_suffix(".onion") { - Ok(HostAddress::Onion { - address: addr - .parse::() - .with_kind(ErrorKind::ParseUrl)?, - }) - } else { - Ok(HostAddress::Domain { address: s.into() }) - } - } -} - -impl fmt::Display for HostAddress { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Onion { address } => write!(f, "{address}"), - Self::Domain { address } => write!(f, "{address}"), - } - } -} - -impl ValueParserFactory for HostAddress { - type Parser = FromStrParser; - fn value_parser() -> Self::Parser { - Self::Parser::new() - } +#[derive(Debug, Deserialize, Serialize, TS)] +pub struct DomainConfig { + pub public: bool, + pub acme: Option, +} + +#[derive(Deserialize, Serialize, Parser)] +pub struct AddressApiParams { + host: HostId, +} + +pub fn address() -> ParentHandler { + ParentHandler::::new() + .subcommand( + "domain", + ParentHandler::::new() + .subcommand( + "add", + from_fn_async(add_domain) + .with_inherited(|_, a| a) + .no_display() + .with_about("Add an address to this host") + .with_call_remote::(), + ) + .subcommand( + "remove", + from_fn_async(remove_domain) + .with_inherited(|_, a| a) + .no_display() + .with_about("Remove an address from this host") + .with_call_remote::(), + ) + .with_inherited(|AddressApiParams { host }, package| (package, host)), + ) + .subcommand( + "onion", + ParentHandler::::new() + .subcommand( + "add", + from_fn_async(add_onion) + .with_inherited(|_, a| a) + .no_display() + .with_about("Add an address to this host") + .with_call_remote::(), + ) + .subcommand( + "remove", + from_fn_async(remove_onion) + .with_inherited(|_, a| a) + .no_display() + .with_about("Remove an address from this host") + .with_call_remote::(), + ) + .with_inherited(|AddressApiParams { host }, package| (package, host)), + ) + .subcommand( + "list", + from_fn_async(list_addresses) + .with_inherited(|AddressApiParams { host }, package| (package, host)) + .with_display_serializable() + .with_custom_display_fn(|HandlerArgs { params, .. }, res| { + use prettytable::*; + + if let Some(format) = params.format { + display_serializable(format, res); + return Ok(()); + } + + let mut table = Table::new(); + table.add_row(row![bc => "ADDRESS", "PUBLIC", "ACME PROVIDER"]); + for address in &res { + match address { + HostAddress::Onion { address } => { + table.add_row(row![address, true, "N/A"]); + } + HostAddress::Domain { + address, + public, + acme, + } => { + table.add_row(row![ + address, + *public, + acme.as_ref().map(|a| a.0.as_str()).unwrap_or("NONE") + ]); + } + } + } + + table.print_tty(false)?; + + Ok(()) + }) + .with_about("List addresses for this host") + .with_call_remote::(), + ) +} + +#[derive(Deserialize, Serialize, Parser)] +pub struct AddDomainParams { + pub domain: InternedString, + #[arg(long)] + pub private: bool, + #[arg(long)] + pub acme: Option, +} + +pub async fn add_domain( + ctx: RpcContext, + AddDomainParams { + domain, + private, + acme, + }: AddDomainParams, + (package, host): (PackageId, HostId), +) -> Result<(), Error> { + ctx.db + .mutate(|db| { + if let Some(acme) = &acme { + if !db.as_public().as_server_info().as_acme().contains_key(&acme)? { + return Err(Error::new(eyre!("unknown acme provider {}, please run acme.init for this provider first", acme.0), ErrorKind::InvalidRequest)); + } + } + + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(&package) + .or_not_found(&package)? + .as_hosts_mut() + .as_idx_mut(&host) + .or_not_found(&host)? + .as_domains_mut() + .insert( + &domain, + &DomainConfig { + public: !private, + acme, + }, + ) + }) + .await?; + let service = ctx.services.get(&package).await; + let service_ref = service.as_ref().or_not_found(&package)?; + service_ref.update_host(host).await?; + + Ok(()) +} + +#[derive(Deserialize, Serialize, Parser)] +pub struct RemoveDomainParams { + pub domain: InternedString, +} + +pub async fn remove_domain( + ctx: RpcContext, + RemoveDomainParams { domain }: RemoveDomainParams, + (package, host): (PackageId, HostId), +) -> Result<(), Error> { + ctx.db + .mutate(|db| { + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(&package) + .or_not_found(&package)? + .as_hosts_mut() + .as_idx_mut(&host) + .or_not_found(&host)? + .as_domains_mut() + .remove(&domain) + }) + .await?; + let service = ctx.services.get(&package).await; + let service_ref = service.as_ref().or_not_found(&package)?; + service_ref.update_host(host).await?; + + Ok(()) +} + +#[derive(Deserialize, Serialize, Parser)] +pub struct OnionParams { + pub onion: String, +} + +pub async fn add_onion( + ctx: RpcContext, + OnionParams { onion }: OnionParams, + (package, host): (PackageId, HostId), +) -> Result<(), Error> { + let onion = onion + .strip_suffix(".onion") + .ok_or_else(|| { + Error::new( + eyre!("onion hostname must end in .onion"), + ErrorKind::InvalidOnionAddress, + ) + })? + .parse::()?; + ctx.db + .mutate(|db| { + db.as_private().as_key_store().as_onion().get_key(&onion)?; + + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(&package) + .or_not_found(&package)? + .as_hosts_mut() + .as_idx_mut(&host) + .or_not_found(&host)? + .as_onions_mut() + .mutate(|a| Ok(a.insert(onion))) + }) + .await?; + let service = ctx.services.get(&package).await; + let service_ref = service.as_ref().or_not_found(&package)?; + service_ref.update_host(host).await?; + + Ok(()) +} + +pub async fn remove_onion( + ctx: RpcContext, + OnionParams { onion }: OnionParams, + (package, host): (PackageId, HostId), +) -> Result<(), Error> { + let onion = onion + .strip_suffix(".onion") + .ok_or_else(|| { + Error::new( + eyre!("onion hostname must end in .onion"), + ErrorKind::InvalidOnionAddress, + ) + })? + .parse::()?; + ctx.db + .mutate(|db| { + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(&package) + .or_not_found(&package)? + .as_hosts_mut() + .as_idx_mut(&host) + .or_not_found(&host)? + .as_onions_mut() + .mutate(|a| Ok(a.remove(&onion))) + }) + .await?; + let service = ctx.services.get(&package).await; + let service_ref = service.as_ref().or_not_found(&package)?; + service_ref.update_host(host).await?; + + Ok(()) +} + +pub async fn list_addresses( + ctx: RpcContext, + _: Empty, + (package, host): (PackageId, HostId), +) -> Result, Error> { + Ok(ctx + .db + .peek() + .await + .into_public() + .into_package_data() + .into_idx(&package) + .or_not_found(&package)? + .into_hosts() + .into_idx(&host) + .or_not_found(&host)? + .de()? + .addresses() + .collect()) } diff --git a/core/startos/src/net/host/binding.rs b/core/startos/src/net/host/binding.rs index 41261b7c6..d56f607a9 100644 --- a/core/startos/src/net/host/binding.rs +++ b/core/startos/src/net/host/binding.rs @@ -1,13 +1,18 @@ +use std::collections::BTreeMap; use std::str::FromStr; use clap::builder::ValueParserFactory; -use models::{FromStrParser, HostId}; +use clap::Parser; +use models::{FromStrParser, HostId, PackageId}; +use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use ts_rs::TS; +use crate::context::{CliContext, RpcContext}; use crate::net::forward::AvailablePorts; use crate::net::vhost::AlpnInfo; use crate::prelude::*; +use crate::util::serde::{display_serializable, HandlerExtSerde}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, TS)] #[ts(export)] @@ -140,3 +145,122 @@ pub struct AddSslOptions { // pub add_x_forwarded_headers: bool, // TODO pub alpn: Option, } + +#[derive(Deserialize, Serialize, Parser)] +pub struct BindingApiParams { + host: HostId, +} + +pub fn binding() -> ParentHandler { + ParentHandler::::new() + .subcommand( + "list", + from_fn_async(list_bindings) + .with_inherited(|BindingApiParams { host }, package| (package, host)) + .with_display_serializable() + .with_custom_display_fn(|HandlerArgs { params, .. }, res| { + use prettytable::*; + + if let Some(format) = params.format { + return Ok(display_serializable(format, res)); + } + + let mut table = Table::new(); + table.add_row(row![bc => "INTERNAL PORT", "ENABLED", "PUBLIC", "EXTERNAL PORT", "EXTERNAL SSL PORT"]); + for (internal, info) in res { + table.add_row(row![ + internal, + info.enabled, + info.net.public, + if let Some(port) = info.net.assigned_port { + port.to_string() + } else { + "N/A".to_owned() + }, + if let Some(port) = info.net.assigned_ssl_port { + port.to_string() + } else { + "N/A".to_owned() + }, + ]); + } + + table.print_tty(false).unwrap(); + + Ok(()) + }) + .with_about("List bindinges for this host") + .with_call_remote::(), + ) + .subcommand( + "set-public", + from_fn_async(set_public) + .with_inherited(|BindingApiParams { host }, package| (package, host)) + .no_display() + .with_about("Add an binding to this host") + .with_call_remote::(), + ) +} + +pub async fn list_bindings( + ctx: RpcContext, + _: Empty, + (package, host): (PackageId, HostId), +) -> Result, Error> { + ctx.db + .peek() + .await + .into_public() + .into_package_data() + .into_idx(&package) + .or_not_found(&package)? + .into_hosts() + .into_idx(&host) + .or_not_found(&host)? + .into_bindings() + .de() +} + +#[derive(Deserialize, Serialize, Parser)] +#[serde(rename_all = "camelCase")] +pub struct SetPublicParams { + internal_port: u16, + #[arg(long)] + public: Option, +} + +pub async fn set_public( + ctx: RpcContext, + SetPublicParams { + internal_port, + public, + }: SetPublicParams, + (package, host): (PackageId, HostId), +) -> Result<(), Error> { + ctx.db + .mutate(|db| { + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(&package) + .or_not_found(&package)? + .as_hosts_mut() + .as_idx_mut(&host) + .or_not_found(&host)? + .as_bindings_mut() + .mutate(|b| { + b.get_mut(&internal_port) + .or_not_found(internal_port)? + .net + .public = public.unwrap_or(true); + Ok(()) + }) + }) + .await?; + ctx.services + .get(&package) + .await + .as_ref() + .or_not_found(&package)? + .update_host(host) + .await +} diff --git a/core/startos/src/net/host/mod.rs b/core/startos/src/net/host/mod.rs index be5db0f2d..a9ae31c8f 100644 --- a/core/startos/src/net/host/mod.rs +++ b/core/startos/src/net/host/mod.rs @@ -5,13 +5,14 @@ use imbl_value::InternedString; use models::{HostId, PackageId}; use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; +use torut::onion::OnionAddressV3; use ts_rs::TS; -use crate::context::{CliContext, RpcContext}; +use crate::context::RpcContext; use crate::db::model::DatabaseModel; use crate::net::forward::AvailablePorts; -use crate::net::host::address::HostAddress; -use crate::net::host::binding::{BindInfo, BindOptions}; +use crate::net::host::address::{address, DomainConfig, HostAddress}; +use crate::net::host::binding::{binding, BindInfo, BindOptions}; use crate::net::service_interface::HostnameInfo; use crate::prelude::*; @@ -25,7 +26,10 @@ pub mod binding; pub struct Host { pub kind: HostKind, pub bindings: BTreeMap, - pub addresses: BTreeSet, + #[ts(type = "string[]")] + pub onions: BTreeSet, + #[ts(as = "BTreeMap::")] + pub domains: BTreeMap, /// COMPUTED: NetService::update pub hostname_info: BTreeMap>, // internal port -> Hostnames } @@ -39,13 +43,28 @@ impl Host { Self { kind, bindings: BTreeMap::new(), - addresses: BTreeSet::new(), + onions: BTreeSet::new(), + domains: BTreeMap::new(), hostname_info: BTreeMap::new(), } } - pub fn addresses(&self) -> impl Iterator { + pub fn addresses<'a>(&'a self) -> impl Iterator + 'a { // TODO: handle primary - self.addresses.iter() + self.onions + .iter() + .cloned() + .map(|address| HostAddress::Onion { address }) + .chain( + self.domains + .iter() + .map( + |(address, DomainConfig { public, acme })| HostAddress::Domain { + address: address.clone(), + public: *public, + acme: acme.clone(), + }, + ), + ) } } @@ -104,12 +123,12 @@ pub fn host_for<'a>( }; host_info(db, package_id)?.upsert(host_id, || { let mut h = Host::new(host_kind); - h.addresses.insert(HostAddress::Onion { - address: tor_key + h.onions.insert( + tor_key .or_not_found("generated tor key")? .public() .get_onion_address(), - }); + ); Ok(h) }) } @@ -161,6 +180,10 @@ pub fn host() -> ParentHandler { "address", address::().with_inherited(|HostParams { package }, _| package), ) + .subcommand( + "binding", + binding::().with_inherited(|HostParams { package }, _| package), + ) } pub async fn list_hosts( @@ -178,122 +201,3 @@ pub async fn list_hosts( .into_hosts() .keys() } - -#[derive(Deserialize, Serialize, Parser)] -pub struct AddressApiParams { - host: HostId, -} - -pub fn address() -> ParentHandler { - ParentHandler::::new() - .subcommand( - "add", - from_fn_async(add_address) - .with_inherited(|AddressApiParams { host }, package| (package, host)) - .no_display() - .with_about("Add an address to this host") - .with_call_remote::(), - ) - .subcommand( - "remove", - from_fn_async(remove_address) - .with_inherited(|AddressApiParams { host }, package| (package, host)) - .no_display() - .with_about("Remove an address from this host") - .with_call_remote::(), - ) - .subcommand( - "list", - from_fn_async(list_addresses) - .with_inherited(|AddressApiParams { host }, package| (package, host)) - .with_custom_display_fn(|_, res| { - for address in res { - println!("{address}") - } - Ok(()) - }) - .with_about("List addresses for this host") - .with_call_remote::(), - ) -} - -#[derive(Deserialize, Serialize, Parser)] -pub struct AddressParams { - pub address: HostAddress, -} - -pub async fn add_address( - ctx: RpcContext, - AddressParams { address }: AddressParams, - (package, host): (PackageId, HostId), -) -> Result<(), Error> { - ctx.db - .mutate(|db| { - if let HostAddress::Onion { address } = address { - db.as_private() - .as_key_store() - .as_onion() - .get_key(&address)?; - } - - db.as_public_mut() - .as_package_data_mut() - .as_idx_mut(&package) - .or_not_found(&package)? - .as_hosts_mut() - .as_idx_mut(&host) - .or_not_found(&host)? - .as_addresses_mut() - .mutate(|a| Ok(a.insert(address))) - }) - .await?; - let service = ctx.services.get(&package).await; - let service_ref = service.as_ref().or_not_found(&package)?; - service_ref.update_host(host).await?; - - Ok(()) -} - -pub async fn remove_address( - ctx: RpcContext, - AddressParams { address }: AddressParams, - (package, host): (PackageId, HostId), -) -> Result<(), Error> { - ctx.db - .mutate(|db| { - db.as_public_mut() - .as_package_data_mut() - .as_idx_mut(&package) - .or_not_found(&package)? - .as_hosts_mut() - .as_idx_mut(&host) - .or_not_found(&host)? - .as_addresses_mut() - .mutate(|a| Ok(a.remove(&address))) - }) - .await?; - let service = ctx.services.get(&package).await; - let service_ref = service.as_ref().or_not_found(&package)?; - service_ref.update_host(host).await?; - - Ok(()) -} - -pub async fn list_addresses( - ctx: RpcContext, - _: Empty, - (package, host): (PackageId, HostId), -) -> Result, Error> { - ctx.db - .peek() - .await - .into_public() - .into_package_data() - .into_idx(&package) - .or_not_found(&package)? - .into_hosts() - .into_idx(&host) - .or_not_found(&host)? - .into_addresses() - .de() -} diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 91e3e8e67..891f6a9a5 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -85,6 +85,7 @@ impl PreInitNetController { hostname, 443, false, + None, ([127, 0, 0, 1], 80).into(), alpn.clone(), )?); @@ -97,6 +98,7 @@ impl PreInitNetController { )), 443, false, + None, ([127, 0, 0, 1], 80).into(), alpn.clone(), )?); @@ -214,7 +216,7 @@ impl NetService { internal_port: u16, options: BindOptions, ) -> Result<(), Error> { - dbg!("bind", &kind, &id, internal_port, &options); + crate::dbg!("bind", &kind, &id, internal_port, &options); let pkg_id = &self.id; let host = self .net_controller()? @@ -281,10 +283,10 @@ impl NetService { let net_ifaces = server_info.as_network_interfaces().de()?; let hostname = server_info.as_hostname().de()?; for (port, bind) in &host.bindings { + let old_lan_bind = binds.lan.remove(port); if !bind.enabled { continue; } - let old_lan_bind = binds.lan.remove(port); let lan_bind = old_lan_bind .as_ref() .filter(|(external, ssl, _, _)| { @@ -295,7 +297,9 @@ impl NetService { let new_lan_bind = if let Some(b) = lan_bind { b } else { - let mut rcs = Vec::with_capacity(2 + host.addresses.len()); + let mut rcs = Vec::with_capacity( + 2 + ctrl.server_hostnames.len() + host.domains.len() + host.onions.len(), + ); let mut hostnames = BTreeSet::new(); if let Some(ssl) = &bind.options.add_ssl { let external = bind @@ -317,6 +321,7 @@ impl NetService { hostname, external, bind.net.public, + None, target, connect_ssl.clone(), )?); @@ -324,39 +329,50 @@ impl NetService { for address in host.addresses() { match address { HostAddress::Onion { address } => { - let hostname = InternedString::from_display(address); + let hostname = InternedString::from_display(&address); if hostnames.insert(hostname.clone()) { rcs.push(ctrl.vhost.add( Some(hostname), external, false, + None, target, connect_ssl.clone(), )?); } } - HostAddress::Domain { address } => { + HostAddress::Domain { + address, + public, + acme, + } => { if hostnames.insert(address.clone()) { let address = Some(address.clone()); - rcs.push(ctrl.vhost.add( - address.clone(), - external, - bind.net.public, - target, - connect_ssl.clone(), - )?); if ssl.preferred_external_port == 443 { + if public && bind.net.public { + rcs.push(ctrl.vhost.add( + address.clone(), + 5443, + false, + acme.clone(), + target, + connect_ssl.clone(), + )?); + } rcs.push(ctrl.vhost.add( address.clone(), - 5443, - false, + 443, + public && bind.net.public, + acme, target, connect_ssl.clone(), )?); + } else { rcs.push(ctrl.vhost.add( address.clone(), - 443, - true, + external, + public && bind.net.public, + acme, target, connect_ssl.clone(), )?); @@ -403,33 +419,40 @@ impl NetService { }); } for address in host.addresses() { - if let HostAddress::Domain { address } = address { - if new_lan_bind - .1 - .as_ref() - .map_or(false, |ssl| ssl.preferred_external_port == 443) - { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public: true, // TODO: check if port forward is active - hostname: IpHostname::Domain { - domain: address.clone(), - subdomain: None, - port: None, - ssl_port: Some(443), - }, - }); - } else if public && new_lan_bind.0.public { - bind_hostname_info.push(HostnameInfo::Ip { - network_interface_id: interface.clone(), - public, - hostname: IpHostname::Domain { - domain: address.clone(), - subdomain: None, - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, - }, - }); + if let HostAddress::Domain { + address, + public: domain_public, + .. + } = address + { + if !public || (domain_public && new_lan_bind.0.public) { + if new_lan_bind + .1 + .as_ref() + .map_or(false, |ssl| ssl.preferred_external_port == 443) + { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: public && domain_public && bind.net.public, // TODO: check if port forward is active + hostname: IpHostname::Domain { + domain: address.clone(), + subdomain: None, + port: None, + ssl_port: Some(443), + }, + }); + } else { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public, + hostname: IpHostname::Domain { + domain: address.clone(), + subdomain: None, + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } } } } @@ -481,10 +504,10 @@ impl NetService { if let Some((lan, _, hostnames, _)) = old_lan_bind { if let Some(external) = lan.assigned_ssl_port { for hostname in ctrl.server_hostnames.iter().cloned() { - ctrl.vhost.gc(hostname, external)?; + ctrl.vhost.gc(hostname, external); } for hostname in hostnames { - ctrl.vhost.gc(Some(hostname), external)?; + ctrl.vhost.gc(Some(hostname), external); } } if let Some(external) = lan.assigned_port { @@ -505,10 +528,10 @@ impl NetService { for (lan, hostnames) in removed { if let Some(external) = lan.assigned_ssl_port { for hostname in ctrl.server_hostnames.iter().cloned() { - ctrl.vhost.gc(hostname, external)?; + ctrl.vhost.gc(hostname, external); } for hostname in hostnames { - ctrl.vhost.gc(Some(hostname), external)?; + ctrl.vhost.gc(Some(hostname), external); } } if let Some(external) = lan.assigned_port { @@ -557,13 +580,7 @@ impl NetService { } let mut keep_tor_addrs = BTreeSet::new(); - for tor_addr in host.addresses().filter_map(|a| { - if let HostAddress::Onion { address } = a { - Some(address) - } else { - None - } - }) { + for tor_addr in host.onions.iter() { keep_tor_addrs.insert(tor_addr); let old_tor_bind = binds.tor.remove(tor_addr); let tor_bind = old_tor_bind.filter(|(ports, _)| ports == &tor_binds); diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index dfb1f34b2..9f7439bfd 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -1,13 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; -use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV6}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; use std::pin::Pin; use std::sync::{Arc, Weak}; use std::task::Poll; use std::time::Duration; use clap::Parser; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use getifaddrs::if_nametoindex; use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; @@ -16,12 +16,14 @@ use itertools::Itertools; use patch_db::json_ptr::JsonPointer; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::net::{TcpListener, TcpStream}; +use tokio::process::Command; use tokio::sync::watch; use ts_rs::TS; use zbus::proxy::{PropertyChanged, PropertyStream, SignalStream}; use zbus::zvariant::{ - DeserializeDict, OwnedObjectPath, OwnedValue, Type as ZType, Value as ZValue, + DeserializeDict, Dict, OwnedObjectPath, OwnedValue, Type as ZType, Value as ZValue, }; use zbus::{proxy, Connection}; @@ -31,8 +33,10 @@ use crate::db::model::Database; use crate::net::network_interface::active_connection::ActiveConnectionProxy; use crate::prelude::*; use crate::util::future::Until; +use crate::util::io::open_file; use crate::util::serde::{display_serializable, HandlerExtSerde}; use crate::util::sync::SyncMutex; +use crate::util::Invoke; pub fn network_interface_api() -> ParentHandler { ParentHandler::new() @@ -48,15 +52,15 @@ pub fn network_interface_api() -> ParentHandler { } let mut table = Table::new(); - table.add_row(row![bc => "INTERFACE", "PUBLIC", "ADDRESSES"]); + table.add_row(row![bc => "INTERFACE", "PUBLIC", "ADDRESSES", "WAN IP"]); for (iface, info) in res { table.add_row(row![ iface, info.public(), - info.ip_info.map_or_else( + info.ip_info.as_ref().map_or_else( || "".to_owned(), |ip_info| ip_info.subnets - .into_iter() + .iter() .map(|ipnet| match ipnet.addr() { IpAddr::V4(ip) => format!("{ip}/{}", ipnet.prefix_len()), IpAddr::V6(ip) => format!( @@ -65,7 +69,10 @@ pub fn network_interface_api() -> ParentHandler { ipnet.prefix_len() ), }) - .join(", ")) + .join(", ")), + info.ip_info.as_ref() + .and_then(|ip_info| ip_info.wan_ip) + .map_or_else(|| "N/A".to_owned(), |ip| ip.to_string()) ]); } @@ -90,6 +97,12 @@ pub fn network_interface_api() -> ParentHandler { .no_display() .with_about("Allow this interface to infer whether it is publicly addressable based on its IPv4 address") .with_call_remote::(), + ).subcommand("forget", + from_fn_async(forget_iface) + .with_metadata("sync_db", Value::Bool(true)) + .no_display() + .with_about("Forget a disconnected interface") + .with_call_remote::() ) } @@ -134,6 +147,20 @@ async fn unset_public( .await } +#[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)] +#[ts(export)] +struct ForgetInterfaceParams { + #[ts(type = "string")] + interface: InternedString, +} + +async fn forget_iface( + ctx: RpcContext, + ForgetInterfaceParams { interface }: ForgetInterfaceParams, +) -> Result<(), Error> { + ctx.net_controller.net_iface.forget(&interface).await +} + #[proxy( interface = "org.freedesktop.NetworkManager", default_service = "org.freedesktop.NetworkManager", @@ -152,6 +179,7 @@ trait NetworkManager { mod active_connection { use zbus::proxy; + use zbus::zvariant::OwnedObjectPath; use crate::prelude::*; @@ -168,6 +196,9 @@ mod active_connection { #[zbus(signal)] fn state_changed(&self) -> Result<(), Error>; + + #[zbus(property)] + fn dhcp4_config(&self) -> Result; } } @@ -202,6 +233,30 @@ impl TryFrom for IpNet { } } +#[proxy( + interface = "org.freedesktop.NetworkManager.DHCP4Config", + default_service = "org.freedesktop.NetworkManager" +)] +trait Dhcp4Config { + #[zbus(property)] + fn options(&self) -> Result; +} + +#[derive(Clone, Debug, DeserializeDict, ZType)] +#[zvariant(signature = "dict")] +struct Dhcp4Options { + ntp_servers: Option, +} +impl TryFrom for Dhcp4Options { + type Error = zbus::Error; + fn try_from(value: OwnedValue) -> Result { + let dict = value.downcast_ref::()?; + Ok(Self { + ntp_servers: dict.get::<_, String>(&zbus::zvariant::Str::from_static("ntp_servers"))?, + }) + } +} + #[proxy( interface = "org.freedesktop.NetworkManager.Device", default_service = "org.freedesktop.NetworkManager" @@ -276,9 +331,9 @@ async fn watcher(write_to: watch::Sender Result, Error> { +async fn get_wan_ipv4(iface: &str) -> Result, Error> { Ok(reqwest::Client::builder() - .local_address(Some(IpAddr::V4(local_addr))) + .interface(iface) .build()? .get("http://ip4only.me/api/") .timeout(Duration::from_secs(10)) @@ -352,7 +407,7 @@ async fn get_wan_ipv4(local_addr: Ipv4Addr) -> Result, Error> { .split(",") .skip(1) .next() - .filter(|s| s.is_empty()) + .filter(|s| !s.is_empty()) .map(|s| s.parse()) .transpose()?) } @@ -387,11 +442,18 @@ async fn watch_ip( .stub(), ) .with_stream(device_proxy.receive_ip4_config_changed().await.stub()) - .with_stream(device_proxy.receive_ip6_config_changed().await.stub()); + .with_stream(device_proxy.receive_ip6_config_changed().await.stub()) + .with_stream( + active_connection_proxy + .receive_dhcp4_config_changed() + .await + .stub(), + ); loop { let ip4_config = device_proxy.ip4_config().await?; let ip6_config = device_proxy.ip6_config().await?; + let dhcp4_config = active_connection_proxy.dhcp4_config().await?; until .run(async { let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config).await?; @@ -400,49 +462,50 @@ async fn watch_ip( .with_stream(ip4_proxy.receive_address_data_changed().await.stub()) .with_stream(ip6_proxy.receive_address_data_changed().await.stub()); + let dhcp4_proxy = if &*dhcp4_config != "/" { + let dhcp4_proxy = Dhcp4ConfigProxy::new(&connection, dhcp4_config).await?; + until = until.with_stream(dhcp4_proxy.receive_options_changed().await.stub()); + Some(dhcp4_proxy) + } else { + None + }; + loop { - let addresses = ip4_proxy - .address_data() - .await? - .into_iter() - .chain(ip6_proxy.address_data().await?) - .collect_vec(); - if iface == "enp1s0" { - dbg!(&addresses); - } until .run(async { + let addresses = ip4_proxy + .address_data() + .await? + .into_iter() + .chain(ip6_proxy.address_data().await?) + .collect_vec(); + let mut ntp_servers = BTreeSet::new(); + if let Some(dhcp4_proxy) = &dhcp4_proxy { + let dhcp = crate::dbg!(dhcp4_proxy.options().await?); + if let Some(ntp) = dhcp.ntp_servers { + ntp_servers + .extend(ntp.split_whitespace().map(InternedString::intern)); + } + } let scope_id = if_nametoindex(&*iface).with_kind(ErrorKind::Network)?; let subnets: BTreeSet = addresses.into_iter().map(TryInto::try_into).try_collect()?; let ip_info = if !subnets.is_empty() { - let wan_ip = if let Some(local_addr) = - subnets.iter().find_map(|s| match s { - IpNet::V4(net) - if !net.addr().is_loopback() - && !net.addr().is_link_local() => - { - Some(net) - } - _ => None, - }) { - match get_wan_ipv4(local_addr.addr()).await { - Ok(a) => a, - Err(e) => { - tracing::error!( - "Failed to determine WAN IP for {iface}: {e}" - ); - tracing::debug!("{e:?}"); - None - } + let wan_ip = match get_wan_ipv4(&*iface).await { + Ok(a) => a, + Err(e) => { + tracing::error!( + "Failed to determine WAN IP for {iface}: {e}" + ); + tracing::debug!("{e:?}"); + None } - } else { - None }; Some(IpInfo { scope_id, subnets, wan_ip, + ntp_servers, }) } else { None @@ -452,12 +515,12 @@ async fn watch_ip( let public = m.get(&iface).map_or(None, |i| i.public); m.insert( iface.clone(), - dbg!(NetworkInterfaceInfo { + NetworkInterfaceInfo { public, ip_info: ip_info.clone(), - }), + }, ) - .filter(|old| &dbg!(old).ip_info == &ip_info) + .filter(|old| &old.ip_info == &ip_info) .is_none() }); @@ -481,6 +544,8 @@ impl NetworkInterfaceController { db: &TypedPatchDb, info: &BTreeMap, ) -> Result<(), Error> { + tracing::debug!("syncronizing {info:?} to db"); + db.mutate(|db| { db.as_public_mut() .as_server_info_mut() @@ -488,6 +553,55 @@ impl NetworkInterfaceController { .ser(info) }) .await?; + + let ntp: BTreeSet<_> = info + .values() + .filter_map(|i| i.ip_info.as_ref()) + .flat_map(|i| &i.ntp_servers) + .cloned() + .collect(); + let prev_ntp = tokio_stream::wrappers::LinesStream::new( + BufReader::new(open_file("/etc/systemd/timesyncd.conf").await?).lines(), + ) + .try_filter_map(|l| async move { + Ok(l.strip_prefix("NTP=").map(|s| { + s.split_whitespace() + .map(InternedString::intern) + .collect::>() + })) + }) + .boxed() + .try_next() + .await? + .unwrap_or_default(); + if ntp != prev_ntp { + // sed -i '/\(^\|#\)NTP=/c\NTP='"${servers}" /etc/systemd/timesyncd.conf + Command::new("sed") + .arg("-i") + .arg( + [r#"/\(^\|#\)NTP=/c\NTP="#] + .into_iter() + .chain(Itertools::intersperse( + { + fn to_str(ntp: &InternedString) -> &str { + &*ntp + } + ntp.iter().map(to_str) + }, + " ", + )) + .join(""), + ) + .arg("/etc/systemd/timesyncd.conf") + .invoke(ErrorKind::Filesystem) + .await?; + Command::new("systemctl") + .arg("restart") + .arg("systemd-timesyncd") + .invoke(ErrorKind::Systemd) + .await?; + } + Ok(()) } pub fn new(db: TypedPatchDb) -> Self { @@ -516,10 +630,19 @@ impl NetworkInterfaceController { } }; tokio::join!(watcher(write_to), async { - loop { - if let Err(e) = async { - let ip_info = read_from.borrow().clone(); - Self::sync(&db, &ip_info).await?; + let res: Result<(), Error> = async { + loop { + if let Err(e) = async { + let ip_info = { read_from.borrow().clone() }; + Self::sync(&db, &ip_info).boxed().await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error syncing ip info to db: {e}"); + tracing::debug!("{e:?}"); + } read_from.changed().await.map_err(|_| { Error::new( @@ -527,15 +650,13 @@ impl NetworkInterfaceController { ErrorKind::Network, ) })?; - - Ok::<_, Error>(()) - } - .await - { - tracing::error!("Error syncing ip info to db: {e}"); - tracing::debug!("{e:?}"); } } + .await; + if let Err(e) = res { + tracing::error!("Error syncing ip info to db: {e}"); + tracing::debug!("{e:?}"); + } }); }) .into(), @@ -599,11 +720,43 @@ impl NetworkInterfaceController { } Ok(()) } + + pub async fn forget(&self, interface: &InternedString) -> Result<(), Error> { + let mut sub = self + .db + .subscribe( + "/public/serverInfo/networkInterfaces" + .parse::>() + .with_kind(ErrorKind::Database)?, + ) + .await; + let mut err = None; + let changed = self.ip_info.send_if_modified(|ip_info| { + if ip_info + .get(interface) + .map_or(false, |i| i.ip_info.is_some()) + { + err = Some(Error::new( + eyre!("Cannot forget currently connected interface"), + ErrorKind::InvalidRequest, + )); + return false; + } + ip_info.remove(interface).is_some() + }); + if let Some(e) = err { + return Err(e); + } + if changed { + sub.recv().await; + } + Ok(()) + } } struct ListenerMap { port: u16, - listeners: BTreeMap<(IpAddr, u32), (TcpListener, bool)>, + listeners: BTreeMap<(IpAddr, u32), (TcpListener, bool, Option)>, } impl ListenerMap { fn new(port: u16) -> Self { @@ -618,14 +771,28 @@ impl ListenerMap { public: bool, ) -> Result<(), Error> { let mut keep = BTreeSet::<(IpAddr, u32)>::new(); - for info in ip_info.values() { + for info in ip_info.values().chain([&NetworkInterfaceInfo { + public: Some(false), + ip_info: Some(IpInfo { + scope_id: 1, + subnets: [ + IpNet::new(Ipv4Addr::LOCALHOST.into(), 8).unwrap(), + IpNet::new(Ipv6Addr::LOCALHOST.into(), 128).unwrap(), + ] + .into_iter() + .collect(), + wan_ip: None, + ntp_servers: Default::default(), + }), + }]) { if public || !info.public() { if let Some(ip_info) = &info.ip_info { for ipnet in &ip_info.subnets { let key = (ipnet.addr(), ip_info.scope_id); keep.insert(key); - if let Some((_, is_public)) = self.listeners.get_mut(&key) { + if let Some((_, is_public, wan_ip)) = self.listeners.get_mut(&key) { *is_public = info.public(); + *wan_ip = info.ip_info.as_ref().and_then(|i| i.wan_ip); continue; } self.listeners.insert( @@ -640,6 +807,7 @@ impl ListenerMap { }) .await?, info.public(), + info.ip_info.as_ref().and_then(|i| i.wan_ip), ), ); } @@ -654,14 +822,14 @@ impl ListenerMap { } } #[pin_project::pin_project] -struct ListenerMapFut<'a>(&'a mut BTreeMap<(IpAddr, u32), (TcpListener, bool)>); +struct ListenerMapFut<'a>(&'a mut BTreeMap<(IpAddr, u32), (TcpListener, bool, Option)>); impl<'a> Future for ListenerMapFut<'a> { - type Output = Result<(IpAddr, bool, TcpStream, SocketAddr), Error>; + type Output = Result<(IpAddr, bool, Option, TcpStream, SocketAddr), Error>; fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { let this = self.project(); for ((ip, _), listener) in this.0.iter() { if let Poll::Ready((stream, addr)) = listener.0.poll_accept(cx)? { - return Poll::Ready(Ok((*ip, listener.1, stream, addr))); + return Poll::Ready(Ok((*ip, listener.1, listener.2, stream, addr))); } } Poll::Pending @@ -675,6 +843,10 @@ pub struct NetworkInterfaceListener { _arc: Arc<()>, } impl NetworkInterfaceListener { + pub fn port(&self) -> u16 { + self.listeners.port + } + pub async fn accept(&mut self, public: bool) -> Result { loop { if self.needs_update { @@ -684,11 +856,12 @@ impl NetworkInterfaceListener { } tokio::select! { accepted = self.listeners.accept() => { - let (ip, is_public, stream, peer) = accepted?; + let (ip, is_public, wan_ip, stream, peer) = accepted?; return Ok(Accepted { stream, peer, is_public, + wan_ip, bind: (ip, self.listeners.port).into(), }) }, @@ -708,6 +881,7 @@ pub struct Accepted { pub stream: TcpStream, pub peer: SocketAddr, pub is_public: bool, + pub wan_ip: Option, pub bind: SocketAddr, } diff --git a/core/startos/src/net/ssl.rs b/core/startos/src/net/ssl.rs index 29bcd9652..a89853591 100644 --- a/core/startos/src/net/ssl.rs +++ b/core/startos/src/net/ssl.rs @@ -17,7 +17,6 @@ use openssl::x509::{X509Builder, X509Extension, X509NameBuilder, X509}; use openssl::*; use patch_db::HasModel; use serde::{Deserialize, Serialize}; -use tokio::time::Instant; use tracing::instrument; use crate::account::AccountInfo; diff --git a/core/startos/src/net/static_server.rs b/core/startos/src/net/static_server.rs index c070d7920..b6961df93 100644 --- a/core/startos/src/net/static_server.rs +++ b/core/startos/src/net/static_server.rs @@ -9,7 +9,7 @@ use async_compression::tokio::bufread::GzipEncoder; use axum::body::Body; use axum::extract::{self as x, Request}; use axum::response::Response; -use axum::routing::{any, get, post}; +use axum::routing::{any, get}; use axum::Router; use base64::display::Base64Display; use digest::Digest; @@ -26,7 +26,6 @@ use new_mime_guess::MimeGuess; use openssl::hash::MessageDigest; use openssl::x509::X509; use rpc_toolkit::{Context, HttpServer, Server}; -use sqlx::query; use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, BufReader}; use tokio_util::io::ReaderStream; use url::Url; diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs index 798471219..809effc7d 100644 --- a/core/startos/src/net/vhost.rs +++ b/core/startos/src/net/vhost.rs @@ -1,22 +1,23 @@ use std::collections::BTreeMap; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::net::{IpAddr, SocketAddr}; use std::str::FromStr; use std::sync::{Arc, Weak}; use std::time::Duration; -use async_acme::acme::ACME_TLS_ALPN_NAME; +use async_acme::acme::{Identifier, ACME_TLS_ALPN_NAME}; use axum::body::Body; use axum::extract::Request; use axum::response::Response; use color_eyre::eyre::eyre; +use futures::FutureExt; use helpers::NonDetachingJoinHandle; use http::Uri; use imbl_value::InternedString; use models::ResultExt; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; -use tokio::net::{TcpListener, TcpStream}; -use tokio::sync::{watch, Mutex, RwLock}; +use tokio::net::TcpStream; +use tokio::sync::watch; use tokio_rustls::rustls::crypto::CryptoProvider; use tokio_rustls::rustls::pki_types::{ CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName, @@ -31,7 +32,7 @@ use tracing::instrument; use ts_rs::TS; use crate::db::model::Database; -use crate::net::acme::AcmeCertCache; +use crate::net::acme::{AcmeCertCache, AcmeProvider}; use crate::net::network_interface::{ Accepted, NetworkInterfaceController, NetworkInterfaceListener, }; @@ -55,6 +56,7 @@ pub struct VHostController { db: TypedPatchDb, interfaces: Arc, crypto_provider: Arc, + acme_tls_alpn_cache: AcmeTlsAlpnCache, servers: SyncMutex>, } impl VHostController { @@ -63,6 +65,7 @@ impl VHostController { db, interfaces, crypto_provider: Arc::new(tokio_rustls::rustls::crypto::ring::default_provider()), + acme_tls_alpn_cache: Arc::new(SyncMutex::new(BTreeMap::new())), servers: SyncMutex::new(BTreeMap::new()), } } @@ -72,6 +75,7 @@ impl VHostController { hostname: Option, external: u16, public: bool, + acme: Option, target: SocketAddr, connect_ssl: Result<(), AlpnInfo>, // Ok: yes, connect using ssl, pass through alpn; Err: connect tcp, use provided strategy for alpn ) -> Result, Error> { @@ -84,12 +88,14 @@ impl VHostController { self.db.clone(), self.interfaces.clone(), self.crypto_provider.clone(), + self.acme_tls_alpn_cache.clone(), )? }; let rc = server.add( hostname, TargetInfo { public, + acme, addr: target, connect_ssl, }, @@ -99,15 +105,14 @@ impl VHostController { }) } #[instrument(skip_all)] - pub fn gc(&self, hostname: Option, external: u16) -> Result<(), Error> { + pub fn gc(&self, hostname: Option, external: u16) { self.servers.mutate(|writable| { if let Some(server) = writable.remove(&external) { - server.gc(hostname)?; - if !server.is_empty()? { + server.gc(hostname); + if !server.is_empty() { writable.insert(external, server); } } - Ok(()) }) } } @@ -115,6 +120,7 @@ impl VHostController { #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] struct TargetInfo { public: bool, + acme: Option, addr: SocketAddr, connect_ssl: Result<(), AlpnInfo>, } @@ -134,26 +140,48 @@ impl Default for AlpnInfo { type AcmeTlsAlpnCache = Arc>>>>>; -type Mapping = SyncMutex, BTreeMap>>>; +type Mapping = BTreeMap, BTreeMap>>; struct VHostServer { - mapping: Weak, + mapping: watch::Sender, _thread: NonDetachingJoinHandle<()>, } impl VHostServer { async fn accept( listener: &mut NetworkInterfaceListener, - mapping: Arc, + mut mapping: watch::Receiver, db: TypedPatchDb, acme_tls_alpn_cache: AcmeTlsAlpnCache, crypto_provider: Arc, ) -> Result<(), Error> { - let any_public = mapping.peek(|m| { - m.iter() - .any(|(_, targets)| targets.keys().any(|target| target.public)) - }); - let accepted = listener.accept(any_public).await?; + let accepted; + + loop { + let any_public = mapping + .borrow() + .iter() + .any(|(_, targets)| targets.iter().any(|(target, _)| target.public)); + + let changed_public = mapping + .wait_for(|m| { + m.iter() + .any(|(_, targets)| targets.iter().any(|(target, _)| target.public)) + != any_public + }) + .boxed(); + + tokio::select! { + a = listener.accept(any_public) => { + accepted = a?; + break; + } + _ = changed_public => { + tracing::debug!("port {} {} public bindings", listener.port(), if any_public { "no longer has" } else { "now has" }); + } + } + } + if let Err(e) = socket2::SockRef::from(&accepted.stream).set_tcp_keepalive( &socket2::TcpKeepalive::new() .with_time(Duration::from_secs(900)) @@ -181,10 +209,11 @@ impl VHostServer { Accepted { stream, is_public, + wan_ip, bind, .. }: Accepted, - mapping: Arc, + mapping: watch::Receiver, db: TypedPatchDb, acme_tls_alpn_cache: AcmeTlsAlpnCache, crypto_provider: Arc, @@ -258,8 +287,58 @@ impl VHostServer { } } }; - let target_name = mid.client_hello().server_name().map(|s| s.into()); - let target = mapping.peek(|m| { + let target_name: Option = + mid.client_hello().server_name().map(|s| s.into()); + if let Some(domain) = target_name.as_ref() { + if mid + .client_hello() + .alpn() + .into_iter() + .flatten() + .any(|alpn| alpn == ACME_TLS_ALPN_NAME) + { + let cert = WatchStream::new( + acme_tls_alpn_cache + .peek(|c| c.get(&**domain).cloned()) + .ok_or_else(|| { + Error::new( + eyre!("No challenge recv available for {domain}"), + ErrorKind::OpenSsl, + ) + })?, + ); + tracing::info!("Waiting for verification cert for {domain}"); + let cert = cert + .filter(|c| c.is_some()) + .next() + .await + .flatten() + .ok_or_else(|| { + Error::new( + eyre!("No challenge available for {domain}"), + ErrorKind::OpenSsl, + ) + })?; + tracing::info!("Verification cert received for {domain}"); + let mut cfg = ServerConfig::builder_with_provider(crypto_provider.clone()) + .with_safe_default_protocol_versions() + .with_kind(crate::ErrorKind::OpenSsl)? + .with_no_client_auth() + .with_cert_resolver(Arc::new(SingleCertResolver(cert))); + + cfg.alpn_protocols = vec![ACME_TLS_ALPN_NAME.to_vec()]; + tracing::info!("performing ACME auth challenge"); + let mut accept = mid.into_stream(Arc::new(cfg)); + let io = accept.get_mut().unwrap(); + let buffered = io.stop_buffering(); + io.write_all(&buffered).await?; + accept.await?; + tracing::info!("ACME auth challenge completed"); + return Ok(()); + } + } + let target = { + let m = mapping.borrow(); m.get(&target_name) .into_iter() .flatten() @@ -279,10 +358,12 @@ impl VHostServer { } }) .map(|(target, _)| target.clone()) - }); - if let Some(target) = dbg!(target) { + }; + if let Some(target) = target { if is_public && !target.public { - log::warn!("Rejecting connection from public interface to private bind"); + log::warn!( + "Rejecting connection from public interface to private bind: {bind} -> {target:?}" + ); return Ok(()); } let peek = db.peek().await; @@ -292,80 +373,52 @@ impl VHostServer { .as_local_certs() .as_root_cert() .de()?; - let mut cfg = match async { - if let Some(acme_settings) = peek.as_public().as_server_info().as_acme().de()? { - if let Some(domain) = target_name - .as_ref() - .filter(|target_name| acme_settings.domains.contains(*target_name)) + let mut cfg = async { + if let Some((domain, provider, settings)) = + target_name.as_ref().and_then(|domain| { + target.acme.as_ref().and_then(|a| { + peek.as_public() + .as_server_info() + .as_acme() + .as_idx(a) + .map(|s| (domain, a, s)) + }) + }) + { + let acme_settings = settings.de()?; + let mut identifiers = vec![Identifier::Dns(domain.to_string())]; + if false + // Requires RFC 8738 { - if mid - .client_hello() - .alpn() - .into_iter() - .flatten() - .any(|alpn| alpn == ACME_TLS_ALPN_NAME) - { - let cert = WatchStream::new( - acme_tls_alpn_cache - .peek(|c| c.get(&**domain).cloned()) - .ok_or_else(|| { - Error::new( - eyre!("No challenge recv available for {domain}"), - ErrorKind::OpenSsl, - ) - })?, - ); - tracing::info!("Waiting for verification cert for {domain}"); - let cert = cert - .filter(|c| c.is_some()) - .next() - .await - .flatten() - .ok_or_else(|| { - Error::new( - eyre!("No challenge available for {domain}"), - ErrorKind::OpenSsl, - ) - })?; - tracing::info!("Verification cert received for {domain}"); - let mut cfg = - ServerConfig::builder_with_provider(crypto_provider.clone()) - .with_safe_default_protocol_versions() - .with_kind(crate::ErrorKind::OpenSsl)? - .with_no_client_auth() - .with_cert_resolver(Arc::new(SingleCertResolver(cert))); - - cfg.alpn_protocols = vec![ACME_TLS_ALPN_NAME.to_vec()]; - return Ok(Err(cfg)); - } else { - let domains = [domain.to_string()]; - let (send, recv) = watch::channel(None); - acme_tls_alpn_cache.mutate(|c| c.insert(domain.clone(), recv)); - let cert = async_acme::rustls_helper::order( - |_, cert| { - send.send_replace(Some(Arc::new(cert))); - Ok(()) - }, - acme_settings.provider.as_str(), - &domains, - Some(&AcmeCertCache(&db)), - &acme_settings.contact, - ) - .await - .with_kind(ErrorKind::OpenSsl)?; - return Ok(Ok(ServerConfig::builder_with_provider( - crypto_provider.clone(), - ) - .with_safe_default_protocol_versions() - .with_kind(crate::ErrorKind::OpenSsl)? - .with_no_client_auth() - .with_cert_resolver(Arc::new(SingleCertResolver(Arc::new(cert)))))); + if let Some(wan_ip) = wan_ip { + identifiers.push(Identifier::Ip(wan_ip.into())); } } + let (send, recv) = watch::channel(None); + acme_tls_alpn_cache.mutate(|c| c.insert(domain.clone(), recv)); + let cert = async_acme::rustls_helper::order( + |_, cert| { + send.send_replace(Some(Arc::new(cert))); + Ok(()) + }, + provider.0.as_str(), + &identifiers, + Some(&AcmeCertCache(&db)), + &acme_settings.contact, + ) + .await + .with_kind(ErrorKind::OpenSsl)?; + return Ok(ServerConfig::builder_with_provider(crypto_provider.clone()) + .with_safe_default_protocol_versions() + .with_kind(crate::ErrorKind::OpenSsl)? + .with_no_client_auth() + .with_cert_resolver(Arc::new(SingleCertResolver(Arc::new(cert))))); } + let hostnames = target_name .into_iter() .chain([InternedString::from_display(&bind.ip())]) + .chain(wan_ip.as_ref().map(InternedString::from_display)) .collect(); let key = db .mutate(|v| { @@ -413,22 +466,8 @@ impl VHostServer { ) } .with_kind(crate::ErrorKind::OpenSsl) - .map(Ok) } - .await? - { - Ok(a) => a, - Err(cfg) => { - tracing::info!("performing ACME auth challenge"); - let mut accept = mid.into_stream(Arc::new(cfg)); - let io = accept.get_mut().unwrap(); - let buffered = io.stop_buffering(); - io.write_all(&buffered).await?; - accept.await?; - tracing::info!("ACME auth challenge completed"); - return Ok(()); - } - }; + .await?; let mut tcp_stream = TcpStream::connect(target.addr).await?; match target.connect_ssl { Ok(()) => { @@ -546,17 +585,17 @@ impl VHostServer { db: TypedPatchDb, iface_ctrl: Arc, crypto_provider: Arc, + acme_tls_alpn_cache: AcmeTlsAlpnCache, ) -> Result { - let acme_tls_alpn_cache = Arc::new(SyncMutex::new(BTreeMap::new())); let mut listener = iface_ctrl.bind(port).with_kind(crate::ErrorKind::Network)?; - let mapping = Arc::new(SyncMutex::new(BTreeMap::new())); + let (map_send, map_recv) = watch::channel(BTreeMap::new()); Ok(Self { - mapping: Arc::downgrade(&mapping), + mapping: map_send, _thread: tokio::spawn(async move { loop { if let Err(e) = Self::accept( &mut listener, - mapping.clone(), + map_recv.clone(), db.clone(), acme_tls_alpn_cache.clone(), crypto_provider.clone(), @@ -574,19 +613,23 @@ impl VHostServer { }) } fn add(&self, hostname: Option, target: TargetInfo) -> Result, Error> { - if let Some(mapping) = Weak::upgrade(&self.mapping) { - mapping.mutate(|writable| { - let mut targets = writable.remove(&hostname).unwrap_or_default(); - let rc = - if let Some(rc) = Weak::upgrade(&targets.remove(&target).unwrap_or_default()) { - rc - } else { - Arc::new(()) - }; - targets.insert(target, Arc::downgrade(&rc)); - writable.insert(hostname, targets); - Ok(rc) - }) + let mut res = Ok(Arc::new(())); + self.mapping.send_if_modified(|writable| { + let mut changed = false; + let mut targets = writable.remove(&hostname).unwrap_or_default(); + let rc = if let Some(rc) = Weak::upgrade(&targets.remove(&target).unwrap_or_default()) { + rc + } else { + changed = true; + Arc::new(()) + }; + targets.insert(target, Arc::downgrade(&rc)); + writable.insert(hostname, targets); + res = Ok(rc); + changed + }); + if !self.mapping.is_closed() { + res } else { Err(Error::new( eyre!("VHost Service Thread has exited"), @@ -594,34 +637,22 @@ impl VHostServer { )) } } - fn gc(&self, hostname: Option) -> Result<(), Error> { - if let Some(mapping) = Weak::upgrade(&self.mapping) { - mapping.mutate(|writable| { - let mut targets = writable.remove(&hostname).unwrap_or_default(); - targets = targets - .into_iter() - .filter(|(_, rc)| rc.strong_count() > 0) - .collect(); - if !targets.is_empty() { - writable.insert(hostname, targets); - } - Ok(()) - }) - } else { - Err(Error::new( - eyre!("VHost Service Thread has exited"), - crate::ErrorKind::Network, - )) - } + fn gc(&self, hostname: Option) { + self.mapping.send_if_modified(|writable| { + let mut targets = writable.remove(&hostname).unwrap_or_default(); + let pre = targets.len(); + targets = targets + .into_iter() + .filter(|(_, rc)| rc.strong_count() > 0) + .collect(); + let post = targets.len(); + if !targets.is_empty() { + writable.insert(hostname, targets); + } + pre == post + }); } - fn is_empty(&self) -> Result { - if let Some(mapping) = Weak::upgrade(&self.mapping) { - Ok(mapping.peek(|m| m.is_empty())) - } else { - Err(Error::new( - eyre!("VHost Service Thread has exited"), - crate::ErrorKind::Network, - )) - } + fn is_empty(&self) -> bool { + self.mapping.borrow().is_empty() } } diff --git a/core/startos/src/net/wifi.rs b/core/startos/src/net/wifi.rs index 5c879399c..b2e59c20f 100644 --- a/core/startos/src/net/wifi.rs +++ b/core/startos/src/net/wifi.rs @@ -298,7 +298,7 @@ fn display_wifi_info(params: WithIoFormat, info: WifiListInfo) { let mut table_global = Table::new(); table_global.add_row(row![bc => "CONNECTED", - "SIGNAL_STRENGTH", + "SIGNAL STRENGTH", "COUNTRY", "ETHERNET", ]); @@ -306,12 +306,12 @@ fn display_wifi_info(params: WithIoFormat, info: WifiListInfo) { &info .connected .as_ref() - .map_or("[N/A]".to_owned(), |c| c.0.clone()), + .map_or("N/A".to_owned(), |c| c.0.clone()), &info .connected .as_ref() .and_then(|x| info.ssids.get(x)) - .map_or("[N/A]".to_owned(), |ss| format!("{}", ss.0)), + .map_or("N/A".to_owned(), |ss| format!("{}", ss.0)), info.country.as_ref().map(|c| c.alpha2()).unwrap_or("00"), &format!("{}", info.ethernet) ]); diff --git a/core/startos/src/prelude.rs b/core/startos/src/prelude.rs index a6a78a58d..702d77c2d 100644 --- a/core/startos/src/prelude.rs +++ b/core/startos/src/prelude.rs @@ -9,9 +9,17 @@ pub use crate::error::{Error, ErrorCollection, ErrorKind, ResultExt}; #[macro_export] macro_rules! dbg { + () => {{ + tracing::debug!("[{}:{}:{}]", file!(), line!(), column!()); + }}; ($e:expr) => {{ let e = $e; - tracing::debug!("[{}:{}:{}] $e = {e:?}", file!(), line!(), column!()); + tracing::debug!("[{}:{}:{}] {} = {e:?}", file!(), line!(), column!(), stringify!($e)); e }}; + ($($e:expr),+) => { + ($( + crate::dbg!($e) + ),+) + } } diff --git a/core/startos/src/registry/context.rs b/core/startos/src/registry/context.rs index 16c6465ed..d78fde50e 100644 --- a/core/startos/src/registry/context.rs +++ b/core/startos/src/registry/context.rs @@ -19,7 +19,6 @@ use crate::context::config::{ContextConfig, CONFIG_PATH}; use crate::context::{CliContext, RpcContext}; use crate::prelude::*; use crate::registry::auth::{SignatureHeader, AUTH_SIG_HEADER}; -use crate::registry::device_info::{DeviceInfo, DEVICE_INFO_HEADER}; use crate::registry::signer::sign::AnySigningKey; use crate::registry::RegistryDatabase; use crate::rpc_continuations::RpcContinuations; diff --git a/core/startos/src/registry/mod.rs b/core/startos/src/registry/mod.rs index 0cbbce4e0..9bb605d92 100644 --- a/core/startos/src/registry/mod.rs +++ b/core/startos/src/registry/mod.rs @@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet}; use axum::Router; use futures::future::ready; -use imbl_value::InternedString; use models::DataUrl; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler, Server}; use serde::{Deserialize, Serialize}; @@ -17,7 +16,7 @@ use crate::registry::auth::Auth; use crate::registry::context::RegistryContext; use crate::registry::device_info::DeviceInfoMiddleware; use crate::registry::os::index::OsIndex; -use crate::registry::package::index::{Category, PackageIndex}; +use crate::registry::package::index::PackageIndex; use crate::registry::signer::SignerInfo; use crate::rpc_continuations::Guid; use crate::util::serde::HandlerExtSerde; diff --git a/core/startos/src/registry/signer/commitment/merkle_archive.rs b/core/startos/src/registry/signer/commitment/merkle_archive.rs index 1b9d7d1e0..b27fb7ef4 100644 --- a/core/startos/src/registry/signer/commitment/merkle_archive.rs +++ b/core/startos/src/registry/signer/commitment/merkle_archive.rs @@ -24,10 +24,10 @@ impl MerkleArchiveCommitment { pub fn from_query(query: &str) -> Result, Error> { let mut root_sighash = None; let mut root_maxsize = None; - for (k, v) in form_urlencoded::parse(dbg!(query).as_bytes()) { + for (k, v) in form_urlencoded::parse(query.as_bytes()) { match &*k { "rootSighash" => { - root_sighash = Some(dbg!(v).parse()?); + root_sighash = Some(v.parse()?); } "rootMaxsize" => { root_maxsize = Some(v.parse()?); diff --git a/core/startos/src/service/effects/callbacks.rs b/core/startos/src/service/effects/callbacks.rs index 65eb707d8..19946672c 100644 --- a/core/startos/src/service/effects/callbacks.rs +++ b/core/startos/src/service/effects/callbacks.rs @@ -294,7 +294,7 @@ impl CallbackHandler { } } pub async fn call(mut self, args: Vector) -> Result<(), Error> { - dbg!(eyre!("callback fired: {}", self.handle.is_active())); + crate::dbg!(eyre!("callback fired: {}", self.handle.is_active())); if let Some(seed) = self.seed.upgrade() { seed.persistent_container .callback(self.handle.take(), args) diff --git a/core/startos/src/service/effects/net/ssl.rs b/core/startos/src/service/effects/net/ssl.rs index d37a2d241..66b4fa1e6 100644 --- a/core/startos/src/service/effects/net/ssl.rs +++ b/core/startos/src/service/effects/net/ssl.rs @@ -51,10 +51,16 @@ pub async fn get_ssl_certificate( .iter() .map(|(_, m)| m.as_hosts().as_entries()) .flatten_ok() - .map_ok(|(_, m)| m.as_addresses().de()) + .map_ok(|(_, m)| { + Ok(m.as_onions() + .de()? + .iter() + .map(InternedString::from_display) + .chain(m.as_domains().keys()?) + .collect::>()) + }) .map(|a| a.and_then(|a| a)) .flatten_ok() - .map_ok(|a| InternedString::from_display(&a)) .try_collect::<_, BTreeSet<_>, _>()?; for hostname in &hostnames { if let Some(internal) = hostname @@ -135,10 +141,16 @@ pub async fn get_ssl_key( .into_iter() .map(|m| m.as_hosts().as_entries()) .flatten_ok() - .map_ok(|(_, m)| m.as_addresses().de()) + .map_ok(|(_, m)| { + Ok(m.as_onions() + .de()? + .iter() + .map(InternedString::from_display) + .chain(m.as_domains().keys()?) + .collect::>()) + }) .map(|a| a.and_then(|a| a)) .flatten_ok() - .map_ok(|a| InternedString::from_display(&a)) .try_collect::<_, BTreeSet<_>, _>()?; for hostname in &hostnames { if let Some(internal) = hostname diff --git a/core/startos/src/service/effects/store.rs b/core/startos/src/service/effects/store.rs index 1d4a07086..39166c333 100644 --- a/core/startos/src/service/effects/store.rs +++ b/core/startos/src/service/effects/store.rs @@ -26,7 +26,7 @@ pub async fn get_store( callback, }: GetStoreParams, ) -> Result { - dbg!(&callback); + crate::dbg!(&callback); let context = context.deref()?; let peeked = context.seed.ctx.db.peek().await; let package_id = package_id.unwrap_or(context.seed.id.clone()); diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index d2e293909..6242e3b12 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -934,7 +934,6 @@ pub async fn attach( .with_kind(ErrorKind::Network)?; current_out = "stdout"; } - dbg!(¤t_out); ws.send(Message::Binary(out)) .await .with_kind(ErrorKind::Network)?; @@ -948,7 +947,6 @@ pub async fn attach( .with_kind(ErrorKind::Network)?; current_out = "stderr"; } - dbg!(¤t_out); ws.send(Message::Binary(err)) .await .with_kind(ErrorKind::Network)?; diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index 13cb7688c..c99e1cac6 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -452,7 +452,7 @@ impl PersistentContainer { #[instrument(skip_all)] pub async fn exit(mut self) -> Result<(), Error> { if let Some(destroy) = self.destroy(false) { - dbg!(destroy.await)?; + destroy.await?; } tracing::info!("Service for {} exited", self.s9pk.as_manifest().id); diff --git a/core/startos/src/service/rpc.rs b/core/startos/src/service/rpc.rs index f008de5c7..61eb5d592 100644 --- a/core/startos/src/service/rpc.rs +++ b/core/startos/src/service/rpc.rs @@ -155,7 +155,7 @@ impl serde::Serialize for Sandbox { pub struct CallbackId(u64); impl CallbackId { pub fn register(self, container: &PersistentContainer) -> CallbackHandle { - dbg!(eyre!( + crate::dbg!(eyre!( "callback {} registered for {}", self.0, container.s9pk.as_manifest().id diff --git a/core/startos/src/service/transition/restore.rs b/core/startos/src/service/transition/restore.rs index 1c4020ea4..08f3be942 100644 --- a/core/startos/src/service/transition/restore.rs +++ b/core/startos/src/service/transition/restore.rs @@ -48,7 +48,7 @@ impl Handler for ServiceActor { Ok::<_, Error>(()) } .map(|x| { - if let Err(err) = dbg!(x) { + if let Err(err) = x { tracing::debug!("{:?}", err); tracing::warn!("{}", err); } diff --git a/core/startos/src/util/rpc_client.rs b/core/startos/src/util/rpc_client.rs index fc93e4c64..82ce11e20 100644 --- a/core/startos/src/util/rpc_client.rs +++ b/core/startos/src/util/rpc_client.rs @@ -47,7 +47,7 @@ impl RpcClient { let mut lines = BufReader::new(reader).lines(); while let Some(line) = lines.next_line().await.transpose() { match line.map_err(Error::from).and_then(|l| { - serde_json::from_str::(dbg!(&l)) + serde_json::from_str::(crate::dbg!(&l)) .with_kind(ErrorKind::Deserialization) }) { Ok(l) => { @@ -114,7 +114,7 @@ impl RpcClient { let (send, recv) = oneshot::channel(); w.lock().await.insert(id.clone(), send); self.writer - .write_all((dbg!(serde_json::to_string(&request))? + "\n").as_bytes()) + .write_all((crate::dbg!(serde_json::to_string(&request))? + "\n").as_bytes()) .await .map_err(|e| { let mut err = rpc_toolkit::yajrc::INTERNAL_ERROR.clone(); @@ -154,7 +154,7 @@ impl RpcClient { params, }; self.writer - .write_all((dbg!(serde_json::to_string(&request))? + "\n").as_bytes()) + .write_all((crate::dbg!(serde_json::to_string(&request))? + "\n").as_bytes()) .await .map_err(|e| { let mut err = rpc_toolkit::yajrc::INTERNAL_ERROR.clone(); diff --git a/core/startos/src/version/v0_3_6_alpha_0.rs b/core/startos/src/version/v0_3_6_alpha_0.rs index 7a6045a3a..64c7d2e12 100644 --- a/core/startos/src/version/v0_3_6_alpha_0.rs +++ b/core/startos/src/version/v0_3_6_alpha_0.rs @@ -191,7 +191,6 @@ async fn init_postgres(datadir: impl AsRef) -> Result { .run(&secret_store) .await .with_kind(crate::ErrorKind::Database)?; - dbg!("Init Postgres Done"); Ok(secret_store) } @@ -315,7 +314,6 @@ impl VersionT for Version { "private": private, }); - dbg!("Should be done with the up"); *db = next; Ok(()) } diff --git a/sdk/base/lib/osBindings/HostAddress.ts b/sdk/base/lib/osBindings/AcmeProvider.ts similarity index 51% rename from sdk/base/lib/osBindings/HostAddress.ts rename to sdk/base/lib/osBindings/AcmeProvider.ts index 73b46d8e5..0ad3f0052 100644 --- a/sdk/base/lib/osBindings/HostAddress.ts +++ b/sdk/base/lib/osBindings/AcmeProvider.ts @@ -1,5 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type HostAddress = - | { kind: "onion"; address: string } - | { kind: "domain"; address: string } +export type AcmeProvider = string diff --git a/sdk/base/lib/osBindings/AcmeSettings.ts b/sdk/base/lib/osBindings/AcmeSettings.ts index bdf151ec7..44e70d9df 100644 --- a/sdk/base/lib/osBindings/AcmeSettings.ts +++ b/sdk/base/lib/osBindings/AcmeSettings.ts @@ -1,13 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type AcmeSettings = { - provider: string - /** - * email addresses for letsencrypt - */ - contact: Array - /** - * domains to get letsencrypt certs for - */ - domains: string[] -} +export type AcmeSettings = { contact: Array } diff --git a/sdk/base/lib/osBindings/DomainConfig.ts b/sdk/base/lib/osBindings/DomainConfig.ts new file mode 100644 index 000000000..433bc65f5 --- /dev/null +++ b/sdk/base/lib/osBindings/DomainConfig.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { AcmeProvider } from "./AcmeProvider" + +export type DomainConfig = { public: boolean; acme: AcmeProvider | null } diff --git a/sdk/base/lib/osBindings/ForgetInterfaceParams.ts b/sdk/base/lib/osBindings/ForgetInterfaceParams.ts new file mode 100644 index 000000000..b3532602c --- /dev/null +++ b/sdk/base/lib/osBindings/ForgetInterfaceParams.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ForgetInterfaceParams = { interface: string } diff --git a/sdk/base/lib/osBindings/Host.ts b/sdk/base/lib/osBindings/Host.ts index 7d8cf3a90..5436e9955 100644 --- a/sdk/base/lib/osBindings/Host.ts +++ b/sdk/base/lib/osBindings/Host.ts @@ -1,13 +1,14 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { BindInfo } from "./BindInfo" -import type { HostAddress } from "./HostAddress" +import type { DomainConfig } from "./DomainConfig" import type { HostKind } from "./HostKind" import type { HostnameInfo } from "./HostnameInfo" export type Host = { kind: HostKind bindings: { [key: number]: BindInfo } - addresses: Array + onions: string[] + domains: { [key: string]: DomainConfig } /** * COMPUTED: NetService::update */ diff --git a/sdk/base/lib/osBindings/IpInfo.ts b/sdk/base/lib/osBindings/IpInfo.ts index 0933b80e6..e9b0c9fb3 100644 --- a/sdk/base/lib/osBindings/IpInfo.ts +++ b/sdk/base/lib/osBindings/IpInfo.ts @@ -4,4 +4,5 @@ export type IpInfo = { scopeId: number subnets: string[] wanIp: string | null + ntpServers: string[] } diff --git a/sdk/base/lib/osBindings/ServerInfo.ts b/sdk/base/lib/osBindings/ServerInfo.ts index b5d5be293..6eec14745 100644 --- a/sdk/base/lib/osBindings/ServerInfo.ts +++ b/sdk/base/lib/osBindings/ServerInfo.ts @@ -1,4 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { AcmeProvider } from "./AcmeProvider" import type { AcmeSettings } from "./AcmeSettings" import type { Governor } from "./Governor" import type { LshwDevice } from "./LshwDevice" @@ -23,7 +24,7 @@ export type ServerInfo = { */ torAddress: string networkInterfaces: { [key: string]: NetworkInterfaceInfo } - acme: AcmeSettings | null + acme: { [key: AcmeProvider]: AcmeSettings } statusInfo: ServerStatus wifi: WifiInfo unreadNotificationCount: number diff --git a/sdk/base/lib/osBindings/index.ts b/sdk/base/lib/osBindings/index.ts index 623ebc23a..b230aea31 100644 --- a/sdk/base/lib/osBindings/index.ts +++ b/sdk/base/lib/osBindings/index.ts @@ -1,4 +1,5 @@ export { AcceptSigners } from "./AcceptSigners" +export { AcmeProvider } from "./AcmeProvider" export { AcmeSettings } from "./AcmeSettings" export { ActionId } from "./ActionId" export { ActionInput } from "./ActionInput" @@ -66,6 +67,7 @@ export { DepInfo } from "./DepInfo" export { Description } from "./Description" export { DestroySubcontainerFsParams } from "./DestroySubcontainerFsParams" export { DeviceFilter } from "./DeviceFilter" +export { DomainConfig } from "./DomainConfig" export { Duration } from "./Duration" export { EchoParams } from "./EchoParams" export { EditSignerParams } from "./EditSignerParams" @@ -73,6 +75,7 @@ export { EncryptedWire } from "./EncryptedWire" export { ExportActionParams } from "./ExportActionParams" export { ExportServiceInterfaceParams } from "./ExportServiceInterfaceParams" export { ExposeForDependentsParams } from "./ExposeForDependentsParams" +export { ForgetInterfaceParams } from "./ForgetInterfaceParams" export { FullIndex } from "./FullIndex" export { FullProgress } from "./FullProgress" export { GetActionInputParams } from "./GetActionInputParams" @@ -95,7 +98,6 @@ export { Governor } from "./Governor" export { Guid } from "./Guid" export { HardwareRequirements } from "./HardwareRequirements" export { HealthCheckId } from "./HealthCheckId" -export { HostAddress } from "./HostAddress" export { HostId } from "./HostId" export { HostKind } from "./HostKind" export { HostnameInfo } from "./HostnameInfo" diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index 9fd777da8..0e6477266 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -48,6 +48,7 @@ export const mockPatchData: DataModel = { scopeId: 1, subnets: ['10.0.0.1/24'], wanIp: null, + ntpServers: [], }, }, wlan0: { @@ -59,6 +60,7 @@ export const mockPatchData: DataModel = { 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64', ], wanIp: null, + ntpServers: [], }, }, }, From 926ef71eaba8b095b92d87be90458d23c440b08f Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 16 Dec 2024 16:40:34 -0700 Subject: [PATCH 18/29] sdk version bump --- sdk/package/package-lock.json | 4 ++-- sdk/package/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/package/package-lock.json b/sdk/package/package-lock.json index bbf15313b..50efc6b57 100644 --- a/sdk/package/package-lock.json +++ b/sdk/package/package-lock.json @@ -1,12 +1,12 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.1", + "version": "0.3.6-beta.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.0", + "version": "0.3.6-beta.2", "license": "MIT", "dependencies": { "@iarna/toml": "^2.2.5", diff --git a/sdk/package/package.json b/sdk/package/package.json index d54276f8a..6750ad372 100644 --- a/sdk/package/package.json +++ b/sdk/package/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.1", + "version": "0.3.6-beta.2", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./package/lib/index.js", "types": "./package/lib/index.d.ts", From 5908ec4779cea172dba737c5df4d83f8de8c7f80 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 18 Dec 2024 14:39:15 -0700 Subject: [PATCH 19/29] migration --- core/startos/src/version/mod.rs | 7 +- core/startos/src/version/v0_3_6_alpha_10.rs | 95 +++++++++++++++++++++ sdk/base/lib/util/getServiceInterface.ts | 11 +-- 3 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 core/startos/src/version/v0_3_6_alpha_10.rs diff --git a/core/startos/src/version/mod.rs b/core/startos/src/version/mod.rs index 94b61176b..2fd3e492b 100644 --- a/core/startos/src/version/mod.rs +++ b/core/startos/src/version/mod.rs @@ -29,7 +29,9 @@ mod v0_3_6_alpha_7; mod v0_3_6_alpha_8; mod v0_3_6_alpha_9; -pub type Current = v0_3_6_alpha_9::Version; // VERSION_BUMP +mod v0_3_6_alpha_10; + +pub type Current = v0_3_6_alpha_10::Version; // VERSION_BUMP impl Current { #[instrument(skip(self, db))] @@ -108,6 +110,7 @@ enum Version { V0_3_6_alpha_7(Wrapper), V0_3_6_alpha_8(Wrapper), V0_3_6_alpha_9(Wrapper), + V0_3_6_alpha_10(Wrapper), Other(exver::Version), } @@ -141,6 +144,7 @@ impl Version { Self::V0_3_6_alpha_7(v) => DynVersion(Box::new(v.0)), Self::V0_3_6_alpha_8(v) => DynVersion(Box::new(v.0)), Self::V0_3_6_alpha_9(v) => DynVersion(Box::new(v.0)), + Self::V0_3_6_alpha_10(v) => DynVersion(Box::new(v.0)), Self::Other(v) => { return Err(Error::new( eyre!("unknown version {v}"), @@ -166,6 +170,7 @@ impl Version { Version::V0_3_6_alpha_7(Wrapper(x)) => x.semver(), Version::V0_3_6_alpha_8(Wrapper(x)) => x.semver(), Version::V0_3_6_alpha_9(Wrapper(x)) => x.semver(), + Version::V0_3_6_alpha_10(Wrapper(x)) => x.semver(), Version::Other(x) => x.clone(), } } diff --git a/core/startos/src/version/v0_3_6_alpha_10.rs b/core/startos/src/version/v0_3_6_alpha_10.rs new file mode 100644 index 000000000..9b7ed3892 --- /dev/null +++ b/core/startos/src/version/v0_3_6_alpha_10.rs @@ -0,0 +1,95 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use exver::{PreReleaseSegment, VersionRange}; +use imbl_value::InternedString; +use serde::{Deserialize, Serialize}; +use torut::onion::OnionAddressV3; + +use super::v0_3_5::V0_3_0_COMPAT; +use super::{v0_3_6_alpha_9, VersionT}; +use crate::db::model::Database; +use crate::net::host::address::DomainConfig; +use crate::prelude::*; + +lazy_static::lazy_static! { + static ref V0_3_6_alpha_10: exver::Version = exver::Version::new( + [0, 3, 6], + [PreReleaseSegment::String("alpha".into()), 10.into()] + ); +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "camelCase")] +#[serde(tag = "kind")] +enum HostAddress { + Onion { address: OnionAddressV3 }, + Domain { address: InternedString }, +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct Version; + +impl VersionT for Version { + type Previous = v0_3_6_alpha_9::Version; + type PreUpRes = (); + + async fn pre_up(self) -> Result { + Ok(()) + } + fn semver(self) -> exver::Version { + V0_3_6_alpha_10.clone() + } + fn compat(self) -> &'static VersionRange { + &V0_3_0_COMPAT + } + fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> { + for (package, data) in db["public"]["packageData"] + .as_object_mut() + .ok_or_else(|| { + Error::new( + eyre!("expected public.packageData to be an object"), + ErrorKind::Database, + ) + })? + .iter_mut() + { + for (host_id, host) in data["hosts"] + .as_object_mut() + .ok_or_else(|| { + Error::new( + eyre!("expected public.packageData to be an object"), + ErrorKind::Database, + ) + })? + .iter_mut() + { + let mut onions = BTreeSet::new(); + let mut domains = BTreeMap::new(); + let addresses = from_value::>(host["addresses"].clone())?; + for address in addresses { + match address { + HostAddress::Onion { address } => { + onions.insert(address); + } + HostAddress::Domain { address } => { + domains.insert( + address, + DomainConfig { + public: true, + acme: None, + }, + ); + } + } + } + host["onions"] = to_value(&onions)?; + host["domains"] = to_value(&domains)?; + } + } + + Ok(()) + } + fn down(self, _db: &mut Value) -> Result<(), Error> { + Ok(()) + } +} diff --git a/sdk/base/lib/util/getServiceInterface.ts b/sdk/base/lib/util/getServiceInterface.ts index cbbb345cb..d1656d7ed 100644 --- a/sdk/base/lib/util/getServiceInterface.ts +++ b/sdk/base/lib/util/getServiceInterface.ts @@ -1,15 +1,6 @@ import { ServiceInterfaceType } from "../types" import { knownProtocols } from "../interfaces/Host" -import { - AddressInfo, - Host, - HostAddress, - Hostname, - HostnameInfo, - HostnameInfoIp, - HostnameInfoOnion, - IpInfo, -} from "../types" +import { AddressInfo, Host, Hostname, HostnameInfo } from "../types" import { Effects } from "../Effects" export type UrlString = string From 3fe54fcf7ce17ebc815523fc5fb239d311165b36 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 19 Dec 2024 10:49:00 -0700 Subject: [PATCH 20/29] misc fixes --- .../src/Adapters/EffectCreator.ts | 6 -- .../Systems/SystemForEmbassy/index.ts | 1 - core/Cargo.lock | 2 +- core/startos/Cargo.toml | 2 +- core/startos/src/net/host/mod.rs | 1 - core/startos/src/net/mod.rs | 4 + core/startos/src/net/net_controller.rs | 1 + core/startos/src/net/network_interface.rs | 6 ++ core/startos/src/net/service_interface.rs | 1 - core/startos/src/net/vhost.rs | 93 +++++++++++++++++-- core/startos/src/service/effects/mod.rs | 4 - core/startos/src/service/effects/net/host.rs | 24 ----- .../src/service/effects/net/interface.rs | 3 - core/startos/src/version/v0_3_6_alpha_10.rs | 6 +- sdk/base/lib/Effects.ts | 6 -- .../lib/actions/input/builder/inputSpec.ts | 4 +- sdk/base/lib/actions/input/builder/value.ts | 3 + sdk/base/lib/interfaces/Origin.ts | 2 - .../lib/interfaces/ServiceInterfaceBuilder.ts | 1 - .../ExportServiceInterfaceParams.ts | 1 - .../lib/osBindings/GetPrimaryUrlParams.ts | 10 -- sdk/base/lib/osBindings/ServiceInterface.ts | 1 - sdk/base/lib/osBindings/index.ts | 1 - .../lib/test/startosTypeValidation.test.ts | 2 - sdk/base/lib/util/getServiceInterface.ts | 16 ---- sdk/base/lib/util/getServiceInterfaces.ts | 12 --- sdk/package/lib/StartSdk.ts | 8 -- sdk/package/lib/test/host.test.ts | 1 - web/package-lock.json | 4 +- web/package.json | 2 +- .../ui/src/app/services/api/api.fixures.ts | 16 ++-- .../ui/src/app/services/api/mock-patch.ts | 17 ++-- 32 files changed, 121 insertions(+), 140 deletions(-) delete mode 100644 sdk/base/lib/osBindings/GetPrimaryUrlParams.ts diff --git a/container-runtime/src/Adapters/EffectCreator.ts b/container-runtime/src/Adapters/EffectCreator.ts index 0123b0cbc..4bda0ed5d 100644 --- a/container-runtime/src/Adapters/EffectCreator.ts +++ b/container-runtime/src/Adapters/EffectCreator.ts @@ -216,12 +216,6 @@ export function makeEffects(context: EffectContext): Effects { }) as ReturnType }, - getPrimaryUrl(...[options]: Parameters) { - return rpcRound("get-primary-url", { - ...options, - callback: context.callbacks?.addCallback(options.callback) || null, - }) as ReturnType - }, getServicePortForward( ...[options]: Parameters ) { diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index 531b30cd2..e74ef317d 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -425,7 +425,6 @@ export class SystemForEmbassy implements System { name: interfaceValue.name, id: `${id}-${internal}`, description: interfaceValue.description, - hasPrimary: false, type: interfaceValue.ui && (origin.scheme === "http" || origin.sslScheme === "https") diff --git a/core/Cargo.lock b/core/Cargo.lock index 4a7914999..7d8f06d1c 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5645,7 +5645,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "start-os" -version = "0.3.6-alpha.9" +version = "0.3.6-alpha.10" dependencies = [ "aes", "async-acme", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 7ee26dbca..3e312c295 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -14,7 +14,7 @@ keywords = [ name = "start-os" readme = "README.md" repository = "https://github.com/Start9Labs/start-os" -version = "0.3.6-alpha.9" +version = "0.3.6-alpha.10" license = "MIT" [lib] diff --git a/core/startos/src/net/host/mod.rs b/core/startos/src/net/host/mod.rs index a9ae31c8f..9f4194866 100644 --- a/core/startos/src/net/host/mod.rs +++ b/core/startos/src/net/host/mod.rs @@ -49,7 +49,6 @@ impl Host { } } pub fn addresses<'a>(&'a self) -> impl Iterator + 'a { - // TODO: handle primary self.onions .iter() .cloned() diff --git a/core/startos/src/net/mod.rs b/core/startos/src/net/mod.rs index 53ba39887..fc0236300 100644 --- a/core/startos/src/net/mod.rs +++ b/core/startos/src/net/mod.rs @@ -38,4 +38,8 @@ pub fn net() -> ParentHandler { network_interface::network_interface_api::() .with_about("View and edit network interface configurations"), ) + .subcommand( + "vhost", + vhost::vhost_api::().with_about("Manage ssl virtual host proxy"), + ) } diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 891f6a9a5..b08b61c6e 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -13,6 +13,7 @@ use tracing::instrument; use crate::db::model::Database; use crate::error::ErrorCollection; use crate::hostname::Hostname; +use crate::net::acme::AcmeProvider; use crate::net::dns::DnsController; use crate::net::forward::LanPortForwardController; use crate::net::host::address::HostAddress; diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 9f7439bfd..5087f0085 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -755,16 +755,20 @@ impl NetworkInterfaceController { } struct ListenerMap { + prev_public: bool, port: u16, listeners: BTreeMap<(IpAddr, u32), (TcpListener, bool, Option)>, } impl ListenerMap { fn new(port: u16) -> Self { Self { + prev_public: false, port, listeners: BTreeMap::new(), } } + + #[instrument(skip(self))] async fn update( &mut self, ip_info: &BTreeMap, @@ -815,6 +819,7 @@ impl ListenerMap { } } self.listeners.retain(|key, _| keep.contains(key)); + self.prev_public = public; Ok(()) } fn accept(&mut self) -> ListenerMapFut { @@ -848,6 +853,7 @@ impl NetworkInterfaceListener { } pub async fn accept(&mut self, public: bool) -> Result { + self.needs_update |= public != self.listeners.prev_public; loop { if self.needs_update { let ip_info = self.ip_info.borrow().clone(); diff --git a/core/startos/src/net/service_interface.rs b/core/startos/src/net/service_interface.rs index b0a8a7676..95d7115cd 100644 --- a/core/startos/src/net/service_interface.rs +++ b/core/startos/src/net/service_interface.rs @@ -70,7 +70,6 @@ pub struct ServiceInterface { pub id: ServiceInterfaceId, pub name: String, pub description: String, - pub has_primary: bool, pub masked: bool, pub address_info: AddressInfo, #[serde(rename = "type")] diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs index 809effc7d..70170584d 100644 --- a/core/startos/src/net/vhost.rs +++ b/core/startos/src/net/vhost.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::net::{IpAddr, SocketAddr}; use std::str::FromStr; use std::sync::{Arc, Weak}; @@ -14,6 +14,7 @@ use helpers::NonDetachingJoinHandle; use http::Uri; use imbl_value::InternedString; use models::ResultExt; +use rpc_toolkit::{from_fn, from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; @@ -31,6 +32,7 @@ use tokio_stream::StreamExt; use tracing::instrument; use ts_rs::TS; +use crate::context::{CliContext, RpcContext}; use crate::db::model::Database; use crate::net::acme::{AcmeCertCache, AcmeProvider}; use crate::net::network_interface::{ @@ -39,9 +41,52 @@ use crate::net::network_interface::{ use crate::net::static_server::server_error; use crate::prelude::*; use crate::util::io::BackTrackingIO; -use crate::util::serde::MaybeUtf8String; +use crate::util::serde::{display_serializable, HandlerExtSerde, MaybeUtf8String}; use crate::util::sync::SyncMutex; +pub fn vhost_api() -> ParentHandler { + ParentHandler::new().subcommand( + "dump-table", + from_fn(|ctx: RpcContext| Ok(ctx.net_controller.vhost.dump_table())) + .with_display_serializable() + .with_custom_display_fn(|HandlerArgs { params, .. }, res| { + use prettytable::*; + + if let Some(format) = params.format { + display_serializable(format, res); + return Ok::<_, Error>(()); + } + + let mut table = Table::new(); + table.add_row(row![bc => "FROM", "TO", "PUBLIC", "ACME", "CONNECT SSL", "ACTIVE"]); + + for (external, targets) in res { + for (host, targets) in targets { + for (idx, target) in targets.into_iter().enumerate() { + table.add_row(row![ + format!( + "{}:{}", + host.as_ref().map(|s| &**s).unwrap_or("*"), + external.0 + ), + target.addr, + target.public, + target.acme.as_ref().map(|a| a.0.as_str()).unwrap_or("NONE"), + target.connect_ssl.is_ok(), + idx == 0 + ]); + } + } + } + + table.print_tty(false)?; + + Ok(()) + }) + .with_call_remote::(), + ) +} + #[derive(Debug)] struct SingleCertResolver(Arc); impl ResolvesServerCert for SingleCertResolver { @@ -77,7 +122,7 @@ impl VHostController { public: bool, acme: Option, target: SocketAddr, - connect_ssl: Result<(), AlpnInfo>, // Ok: yes, connect using ssl, pass through alpn; Err: connect tcp, use provided strategy for alpn + connect_ssl: Result<(), AlpnInfo>, ) -> Result, Error> { self.servers.mutate(|writable| { let server = if let Some(server) = writable.remove(&external) { @@ -104,6 +149,36 @@ impl VHostController { Ok(rc?) }) } + + pub fn dump_table( + &self, + ) -> BTreeMap, BTreeMap>, BTreeSet>> + { + self.servers.peek(|s| { + s.iter() + .map(|(k, v)| { + ( + JsonKey::new(*k), + v.mapping + .borrow() + .iter() + .map(|(k, v)| { + ( + JsonKey::new(k.clone()), + v.iter() + .filter(|(_, v)| v.strong_count() > 0) + .map(|(k, _)| k) + .cloned() + .collect(), + ) + }) + .collect(), + ) + }) + .collect() + }) + } + #[instrument(skip_all)] pub fn gc(&self, hostname: Option, external: u16) { self.servers.mutate(|writable| { @@ -117,12 +192,12 @@ impl VHostController { } } -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -struct TargetInfo { - public: bool, - acme: Option, - addr: SocketAddr, - connect_ssl: Result<(), AlpnInfo>, +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct TargetInfo { + pub public: bool, + pub acme: Option, + pub addr: SocketAddr, + pub connect_ssl: Result<(), AlpnInfo>, // Ok: yes, connect using ssl, pass through alpn; Err: connect tcp, use provided strategy for alpn } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)] diff --git a/core/startos/src/service/effects/mod.rs b/core/startos/src/service/effects/mod.rs index f68985268..e9df6f9f2 100644 --- a/core/startos/src/service/effects/mod.rs +++ b/core/startos/src/service/effects/mod.rs @@ -130,10 +130,6 @@ pub fn handler() -> ParentHandler { "get-host-info", from_fn_async(net::host::get_host_info).no_cli(), ) - .subcommand( - "get-primary-url", - from_fn_async(net::host::get_primary_url).no_cli(), - ) .subcommand( "get-container-ip", from_fn_async(net::info::get_container_ip).no_cli(), diff --git a/core/startos/src/service/effects/net/host.rs b/core/startos/src/service/effects/net/host.rs index d320e7fe9..51f9eceec 100644 --- a/core/startos/src/service/effects/net/host.rs +++ b/core/startos/src/service/effects/net/host.rs @@ -6,30 +6,6 @@ use crate::service::effects::callbacks::CallbackHandler; use crate::service::effects::prelude::*; use crate::service::rpc::CallbackId; -#[derive(Debug, Clone, Serialize, Deserialize, TS)] -#[ts(export)] -#[serde(rename_all = "camelCase")] -pub struct GetPrimaryUrlParams { - #[ts(optional)] - package_id: Option, - host_id: HostId, - #[ts(optional)] - callback: Option, -} -pub async fn get_primary_url( - context: EffectContext, - GetPrimaryUrlParams { - package_id, - host_id, - callback, - }: GetPrimaryUrlParams, -) -> Result, Error> { - let context = context.deref()?; - let package_id = package_id.unwrap_or_else(|| context.seed.id.clone()); - - Ok(None) // TODO -} - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] #[serde(rename_all = "camelCase")] #[ts(export)] diff --git a/core/startos/src/service/effects/net/interface.rs b/core/startos/src/service/effects/net/interface.rs index 44258c36a..5de9638c4 100644 --- a/core/startos/src/service/effects/net/interface.rs +++ b/core/startos/src/service/effects/net/interface.rs @@ -15,7 +15,6 @@ pub struct ExportServiceInterfaceParams { id: ServiceInterfaceId, name: String, description: String, - has_primary: bool, masked: bool, address_info: AddressInfo, r#type: ServiceInterfaceType, @@ -26,7 +25,6 @@ pub async fn export_service_interface( id, name, description, - has_primary, masked, address_info, r#type, @@ -39,7 +37,6 @@ pub async fn export_service_interface( id: id.clone(), name, description, - has_primary, masked, address_info, interface_type: r#type, diff --git a/core/startos/src/version/v0_3_6_alpha_10.rs b/core/startos/src/version/v0_3_6_alpha_10.rs index 9b7ed3892..f65479488 100644 --- a/core/startos/src/version/v0_3_6_alpha_10.rs +++ b/core/startos/src/version/v0_3_6_alpha_10.rs @@ -43,7 +43,7 @@ impl VersionT for Version { &V0_3_0_COMPAT } fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> { - for (package, data) in db["public"]["packageData"] + for (_, package) in db["public"]["packageData"] .as_object_mut() .ok_or_else(|| { Error::new( @@ -53,11 +53,11 @@ impl VersionT for Version { })? .iter_mut() { - for (host_id, host) in data["hosts"] + for (_, host) in package["hosts"] .as_object_mut() .ok_or_else(|| { Error::new( - eyre!("expected public.packageData to be an object"), + eyre!("expected public.packageData[id].hosts to be an object"), ErrorKind::Database, ) })? diff --git a/sdk/base/lib/Effects.ts b/sdk/base/lib/Effects.ts index a0eb08d67..dcb03af4e 100644 --- a/sdk/base/lib/Effects.ts +++ b/sdk/base/lib/Effects.ts @@ -129,12 +129,6 @@ export type Effects = { hostId: HostId callback?: () => void }): Promise - /** Returns the primary url that a user has selected for a host, if it exists */ - getPrimaryUrl(options: { - packageId?: PackageId - hostId: HostId - callback?: () => void - }): Promise /** Returns the IP address of the container */ getContainerIp(): Promise // interface diff --git a/sdk/base/lib/actions/input/builder/inputSpec.ts b/sdk/base/lib/actions/input/builder/inputSpec.ts index 31e06df4f..5d4d5c6bb 100644 --- a/sdk/base/lib/actions/input/builder/inputSpec.ts +++ b/sdk/base/lib/actions/input/builder/inputSpec.ts @@ -94,8 +94,8 @@ export class InputSpec, Store = never> { }, public validator: Parser, ) {} - _TYPE: Type = null as any as Type - _PARTIAL: DeepPartial = null as any as DeepPartial + public _TYPE: Type = null as any as Type + public _PARTIAL: DeepPartial = null as any as DeepPartial async build(options: LazyBuildOptions) { const answer = {} as { [K in keyof Type]: ValueSpec diff --git a/sdk/base/lib/actions/input/builder/value.ts b/sdk/base/lib/actions/input/builder/value.ts index 676c4aac1..3ff3c2d24 100644 --- a/sdk/base/lib/actions/input/builder/value.ts +++ b/sdk/base/lib/actions/input/builder/value.ts @@ -49,6 +49,9 @@ export class Value { public build: LazyBuild, public validator: Parser, ) {} + public _TYPE: Type = null as any as Type + public _PARTIAL: DeepPartial = null as any as DeepPartial + static toggle(a: { name: string description?: string | null diff --git a/sdk/base/lib/interfaces/Origin.ts b/sdk/base/lib/interfaces/Origin.ts index 5e12713e6..9985176e4 100644 --- a/sdk/base/lib/interfaces/Origin.ts +++ b/sdk/base/lib/interfaces/Origin.ts @@ -46,7 +46,6 @@ export class Origin { const { name, description, - hasPrimary, id, type, username, @@ -67,7 +66,6 @@ export class Origin { id, name, description, - hasPrimary, addressInfo, type, masked, diff --git a/sdk/base/lib/interfaces/ServiceInterfaceBuilder.ts b/sdk/base/lib/interfaces/ServiceInterfaceBuilder.ts index 4ef294b4f..036180ad3 100644 --- a/sdk/base/lib/interfaces/ServiceInterfaceBuilder.ts +++ b/sdk/base/lib/interfaces/ServiceInterfaceBuilder.ts @@ -20,7 +20,6 @@ export class ServiceInterfaceBuilder { name: string id: string description: string - hasPrimary: boolean type: ServiceInterfaceType username: string | null path: string diff --git a/sdk/base/lib/osBindings/ExportServiceInterfaceParams.ts b/sdk/base/lib/osBindings/ExportServiceInterfaceParams.ts index 28ac89916..675c3e06d 100644 --- a/sdk/base/lib/osBindings/ExportServiceInterfaceParams.ts +++ b/sdk/base/lib/osBindings/ExportServiceInterfaceParams.ts @@ -7,7 +7,6 @@ export type ExportServiceInterfaceParams = { id: ServiceInterfaceId name: string description: string - hasPrimary: boolean masked: boolean addressInfo: AddressInfo type: ServiceInterfaceType diff --git a/sdk/base/lib/osBindings/GetPrimaryUrlParams.ts b/sdk/base/lib/osBindings/GetPrimaryUrlParams.ts deleted file mode 100644 index 06bf73976..000000000 --- a/sdk/base/lib/osBindings/GetPrimaryUrlParams.ts +++ /dev/null @@ -1,10 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { CallbackId } from "./CallbackId" -import type { HostId } from "./HostId" -import type { PackageId } from "./PackageId" - -export type GetPrimaryUrlParams = { - packageId?: PackageId - hostId: HostId - callback?: CallbackId -} diff --git a/sdk/base/lib/osBindings/ServiceInterface.ts b/sdk/base/lib/osBindings/ServiceInterface.ts index 9bcec0056..6a58675a4 100644 --- a/sdk/base/lib/osBindings/ServiceInterface.ts +++ b/sdk/base/lib/osBindings/ServiceInterface.ts @@ -7,7 +7,6 @@ export type ServiceInterface = { id: ServiceInterfaceId name: string description: string - hasPrimary: boolean masked: boolean addressInfo: AddressInfo type: ServiceInterfaceType diff --git a/sdk/base/lib/osBindings/index.ts b/sdk/base/lib/osBindings/index.ts index b230aea31..f3fec0b71 100644 --- a/sdk/base/lib/osBindings/index.ts +++ b/sdk/base/lib/osBindings/index.ts @@ -85,7 +85,6 @@ export { GetOsVersionParams } from "./GetOsVersionParams" export { GetPackageParams } from "./GetPackageParams" export { GetPackageResponseFull } from "./GetPackageResponseFull" export { GetPackageResponse } from "./GetPackageResponse" -export { GetPrimaryUrlParams } from "./GetPrimaryUrlParams" export { GetServiceInterfaceParams } from "./GetServiceInterfaceParams" export { GetServicePortForwardParams } from "./GetServicePortForwardParams" export { GetSslCertificateParams } from "./GetSslCertificateParams" diff --git a/sdk/base/lib/test/startosTypeValidation.test.ts b/sdk/base/lib/test/startosTypeValidation.test.ts index 509da0894..2de7b43a4 100644 --- a/sdk/base/lib/test/startosTypeValidation.test.ts +++ b/sdk/base/lib/test/startosTypeValidation.test.ts @@ -26,7 +26,6 @@ import { SetDependenciesParams } from ".././osBindings" import { GetSystemSmtpParams } from ".././osBindings" import { GetServicePortForwardParams } from ".././osBindings" import { ExportServiceInterfaceParams } from ".././osBindings" -import { GetPrimaryUrlParams } from ".././osBindings" import { ListServiceInterfacesParams } from ".././osBindings" import { ExportActionParams } from ".././osBindings" import { MountParams } from ".././osBindings" @@ -83,7 +82,6 @@ describe("startosTypeValidation ", () => { getServicePortForward: {} as GetServicePortForwardParams, clearServiceInterfaces: {} as ClearServiceInterfacesParams, exportServiceInterface: {} as ExportServiceInterfaceParams, - getPrimaryUrl: {} as WithCallback, listServiceInterfaces: {} as WithCallback, mount: {} as MountParams, checkDependencies: {} as CheckDependenciesParam, diff --git a/sdk/base/lib/util/getServiceInterface.ts b/sdk/base/lib/util/getServiceInterface.ts index d1656d7ed..393c4f9a7 100644 --- a/sdk/base/lib/util/getServiceInterface.ts +++ b/sdk/base/lib/util/getServiceInterface.ts @@ -39,8 +39,6 @@ export type ServiceInterfaceFilled = { name: string /** Human readable description, used as tooltip usually */ description: string - /** Whether or not the interface has a primary URL */ - hasPrimary: boolean /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */ masked: boolean /** Information about the host for this binding */ @@ -49,10 +47,6 @@ export type ServiceInterfaceFilled = { addressInfo: FilledAddressInfo | null /** Indicates if we are a ui/p2p/api for the kind of interface that this is representing */ type: ServiceInterfaceType - /** The primary hostname for the service, as chosen by the user */ - primaryHostname: Hostname | null - /** The primary URL for the service, as chosen by the user */ - primaryUrl: UrlString | null } const either = (...args: ((a: A) => boolean)[]) => @@ -191,23 +185,13 @@ const makeInterfaceFilled = async ({ hostId, callback, }) - const primaryUrl = await effects.getPrimaryUrl({ - hostId, - packageId, - callback, - }) const interfaceFilled: ServiceInterfaceFilled = { ...serviceInterfaceValue, - primaryUrl: primaryUrl, host, addressInfo: host ? filledAddress(host, serviceInterfaceValue.addressInfo) : null, - get primaryHostname() { - if (primaryUrl == null) return null - return getHostname(primaryUrl) - }, } return interfaceFilled } diff --git a/sdk/base/lib/util/getServiceInterfaces.ts b/sdk/base/lib/util/getServiceInterfaces.ts index 1d83684d6..faeb508b4 100644 --- a/sdk/base/lib/util/getServiceInterfaces.ts +++ b/sdk/base/lib/util/getServiceInterfaces.ts @@ -30,22 +30,10 @@ const makeManyInterfaceFilled = async ({ if (!host) { throw new Error(`host ${hostId} not found!`) } - const primaryUrl = await effects - .getPrimaryUrl({ - hostId, - packageId, - callback, - }) - .catch(() => null) return { ...serviceInterfaceValue, - primaryUrl: primaryUrl, host, addressInfo: filledAddress(host, serviceInterfaceValue.addressInfo), - get primaryHostname() { - if (primaryUrl == null) return null - return getHostname(primaryUrl) - }, } }), ) diff --git a/sdk/package/lib/StartSdk.ts b/sdk/package/lib/StartSdk.ts index e7e87f963..390a5fe31 100644 --- a/sdk/package/lib/StartSdk.ts +++ b/sdk/package/lib/StartSdk.ts @@ -102,7 +102,6 @@ export class StartSdk { | "clearServiceInterfaces" | "bind" | "getHostInfo" - | "getPrimaryUrl" type MainUsedEffects = "setMainStatus" | "setHealth" type CallbackEffects = "constRetry" | "clearCallbacks" type AlreadyExposed = "getSslCertificate" | "getSystemSmtp" @@ -379,7 +378,6 @@ export class StartSdk { id: 'ui', description: 'The primary web app for this service.', type: 'ui', - hasPrimary: false, masked: false, schemeOverride: null, username: null, @@ -397,8 +395,6 @@ export class StartSdk { id: string /** The human readable description. */ description: string - /** No effect until StartOS v0.4.0. If true, forces the user to select one URL (i.e. .onion, .local, or IP address) as the primary URL. This is needed by some services to function properly. */ - hasPrimary: boolean /** Affects how the interface appears to the user. One of: 'ui', 'api', 'p2p'. If 'ui', the user will see a "Launch UI" button */ type: ServiceInterfaceType /** (optional) prepends the provided username to all URLs. */ @@ -562,7 +558,6 @@ export class StartSdk { id: 'primary-ui', description: 'The primary web app for this service.', type: 'ui', - hasPrimary: false, masked: false, schemeOverride: null, username: null, @@ -575,7 +570,6 @@ export class StartSdk { id: 'admin-ui', description: 'The admin web app for this service.', type: 'ui', - hasPrimary: false, masked: false, schemeOverride: null, username: null, @@ -596,7 +590,6 @@ export class StartSdk { id: 'api', description: 'The advanced API for this service.', type: 'api', - hasPrimary: false, masked: false, schemeOverride: null, username: null, @@ -1269,7 +1262,6 @@ export class StartSdk { * @example default: 'radio1' */ default: keyof Variants & string - required: boolean /** * @description A mapping of unique radio options to their human readable display format. * @example diff --git a/sdk/package/lib/test/host.test.ts b/sdk/package/lib/test/host.test.ts index 87f22b8bd..4492804ec 100644 --- a/sdk/package/lib/test/host.test.ts +++ b/sdk/package/lib/test/host.test.ts @@ -15,7 +15,6 @@ describe("host", () => { name: "Foo", id: "foo", description: "A Foo", - hasPrimary: false, type: "ui", username: "bar", path: "/baz", diff --git a/web/package-lock.json b/web/package-lock.json index a36b7511e..7f01cb133 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "startos-ui", - "version": "0.3.6-alpha.9", + "version": "0.3.6-alpha.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "startos-ui", - "version": "0.3.6-alpha.9", + "version": "0.3.6-alpha.10", "license": "MIT", "dependencies": { "@angular/animations": "^14.1.0", diff --git a/web/package.json b/web/package.json index 3f3709e11..b7a13a4b3 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "startos-ui", - "version": "0.3.6-alpha.9", + "version": "0.3.6-alpha.10", "author": "Start9 Labs, Inc", "homepage": "https://start9.com/", "license": "MIT", diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts index a3562caa6..589a562db 100644 --- a/web/projects/ui/src/app/services/api/api.fixures.ts +++ b/web/projects/ui/src/app/services/api/api.fixures.ts @@ -1721,7 +1721,6 @@ export module Mock { serviceInterfaces: { ui: { id: 'ui', - hasPrimary: false, masked: false, name: 'Web UI', description: @@ -1738,7 +1737,6 @@ export module Mock { }, rpc: { id: 'rpc', - hasPrimary: false, masked: false, name: 'RPC', description: @@ -1755,7 +1753,6 @@ export module Mock { }, p2p: { id: 'p2p', - hasPrimary: true, masked: false, name: 'P2P', description: @@ -1776,7 +1773,8 @@ export module Mock { abcdefg: { kind: 'multi', bindings: [], - addresses: [], + onions: [], + domains: {}, hostnameInfo: { 80: [ { @@ -1859,7 +1857,8 @@ export module Mock { bcdefgh: { kind: 'multi', bindings: [], - addresses: [], + onions: [], + domains: {}, hostnameInfo: { 8332: [], }, @@ -1867,7 +1866,8 @@ export module Mock { cdefghi: { kind: 'multi', bindings: [], - addresses: [], + onions: [], + domains: {}, hostnameInfo: { 8333: [], }, @@ -1914,7 +1914,6 @@ export module Mock { serviceInterfaces: { ui: { id: 'ui', - hasPrimary: false, masked: false, name: 'Web UI', description: 'A launchable web app for Bitcoin Proxy', @@ -1960,7 +1959,6 @@ export module Mock { serviceInterfaces: { grpc: { id: 'grpc', - hasPrimary: false, masked: false, name: 'GRPC', description: @@ -1977,7 +1975,6 @@ export module Mock { }, lndconnect: { id: 'lndconnect', - hasPrimary: false, masked: true, name: 'LND Connect', description: @@ -1994,7 +1991,6 @@ export module Mock { }, p2p: { id: 'p2p', - hasPrimary: true, masked: false, name: 'P2P', description: diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index 0e6477266..6a26955aa 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -64,7 +64,7 @@ export const mockPatchData: DataModel = { }, }, }, - acme: null, + acme: {}, unreadNotificationCount: 4, // password is asdfasdf passwordHash: @@ -149,7 +149,6 @@ export const mockPatchData: DataModel = { serviceInterfaces: { ui: { id: 'ui', - hasPrimary: false, masked: false, name: 'Web UI', description: @@ -166,7 +165,6 @@ export const mockPatchData: DataModel = { }, rpc: { id: 'rpc', - hasPrimary: false, masked: false, name: 'RPC', description: @@ -183,7 +181,6 @@ export const mockPatchData: DataModel = { }, p2p: { id: 'p2p', - hasPrimary: true, masked: false, name: 'P2P', description: @@ -204,7 +201,8 @@ export const mockPatchData: DataModel = { abcdefg: { kind: 'multi', bindings: [], - addresses: [], + onions: [], + domains: {}, hostnameInfo: { 80: [ { @@ -287,7 +285,8 @@ export const mockPatchData: DataModel = { bcdefgh: { kind: 'multi', bindings: [], - addresses: [], + onions: [], + domains: {}, hostnameInfo: { 8332: [], }, @@ -295,7 +294,8 @@ export const mockPatchData: DataModel = { cdefghi: { kind: 'multi', bindings: [], - addresses: [], + onions: [], + domains: {}, hostnameInfo: { 8333: [], }, @@ -344,7 +344,6 @@ export const mockPatchData: DataModel = { serviceInterfaces: { grpc: { id: 'grpc', - hasPrimary: false, masked: false, name: 'GRPC', description: @@ -361,7 +360,6 @@ export const mockPatchData: DataModel = { }, lndconnect: { id: 'lndconnect', - hasPrimary: false, masked: true, name: 'LND Connect', description: @@ -378,7 +376,6 @@ export const mockPatchData: DataModel = { }, p2p: { id: 'p2p', - hasPrimary: true, masked: false, name: 'P2P', description: From 79725c8deca78908a7ea5e3c849f04c0663e52dd Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 19 Dec 2024 12:19:21 -0700 Subject: [PATCH 21/29] refactor Host::update --- core/startos/src/net/net_controller.rs | 432 +++++++++---------- core/startos/src/net/vhost.rs | 14 +- core/startos/src/service/effects/net/bind.rs | 33 +- 3 files changed, 244 insertions(+), 235 deletions(-) diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index b08b61c6e..d460faeca 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -6,6 +6,7 @@ use color_eyre::eyre::eyre; use imbl::OrdMap; use imbl_value::InternedString; use ipnet::IpNet; +use itertools::Itertools; use models::{HostId, OptionExt, PackageId}; use torut::onion::{OnionAddressV3, TorSecretKeyV3}; use tracing::instrument; @@ -13,17 +14,16 @@ use tracing::instrument; use crate::db::model::Database; use crate::error::ErrorCollection; use crate::hostname::Hostname; -use crate::net::acme::AcmeProvider; use crate::net::dns::DnsController; use crate::net::forward::LanPortForwardController; use crate::net::host::address::HostAddress; -use crate::net::host::binding::{AddSslOptions, BindId, BindOptions, NetInfo}; +use crate::net::host::binding::{BindId, BindOptions, NetInfo}; use crate::net::host::{host_for, Host, HostKind, Hosts}; use crate::net::network_interface::NetworkInterfaceController; use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname}; use crate::net::tor::TorController; use crate::net::utils::ipv6_is_local; -use crate::net::vhost::{AlpnInfo, VHostController}; +use crate::net::vhost::{AlpnInfo, TargetInfo, VHostController}; use crate::prelude::*; use crate::util::serde::MaybeUtf8String; use crate::HOST_IP; @@ -63,11 +63,6 @@ impl PreInitNetController { hostname: &Hostname, tor_key: TorSecretKeyV3, ) -> Result<(), Error> { - let alpn = Err(AlpnInfo::Specified(vec![ - MaybeUtf8String("http/1.1".into()), - MaybeUtf8String("h2".into()), - ])); - self.server_hostnames = vec![ // LAN IP None, @@ -81,15 +76,19 @@ impl PreInitNetController { Some(hostname.local_domain_name()), ]; + let vhost_target = TargetInfo { + public: false, + acme: None, + addr: ([127, 0, 0, 1], 80).into(), + connect_ssl: Err(AlpnInfo::Specified(vec![ + MaybeUtf8String("http/1.1".into()), + MaybeUtf8String("h2".into()), + ])), + }; + for hostname in self.server_hostnames.iter().cloned() { - self.os_bindings.push(self.vhost.add( - hostname, - 443, - false, - None, - ([127, 0, 0, 1], 80).into(), - alpn.clone(), - )?); + self.os_bindings + .push(self.vhost.add(hostname, 443, vhost_target.clone())?); } // Tor @@ -98,10 +97,7 @@ impl PreInitNetController { &tor_key.public().get_onion_address(), )), 443, - false, - None, - ([127, 0, 0, 1], 80).into(), - alpn.clone(), + vhost_target, )?); self.os_bindings.extend( self.tor @@ -180,15 +176,8 @@ impl NetController { #[derive(Default, Debug)] struct HostBinds { - lan: BTreeMap< - u16, - ( - NetInfo, - Option, - BTreeSet, - Vec>, - ), - >, + forwards: BTreeMap)>, + vhosts: BTreeMap<(Option, u16), (TargetInfo, Arc<()>)>, tor: BTreeMap, Vec>)>, } @@ -274,7 +263,11 @@ impl NetService { pub async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> { let ctrl = self.net_controller()?; - let mut hostname_info = BTreeMap::new(); + let mut forwards: BTreeMap = BTreeMap::new(); + let mut vhosts: BTreeMap<(Option, u16), TargetInfo> = BTreeMap::new(); + let mut tor: BTreeMap)> = + BTreeMap::new(); + let mut hostname_info: BTreeMap> = BTreeMap::new(); let binds = self.binds.entry(id.clone()).or_default(); let peek = ctrl.db.peek().await; @@ -284,116 +277,105 @@ impl NetService { let net_ifaces = server_info.as_network_interfaces().de()?; let hostname = server_info.as_hostname().de()?; for (port, bind) in &host.bindings { - let old_lan_bind = binds.lan.remove(port); if !bind.enabled { continue; } - let lan_bind = old_lan_bind - .as_ref() - .filter(|(external, ssl, _, _)| { - ssl == &bind.options.add_ssl && bind.net == *external - }) - .cloned(); // only keep existing binding if relevant details match if bind.net.assigned_port.is_some() || bind.net.assigned_ssl_port.is_some() { - let new_lan_bind = if let Some(b) = lan_bind { - b - } else { - let mut rcs = Vec::with_capacity( - 2 + ctrl.server_hostnames.len() + host.domains.len() + host.onions.len(), - ); - let mut hostnames = BTreeSet::new(); - if let Some(ssl) = &bind.options.add_ssl { - let external = bind - .net - .assigned_ssl_port - .or_not_found("assigned ssl port")?; - let target = (self.ip, *port).into(); - let connect_ssl = if let Some(alpn) = ssl.alpn.clone() { - Err(alpn) + let mut hostnames = BTreeSet::new(); + if let Some(ssl) = &bind.options.add_ssl { + let external = bind + .net + .assigned_ssl_port + .or_not_found("assigned ssl port")?; + let addr = (self.ip, *port).into(); + let connect_ssl = if let Some(alpn) = ssl.alpn.clone() { + Err(alpn) + } else { + if bind.options.secure.as_ref().map_or(false, |s| s.ssl) { + Ok(()) } else { - if bind.options.secure.as_ref().map_or(false, |s| s.ssl) { - Ok(()) - } else { - Err(AlpnInfo::Reflect) - } - }; - for hostname in ctrl.server_hostnames.iter().cloned() { - rcs.push(ctrl.vhost.add( - hostname, - external, - bind.net.public, - None, - target, - connect_ssl.clone(), - )?); + Err(AlpnInfo::Reflect) } - for address in host.addresses() { - match address { - HostAddress::Onion { address } => { - let hostname = InternedString::from_display(&address); - if hostnames.insert(hostname.clone()) { - rcs.push(ctrl.vhost.add( - Some(hostname), - external, - false, - None, - target, - connect_ssl.clone(), - )?); - } + }; + for hostname in ctrl.server_hostnames.iter().cloned() { + vhosts.insert( + (hostname, external), + TargetInfo { + public: bind.net.public, + acme: None, + addr, + connect_ssl: connect_ssl.clone(), + }, + ); + } + for address in host.addresses() { + match address { + HostAddress::Onion { address } => { + let hostname = InternedString::from_display(&address); + if hostnames.insert(hostname.clone()) { + vhosts.insert( + (Some(hostname), external), + TargetInfo { + public: false, + acme: None, + addr, + connect_ssl: connect_ssl.clone(), + }, + ); } - HostAddress::Domain { - address, - public, - acme, - } => { - if hostnames.insert(address.clone()) { - let address = Some(address.clone()); - if ssl.preferred_external_port == 443 { - if public && bind.net.public { - rcs.push(ctrl.vhost.add( - address.clone(), - 5443, - false, - acme.clone(), - target, - connect_ssl.clone(), - )?); - } - rcs.push(ctrl.vhost.add( - address.clone(), - 443, - public && bind.net.public, + } + HostAddress::Domain { + address, + public, + acme, + } => { + if hostnames.insert(address.clone()) { + let address = Some(address.clone()); + if ssl.preferred_external_port == 443 { + if public && bind.net.public { + vhosts.insert( + (address.clone(), 5443), + TargetInfo { + public: false, + acme: acme.clone(), + addr, + connect_ssl: connect_ssl.clone(), + }, + ); + } + vhosts.insert( + (address.clone(), 443), + TargetInfo { + public: public && bind.net.public, acme, - target, - connect_ssl.clone(), - )?); - } else { - rcs.push(ctrl.vhost.add( - address.clone(), - external, - public && bind.net.public, + addr, + connect_ssl: connect_ssl.clone(), + }, + ); + } else { + vhosts.insert( + (address.clone(), external), + TargetInfo { + public: public && bind.net.public, acme, - target, - connect_ssl.clone(), - )?); - } + addr, + connect_ssl: connect_ssl.clone(), + }, + ); } } } } } - if let Some(security) = bind.options.secure { - if bind.options.add_ssl.is_some() && security.ssl { - // doesn't make sense to have 2 listening ports, both with ssl - } else { - let external = - bind.net.assigned_port.or_not_found("assigned lan port")?; - rcs.push(ctrl.forward.add(external, (self.ip, *port).into()).await?); - } + } + if let Some(security) = bind.options.secure { + if bind.options.add_ssl.is_some() && security.ssl { + // doesn't make sense to have 2 listening ports, both with ssl + } else { + let external = bind.net.assigned_port.or_not_found("assigned lan port")?; + forwards.insert(external, (self.ip, *port).into()); } - (bind.net, bind.options.add_ssl.clone(), hostnames, rcs) - }; + } let mut bind_hostname_info: Vec = hostname_info.remove(port).unwrap_or_default(); for (interface, public, ip_info) in @@ -414,8 +396,8 @@ impl NetService { let hostname = &hostname; lazy_format!("{hostname}.local") }), - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, + port: bind.net.assigned_port, + ssl_port: bind.net.assigned_ssl_port, }, }); } @@ -426,9 +408,10 @@ impl NetService { .. } = address { - if !public || (domain_public && new_lan_bind.0.public) { - if new_lan_bind - .1 + if !public || (domain_public && bind.net.public) { + if bind + .options + .add_ssl .as_ref() .map_or(false, |ssl| ssl.preferred_external_port == 443) { @@ -449,23 +432,23 @@ impl NetService { hostname: IpHostname::Domain { domain: address.clone(), subdomain: None, - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, + port: bind.net.assigned_port, + ssl_port: bind.net.assigned_ssl_port, }, }); } } } } - if !public || new_lan_bind.0.public { + if !public || bind.net.public { if let Some(wan_ip) = ip_info.wan_ip.filter(|_| public) { bind_hostname_info.push(HostnameInfo::Ip { network_interface_id: interface.clone(), public, hostname: IpHostname::Ipv4 { value: wan_ip, - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, + port: bind.net.assigned_port, + ssl_port: bind.net.assigned_ssl_port, }, }); } @@ -478,8 +461,8 @@ impl NetService { public, hostname: IpHostname::Ipv4 { value: net.addr(), - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, + port: bind.net.assigned_port, + ssl_port: bind.net.assigned_ssl_port, }, }); } @@ -490,8 +473,8 @@ impl NetService { public: public && !ipv6_is_local(net.addr()), hostname: IpHostname::Ipv6 { value: net.addr(), - port: new_lan_bind.0.assigned_port, - ssl_port: new_lan_bind.0.assigned_ssl_port, + port: bind.net.assigned_port, + ssl_port: bind.net.assigned_ssl_port, }, }); } @@ -500,43 +483,6 @@ impl NetService { } } hostname_info.insert(*port, bind_hostname_info); - binds.lan.insert(*port, new_lan_bind); - } - if let Some((lan, _, hostnames, _)) = old_lan_bind { - if let Some(external) = lan.assigned_ssl_port { - for hostname in ctrl.server_hostnames.iter().cloned() { - ctrl.vhost.gc(hostname, external); - } - for hostname in hostnames { - ctrl.vhost.gc(Some(hostname), external); - } - } - if let Some(external) = lan.assigned_port { - ctrl.forward.gc(external).await?; - } - } - } - let mut removed = BTreeSet::new(); - binds.lan.retain(|internal, (external, _, hostnames, _)| { - if host.bindings.get(internal).map_or(false, |b| b.enabled) { - true - } else { - removed.insert((*external, std::mem::take(hostnames))); - - false - } - }); - for (lan, hostnames) in removed { - if let Some(external) = lan.assigned_ssl_port { - for hostname in ctrl.server_hostnames.iter().cloned() { - ctrl.vhost.gc(hostname, external); - } - for hostname in hostnames { - ctrl.vhost.gc(Some(hostname), external); - } - } - if let Some(external) = lan.assigned_port { - ctrl.forward.gc(external).await?; } } @@ -580,25 +526,13 @@ impl NetService { } } - let mut keep_tor_addrs = BTreeSet::new(); for tor_addr in host.onions.iter() { - keep_tor_addrs.insert(tor_addr); - let old_tor_bind = binds.tor.remove(tor_addr); - let tor_bind = old_tor_bind.filter(|(ports, _)| ports == &tor_binds); - let new_tor_bind = if let Some(tor_bind) = tor_bind { - tor_bind - } else { - let key = peek - .as_private() - .as_key_store() - .as_onion() - .get_key(tor_addr)?; - let rcs = ctrl - .tor - .add(key, tor_binds.clone().into_iter().collect()) - .await?; - (tor_binds.clone(), rcs) - }; + let key = peek + .as_private() + .as_key_store() + .as_onion() + .get_key(tor_addr)?; + tor.insert(key.public().get_onion_address(), (key, tor_binds.clone())); for (internal, ports) in &tor_hostname_ports { let mut bind_hostname_info = hostname_info.remove(internal).unwrap_or_default(); bind_hostname_info.push(HostnameInfo::Onion { @@ -610,16 +544,91 @@ impl NetService { }); hostname_info.insert(*internal, bind_hostname_info); } - binds.tor.insert(tor_addr.clone(), new_tor_bind); } - for addr in binds.tor.keys() { - if !keep_tor_addrs.contains(addr) { - ctrl.tor.gc(Some(addr.clone()), None).await?; + + let all = binds + .forwards + .keys() + .chain(forwards.keys()) + .copied() + .collect::>(); + for external in all { + let mut prev = binds.forwards.remove(&external); + if let Some(internal) = forwards.remove(&external) { + prev = prev.filter(|(i, _)| i == &internal); + binds.forwards.insert( + external, + if let Some(prev) = prev { + prev + } else { + (internal, ctrl.forward.add(external, internal).await?) + }, + ); + } else { + if let Some((_, rc)) = prev { + drop(rc); + ctrl.forward.gc(external).await?; + } } } - self.net_controller()? - .db + let all = binds + .vhosts + .keys() + .chain(vhosts.keys()) + .cloned() + .collect::>(); + for key in all { + let mut prev = binds.vhosts.remove(&key); + if let Some(target) = vhosts.remove(&key) { + prev = prev.filter(|(t, _)| t == &target); + binds.vhosts.insert( + key.clone(), + if let Some(prev) = prev { + prev + } else { + (target.clone(), ctrl.vhost.add(key.0, key.1, target)?) + }, + ); + } else { + if let Some((_, rc)) = prev { + drop(rc); + ctrl.vhost.gc(key.0, key.1); + } + } + } + + let all = binds + .tor + .keys() + .chain(tor.keys()) + .cloned() + .collect::>(); + for onion in all { + let mut prev = binds.tor.remove(&onion); + if let Some((key, tor_binds)) = tor.remove(&onion) { + prev = prev.filter(|(b, _)| b == &tor_binds); + binds.tor.insert( + onion, + if let Some(prev) = prev { + prev + } else { + let rcs = ctrl + .tor + .add(key, tor_binds.iter().map(|(k, v)| (*k, *v)).collect()) + .await?; + (tor_binds, rcs) + }, + ); + } else { + if let Some((_, rc)) = prev { + drop(rc); + ctrl.tor.gc(Some(onion), None).await?; + } + } + } + + ctrl.db .mutate(|db| { host_for(db, &self.id, &id, host.kind)? .as_hostname_info_mut() @@ -647,29 +656,6 @@ impl NetService { pub fn get_ip(&self) -> Ipv4Addr { self.ip } - - pub fn get_lan_port(&self, host_id: HostId, internal_port: u16) -> Result { - let host_id_binds = self.binds.get_key_value(&host_id); - match host_id_binds { - Some((_, binds)) => { - if let Some((lan, _, _, _)) = binds.lan.get(&internal_port) { - Ok(*lan) - } else { - Err(Error::new( - eyre!( - "Internal Port {} not found in NetService binds", - internal_port - ), - crate::ErrorKind::NotFound, - )) - } - } - None => Err(Error::new( - eyre!("HostID {} not found in NetService binds", host_id), - crate::ErrorKind::NotFound, - )), - } - } } impl Drop for NetService { diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs index 70170584d..dc6f422bf 100644 --- a/core/startos/src/net/vhost.rs +++ b/core/startos/src/net/vhost.rs @@ -14,7 +14,7 @@ use helpers::NonDetachingJoinHandle; use http::Uri; use imbl_value::InternedString; use models::ResultExt; -use rpc_toolkit::{from_fn, from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; +use rpc_toolkit::{from_fn, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; @@ -119,10 +119,12 @@ impl VHostController { &self, hostname: Option, external: u16, - public: bool, - acme: Option, - target: SocketAddr, - connect_ssl: Result<(), AlpnInfo>, + TargetInfo { + public, + acme, + addr, + connect_ssl, + }: TargetInfo, ) -> Result, Error> { self.servers.mutate(|writable| { let server = if let Some(server) = writable.remove(&external) { @@ -141,7 +143,7 @@ impl VHostController { TargetInfo { public, acme, - addr: target, + addr, connect_ssl, }, ); diff --git a/core/startos/src/service/effects/net/bind.rs b/core/startos/src/service/effects/net/bind.rs index d30b45f72..2f3edb07b 100644 --- a/core/startos/src/service/effects/net/bind.rs +++ b/core/startos/src/service/effects/net/bind.rs @@ -53,15 +53,36 @@ pub struct GetServicePortForwardParams { #[ts(optional)] package_id: Option, host_id: HostId, - internal_port: u32, + internal_port: u16, } pub async fn get_service_port_forward( context: EffectContext, - data: GetServicePortForwardParams, + GetServicePortForwardParams { + package_id, + host_id, + internal_port, + }: GetServicePortForwardParams, ) -> Result { - let internal_port = data.internal_port as u16; - let context = context.deref()?; - let net_service = context.seed.persistent_container.net_service.lock().await; - net_service.get_lan_port(data.host_id, internal_port) + + let package_id = package_id.unwrap_or_else(|| context.seed.id.clone()); + + Ok(context + .seed + .ctx + .db + .peek() + .await + .as_public() + .as_package_data() + .as_idx(&package_id) + .or_not_found(&package_id)? + .as_hosts() + .as_idx(&host_id) + .or_not_found(&host_id)? + .as_bindings() + .de()? + .get(&internal_port) + .or_not_found(lazy_format!("binding for port {internal_port}"))? + .net) } From 4a0716c9ba967e0da5852169487be8e3891033c2 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Fri, 20 Dec 2024 12:49:17 -0700 Subject: [PATCH 22/29] debug info --- core/startos/src/net/network_interface.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 5087f0085..3a45d2cba 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -820,6 +820,7 @@ impl ListenerMap { } self.listeners.retain(|key, _| keep.contains(key)); self.prev_public = public; + crate::dbg!(&self.listeners); Ok(()) } fn accept(&mut self) -> ListenerMapFut { From c8c5ee037a2b1e008c86269524f80f5e807cab26 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 30 Dec 2024 17:35:55 -0700 Subject: [PATCH 23/29] refactor webserver --- core/Cargo.lock | 38 +- core/startos/Cargo.toml | 10 +- core/startos/src/backup/restore.rs | 4 +- core/startos/src/bins/registry.rs | 6 +- core/startos/src/bins/start_init.rs | 21 +- core/startos/src/bins/startd.rs | 22 +- core/startos/src/context/config.rs | 3 - core/startos/src/context/rpc.rs | 8 +- core/startos/src/context/setup.rs | 8 +- core/startos/src/db/model/public.rs | 10 +- core/startos/src/init.rs | 4 + core/startos/src/lib.rs | 2 +- core/startos/src/net/net_controller.rs | 4 +- core/startos/src/net/network_interface.rs | 413 +++++++++++++--------- core/startos/src/net/static_server.rs | 18 +- core/startos/src/net/utils.rs | 57 ++- core/startos/src/net/web_server.rs | 332 ++++++++++++----- core/startos/src/registry/mod.rs | 4 +- core/startos/src/setup.rs | 24 +- core/startos/src/util/actor/background.rs | 9 +- core/startos/src/util/future.rs | 4 +- core/startos/src/util/sync.rs | 126 +++++++ 22 files changed, 787 insertions(+), 340 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 7d8f06d1c..b8cbe8d5b 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -533,7 +533,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "itoa", "matchit", @@ -594,25 +594,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-server" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" -dependencies = [ - "bytes", - "futures-util", - "http 1.2.0", - "http-body 1.0.1", - "http-body-util", - "hyper 1.5.1", - "hyper-util", - "pin-project-lite", - "tokio", - "tower 0.4.13", - "tower-service", -] - [[package]] name = "backhand" version = "0.18.0" @@ -2232,7 +2213,7 @@ checksum = "75cec8bb4d3d32542cfcb9517f78366b52c17931e30d7ee1682c13686c19cee7" dependencies = [ "futures", "futures-rustls", - "hyper 1.5.1", + "hyper 1.5.2", "log", "serde", "serde_json", @@ -2601,9 +2582,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -2628,7 +2609,7 @@ checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "rustls 0.23.20", "rustls-pki-types", @@ -2657,7 +2638,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "native-tls", "tokio", @@ -2676,7 +2657,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -4629,7 +4610,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-rustls", "hyper-tls", "hyper-util", @@ -5653,7 +5634,6 @@ dependencies = [ "async-stream", "async-trait", "axum 0.7.9", - "axum-server", "backhand", "barrage", "base32 0.5.1", @@ -5687,6 +5667,7 @@ dependencies = [ "hmac", "http 1.2.0", "http-body-util", + "hyper 1.5.2", "hyper-util", "id-pool", "imbl", @@ -5709,6 +5690,7 @@ dependencies = [ "libc", "log", "mbrman", + "mio", "models", "new_mime_guess", "nix 0.29.0", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 3e312c295..2e9e49f91 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -62,7 +62,6 @@ async-compression = { version = "0.4.4", features = [ async-stream = "0.3.5" async-trait = "0.1.74" axum = { version = "0.7.3", features = ["ws"] } -axum-server = "0.6.0" barrage = "0.2.3" backhand = "0.18.0" base32 = "0.5.0" @@ -103,11 +102,15 @@ hex = "0.4.3" hmac = "0.12.1" http = "1.0.0" http-body-util = "0.1" -hyper-util = { version = "0.1.5", features = [ - "tokio", +hyper = { version = "1.5", features = ["server", "http1", "http2"] } +hyper-util = { version = "0.1.10", features = [ + "server", + "server-auto", + "server-graceful", "service", "http1", "http2", + "tokio", ] } id-pool = { version = "0.2.2", default-features = false, features = [ "serde", @@ -132,6 +135,7 @@ lazy_format = "2.0" lazy_static = "1.4.0" libc = "0.2.149" log = "0.4.20" +mio = "1" mbrman = "0.5.2" models = { version = "*", path = "../models" } new_mime_guess = "4" diff --git a/core/startos/src/backup/restore.rs b/core/startos/src/backup/restore.rs index 28d70653f..b22d9027d 100644 --- a/core/startos/src/backup/restore.rs +++ b/core/startos/src/backup/restore.rs @@ -19,6 +19,7 @@ use crate::disk::mount::backup::BackupMountGuard; use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; use crate::init::{init, InitResult}; +use crate::net::web_server::WebServer; use crate::prelude::*; use crate::s9pk::S9pk; use crate::service::service_map::DownloadInstallFuture; @@ -109,9 +110,10 @@ pub async fn recover_full_embassy( db.put(&ROOT, &Database::init(&os_backup.account)?).await?; drop(db); - let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; + let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?; let rpc_ctx = RpcContext::init( + &ctx.webserver, &ctx.config, disk_guid.clone(), Some(net_ctrl), diff --git a/core/startos/src/bins/registry.rs b/core/startos/src/bins/registry.rs index a455167fc..8a52b2485 100644 --- a/core/startos/src/bins/registry.rs +++ b/core/startos/src/bins/registry.rs @@ -1,11 +1,11 @@ use std::ffi::OsString; use clap::Parser; -use futures::FutureExt; +use futures::{FutureExt, TryStreamExt}; use tokio::signal::unix::signal; use tracing::instrument; -use crate::net::web_server::WebServer; +use crate::net::web_server::{Acceptor, WebServer}; use crate::prelude::*; use crate::registry::context::{RegistryConfig, RegistryContext}; use crate::util::logger::LOGGER; @@ -14,7 +14,7 @@ use crate::util::logger::LOGGER; async fn inner_main(config: &RegistryConfig) -> Result<(), Error> { let server = async { let ctx = RegistryContext::init(config).await?; - let mut server = WebServer::new(ctx.listen); + let mut server = WebServer::new(Acceptor::bind([ctx.listen]).await?); server.serve_registry(ctx.clone()); let mut shutdown_recv = ctx.shutdown.subscribe(); diff --git a/core/startos/src/bins/start_init.rs b/core/startos/src/bins/start_init.rs index fc65feae9..aad29c0c5 100644 --- a/core/startos/src/bins/start_init.rs +++ b/core/startos/src/bins/start_init.rs @@ -11,7 +11,7 @@ use crate::disk::main::DEFAULT_PASSWORD; use crate::disk::REPAIR_DISK_PATH; use crate::firmware::{check_for_firmware_update, update_firmware}; use crate::init::{InitPhases, InitResult, STANDBY_MODE_PATH}; -use crate::net::web_server::WebServer; +use crate::net::web_server::{UpgradableListener, WebServer}; use crate::prelude::*; use crate::progress::FullProgressTracker; use crate::shutdown::Shutdown; @@ -20,7 +20,7 @@ use crate::PLATFORM; #[instrument(skip_all)] async fn setup_or_init( - server: &mut WebServer, + server: &mut WebServer, config: &ServerConfig, ) -> Result, Error> { if let Some(firmware) = check_for_firmware_update() @@ -111,7 +111,7 @@ async fn setup_or_init( .await .is_err() { - let ctx = SetupContext::init(config)?; + let ctx = SetupContext::init(server, config)?; server.serve_setup(ctx.clone()); @@ -187,10 +187,17 @@ async fn setup_or_init( })); } - let InitResult { net_ctrl } = crate::init::init(config, init_phases).await?; + let InitResult { net_ctrl } = + crate::init::init(&server.acceptor_setter(), config, init_phases).await?; - let rpc_ctx = - RpcContext::init(config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?; + let rpc_ctx = RpcContext::init( + &server.acceptor_setter(), + config, + disk_guid, + Some(net_ctrl), + rpc_ctx_phases, + ) + .await?; Ok::<_, Error>(Ok((rpc_ctx, handle))) } @@ -204,7 +211,7 @@ async fn setup_or_init( #[instrument(skip_all)] pub async fn main( - server: &mut WebServer, + server: &mut WebServer, config: &ServerConfig, ) -> Result, Error> { if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() { diff --git a/core/startos/src/bins/startd.rs b/core/startos/src/bins/startd.rs index e93857c38..2bf32724e 100644 --- a/core/startos/src/bins/startd.rs +++ b/core/startos/src/bins/startd.rs @@ -1,19 +1,19 @@ use std::cmp::max; use std::ffi::OsString; -use std::net::{Ipv6Addr, SocketAddr}; +use std::net::IpAddr; use std::sync::Arc; use clap::Parser; use color_eyre::eyre::eyre; use futures::{FutureExt, TryFutureExt}; -use tokio::fs::OpenOptions; use tokio::signal::unix::signal; use tracing::instrument; use crate::context::config::ServerConfig; use crate::context::rpc::InitRpcContextPhases; use crate::context::{DiagnosticContext, InitContext, RpcContext}; -use crate::net::web_server::WebServer; +use crate::net::utils::ipv6_is_local; +use crate::net::web_server::{Acceptor, UpgradableListener, WebServer}; use crate::shutdown::Shutdown; use crate::system::launch_metrics_task; use crate::util::io::append_file; @@ -22,7 +22,7 @@ use crate::{Error, ErrorKind, ResultExt}; #[instrument(skip_all)] async fn inner_main( - server: &mut WebServer, + server: &mut WebServer, config: &ServerConfig, ) -> Result, Error> { let rpc_ctx = if !tokio::fs::metadata("/run/startos/initialized") @@ -50,6 +50,7 @@ async fn inner_main( server.serve_init(init_ctx); let ctx = RpcContext::init( + &server.acceptor_setter(), config, Arc::new( tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy @@ -148,7 +149,18 @@ pub fn main(args: impl IntoIterator) { .build() .expect("failed to initialize runtime"); rt.block_on(async { - let mut server = WebServer::new(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80)); + let addrs = crate::net::utils::all_socket_addrs_for(80).await?; + let mut server = WebServer::new( + Acceptor::bind_upgradable(addrs.into_iter().filter(|addr| match addr.ip() { + IpAddr::V4(ip4) => { + ip4.is_loopback() + || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations + || ip4.is_link_local() + } + IpAddr::V6(ip6) => ipv6_is_local(ip6), + })) + .await?, + ); match inner_main(&mut server, &config).await { Ok(a) => { server.shutdown().await; diff --git a/core/startos/src/context/config.rs b/core/startos/src/context/config.rs index e02648919..811819479 100644 --- a/core/startos/src/context/config.rs +++ b/core/startos/src/context/config.rs @@ -103,8 +103,6 @@ pub struct ServerConfig { #[arg(skip)] pub os_partitions: Option, #[arg(long)] - pub bind_rpc: Option, - #[arg(long)] pub tor_control: Option, #[arg(long)] pub tor_socks: Option, @@ -126,7 +124,6 @@ impl ContextConfig for ServerConfig { fn merge_with(&mut self, other: Self) { self.ethernet_interface = self.ethernet_interface.take().or(other.ethernet_interface); self.os_partitions = self.os_partitions.take().or(other.os_partitions); - self.bind_rpc = self.bind_rpc.take().or(other.bind_rpc); self.tor_control = self.tor_control.take().or(other.tor_control); self.tor_socks = self.tor_socks.take().or(other.tor_socks); self.dns_bind = self.dns_bind.take().or(other.dns_bind); diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index 73d103adc..5b65f3d6d 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -31,6 +31,7 @@ use crate::init::check_time_is_synchronized; use crate::lxc::{ContainerId, LxcContainer, LxcManager}; use crate::net::net_controller::{NetController, PreInitNetController}; use crate::net::utils::{find_eth_iface, find_wifi_iface}; +use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter}; use crate::net::wifi::WpaCli; use crate::prelude::*; use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle}; @@ -117,6 +118,7 @@ pub struct RpcContext(Arc); impl RpcContext { #[instrument(skip_all)] pub async fn init( + webserver: &WebServerAcceptorSetter, config: &ServerConfig, disk_guid: Arc, net_ctrl: Option, @@ -149,7 +151,7 @@ impl RpcContext { if let Some(net_ctrl) = net_ctrl { net_ctrl } else { - PreInitNetController::init( + let net_ctrl = PreInitNetController::init( db.clone(), config .tor_control @@ -158,7 +160,9 @@ impl RpcContext { &account.hostname, account.tor_key.clone(), ) - .await? + .await?; + webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a)); + net_ctrl }, config .dns_bind diff --git a/core/startos/src/context/setup.rs b/core/startos/src/context/setup.rs index 96ec07700..2db5668b9 100644 --- a/core/startos/src/context/setup.rs +++ b/core/startos/src/context/setup.rs @@ -23,6 +23,7 @@ use crate::context::RpcContext; use crate::disk::OsPartitionInfo; use crate::hostname::Hostname; use crate::init::init_postgres; +use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter}; use crate::prelude::*; use crate::progress::FullProgressTracker; use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations}; @@ -61,6 +62,7 @@ impl TryFrom<&AccountInfo> for SetupResult { } pub struct SetupContextSeed { + pub webserver: WebServerAcceptorSetter, pub config: ServerConfig, pub os_partitions: OsPartitionInfo, pub disable_encryption: bool, @@ -76,10 +78,14 @@ pub struct SetupContextSeed { pub struct SetupContext(Arc); impl SetupContext { #[instrument(skip_all)] - pub fn init(config: &ServerConfig) -> Result { + pub fn init( + webserver: &WebServer, + config: &ServerConfig, + ) -> Result { let (shutdown, _) = tokio::sync::broadcast::channel(1); let datadir = config.datadir().to_owned(); Ok(Self(Arc::new(SetupContextSeed { + webserver: webserver.acceptor_setter(), config: config.clone(), os_partitions: config.os_partitions.clone().ok_or_else(|| { Error::new( diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 3463d4db6..f5ab2120c 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -18,6 +18,7 @@ use ts_rs::TS; use crate::account::AccountInfo; use crate::db::model::package::AllPackageData; use crate::net::acme::AcmeProvider; +use crate::net::utils::ipv6_is_local; use crate::prelude::*; use crate::progress::FullProgress; use crate::system::SmtpValue; @@ -167,12 +168,13 @@ impl NetworkInterfaceInfo { self.public.unwrap_or_else(|| { !self.ip_info.as_ref().map_or(true, |ip_info| { ip_info.subnets.iter().all(|ipnet| { - if let IpAddr::V4(ip4) = ipnet.addr() { - ip4.is_loopback() + match ipnet.addr() { + IpAddr::V4(ip4) => { + ip4.is_loopback() || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations || ip4.is_link_local() - } else { - true + } + IpAddr::V6(ip6) => true, } }) }) diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index b514ebbe8..92715683b 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -7,6 +7,7 @@ use std::time::{Duration, SystemTime}; use axum::extract::ws::{self}; use color_eyre::eyre::eyre; +use futures::future::Either; use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; use models::ResultExt; @@ -25,6 +26,7 @@ use crate::db::model::Database; use crate::disk::mount::util::unmount; use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH; use crate::net::net_controller::PreInitNetController; +use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter}; use crate::prelude::*; use crate::progress::{ FullProgress, FullProgressTracker, PhaseProgressTrackerHandle, PhasedProgressBar, @@ -274,6 +276,7 @@ pub async fn run_script>(path: P, mut progress: PhaseProgressTrac #[instrument(skip_all)] pub async fn init( + webserver: &WebServerAcceptorSetter, cfg: &ServerConfig, InitPhases { preinit, @@ -356,6 +359,7 @@ pub async fn init( account.tor_key, ) .await?; + webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a)); start_net.complete(); mount_logs.start(); diff --git a/core/startos/src/lib.rs b/core/startos/src/lib.rs index 3c5875e36..1285b4811 100644 --- a/core/startos/src/lib.rs +++ b/core/startos/src/lib.rs @@ -1,6 +1,6 @@ pub const DEFAULT_REGISTRY: &str = "https://registry.start9.com"; // pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com"; -pub const HOST_IP: [u8; 4] = [172, 18, 0, 1]; +pub const HOST_IP: [u8; 4] = [10, 0, 3, 1]; pub use std::env::consts::ARCH; lazy_static::lazy_static! { pub static ref PLATFORM: String = { diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index d460faeca..518384925 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -17,7 +17,7 @@ use crate::hostname::Hostname; use crate::net::dns::DnsController; use crate::net::forward::LanPortForwardController; use crate::net::host::address::HostAddress; -use crate::net::host::binding::{BindId, BindOptions, NetInfo}; +use crate::net::host::binding::{BindId, BindOptions}; use crate::net::host::{host_for, Host, HostKind, Hosts}; use crate::net::network_interface::NetworkInterfaceController; use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname}; @@ -32,7 +32,7 @@ pub struct PreInitNetController { pub db: TypedPatchDb, tor: TorController, vhost: VHostController, - net_iface: Arc, + pub net_iface: Arc, os_bindings: Vec>, server_hostnames: Vec>, } diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 3a45d2cba..bdb81edae 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -19,7 +19,6 @@ use serde::{Deserialize, Serialize}; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::net::{TcpListener, TcpStream}; use tokio::process::Command; -use tokio::sync::watch; use ts_rs::TS; use zbus::proxy::{PropertyChanged, PropertyStream, SignalStream}; use zbus::zvariant::{ @@ -30,12 +29,12 @@ use zbus::{proxy, Connection}; use crate::context::{CliContext, RpcContext}; use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; use crate::db::model::Database; -use crate::net::network_interface::active_connection::ActiveConnectionProxy; +use crate::net::utils::ipv6_is_local; use crate::prelude::*; use crate::util::future::Until; use crate::util::io::open_file; use crate::util::serde::{display_serializable, HandlerExtSerde}; -use crate::util::sync::SyncMutex; +use crate::util::sync::{SyncMutex, Watch}; use crate::util::Invoke; pub fn network_interface_api() -> ParentHandler { @@ -109,7 +108,7 @@ pub fn network_interface_api() -> ParentHandler { async fn list_interfaces( ctx: RpcContext, ) -> Result, Error> { - Ok(ctx.net_controller.net_iface.ip_info.borrow().clone()) + Ok(ctx.net_controller.net_iface.ip_info.read()) } #[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)] @@ -306,7 +305,7 @@ impl<'a> StubStream<'a> for SignalStream<'a> { } #[instrument(skip_all)] -async fn watcher(write_to: watch::Sender>) { +async fn watcher(write_to: Watch>) { loop { let res: Result<(), Error> = async { let connection = Connection::system().await?; @@ -343,24 +342,9 @@ async fn watcher(write_to: watch::Sender Result, Error> { #[instrument(skip_all)] async fn watch_ip( connection: &Connection, - active_connection_proxy: ActiveConnectionProxy<'_>, device_proxy: DeviceProxy<'_>, iface: InternedString, - write_to: &watch::Sender>, + write_to: &Watch>, ) -> Result<(), Error> { let mut until = Until::new() .with_stream( @@ -434,100 +417,129 @@ async fn watch_ip( .into_inner() .stub(), ) - .with_stream( - active_connection_proxy - .receive_state_changed() - .await? - .into_inner() - .stub(), - ) .with_stream(device_proxy.receive_ip4_config_changed().await.stub()) - .with_stream(device_proxy.receive_ip6_config_changed().await.stub()) - .with_stream( - active_connection_proxy - .receive_dhcp4_config_changed() - .await - .stub(), - ); + .with_stream(device_proxy.receive_ip6_config_changed().await.stub()); loop { - let ip4_config = device_proxy.ip4_config().await?; - let ip6_config = device_proxy.ip6_config().await?; - let dhcp4_config = active_connection_proxy.dhcp4_config().await?; until .run(async { - let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config).await?; - let ip6_proxy = Ip6ConfigProxy::new(&connection, ip6_config).await?; + let ip4_config = device_proxy.ip4_config().await?; + let ip6_config = device_proxy.ip6_config().await?; + + let managed = device_proxy.managed().await?; + if !managed { + return Ok(()); + } + let dac = device_proxy.active_connection().await?; + if &*dac == "/" { + return Ok(()); + } + let active_connection_proxy = + active_connection::ActiveConnectionProxy::new(&connection, dac).await?; + let mut until = Until::new() - .with_stream(ip4_proxy.receive_address_data_changed().await.stub()) - .with_stream(ip6_proxy.receive_address_data_changed().await.stub()); - - let dhcp4_proxy = if &*dhcp4_config != "/" { - let dhcp4_proxy = Dhcp4ConfigProxy::new(&connection, dhcp4_config).await?; - until = until.with_stream(dhcp4_proxy.receive_options_changed().await.stub()); - Some(dhcp4_proxy) - } else { - None - }; + .with_stream( + active_connection_proxy + .receive_state_changed() + .await? + .into_inner() + .stub(), + ) + .with_stream( + active_connection_proxy + .receive_dhcp4_config_changed() + .await + .stub(), + ); - loop { - until - .run(async { - let addresses = ip4_proxy - .address_data() - .await? - .into_iter() - .chain(ip6_proxy.address_data().await?) - .collect_vec(); - let mut ntp_servers = BTreeSet::new(); - if let Some(dhcp4_proxy) = &dhcp4_proxy { - let dhcp = crate::dbg!(dhcp4_proxy.options().await?); - if let Some(ntp) = dhcp.ntp_servers { - ntp_servers - .extend(ntp.split_whitespace().map(InternedString::intern)); - } - } - let scope_id = if_nametoindex(&*iface).with_kind(ErrorKind::Network)?; - let subnets: BTreeSet = - addresses.into_iter().map(TryInto::try_into).try_collect()?; - let ip_info = if !subnets.is_empty() { - let wan_ip = match get_wan_ipv4(&*iface).await { - Ok(a) => a, - Err(e) => { - tracing::error!( - "Failed to determine WAN IP for {iface}: {e}" - ); - tracing::debug!("{e:?}"); - None + until + .run(async { + let external = active_connection_proxy.state_flags().await? & 0x80 != 0; + if external { + return Ok(()); + } + + let dhcp4_config = active_connection_proxy.dhcp4_config().await?; + let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config).await?; + let ip6_proxy = Ip6ConfigProxy::new(&connection, ip6_config).await?; + let mut until = Until::new() + .with_stream(ip4_proxy.receive_address_data_changed().await.stub()) + .with_stream(ip6_proxy.receive_address_data_changed().await.stub()); + + let dhcp4_proxy = if &*dhcp4_config != "/" { + let dhcp4_proxy = + Dhcp4ConfigProxy::new(&connection, dhcp4_config).await?; + until = until + .with_stream(dhcp4_proxy.receive_options_changed().await.stub()); + Some(dhcp4_proxy) + } else { + None + }; + + loop { + until + .run(async { + let addresses = ip4_proxy + .address_data() + .await? + .into_iter() + .chain(ip6_proxy.address_data().await?) + .collect_vec(); + let mut ntp_servers = BTreeSet::new(); + if let Some(dhcp4_proxy) = &dhcp4_proxy { + let dhcp = crate::dbg!(dhcp4_proxy.options().await?); + if let Some(ntp) = dhcp.ntp_servers { + ntp_servers.extend( + ntp.split_whitespace().map(InternedString::intern), + ); + } } - }; - Some(IpInfo { - scope_id, - subnets, - wan_ip, - ntp_servers, + let scope_id = + if_nametoindex(&*iface).with_kind(ErrorKind::Network)?; + let subnets: BTreeSet = addresses + .into_iter() + .map(TryInto::try_into) + .try_collect()?; + let ip_info = if !subnets.is_empty() { + let wan_ip = match get_wan_ipv4(&*iface).await { + Ok(a) => a, + Err(e) => { + tracing::error!( + "Failed to determine WAN IP for {iface}: {e}" + ); + tracing::debug!("{e:?}"); + None + } + }; + Some(IpInfo { + scope_id, + subnets, + wan_ip, + ntp_servers, + }) + } else { + None + }; + + write_to.send_if_modified(|m| { + let public = m.get(&iface).map_or(None, |i| i.public); + m.insert( + iface.clone(), + NetworkInterfaceInfo { + public, + ip_info: ip_info.clone(), + }, + ) + .filter(|old| &old.ip_info == &ip_info) + .is_none() + }); + + Ok::<_, Error>(()) }) - } else { - None - }; - - write_to.send_if_modified(|m| { - let public = m.get(&iface).map_or(None, |i| i.public); - m.insert( - iface.clone(), - NetworkInterfaceInfo { - public, - ip_info: ip_info.clone(), - }, - ) - .filter(|old| &old.ip_info == &ip_info) - .is_none() - }); - - Ok::<_, Error>(()) - }) - .await?; - } + .await?; + } + }) + .await }) .await?; } @@ -535,7 +547,7 @@ async fn watch_ip( pub struct NetworkInterfaceController { db: TypedPatchDb, - ip_info: watch::Sender>, + ip_info: Watch>, _watcher: NonDetachingJoinHandle<()>, listeners: SyncMutex>>, } @@ -605,10 +617,10 @@ impl NetworkInterfaceController { Ok(()) } pub fn new(db: TypedPatchDb) -> Self { - let (write_to, mut read_from) = watch::channel(BTreeMap::new()); + let mut ip_info = Watch::new(BTreeMap::new()); Self { db: db.clone(), - ip_info: write_to.clone(), + ip_info: ip_info.clone(), _watcher: tokio::spawn(async move { match db .peek() @@ -622,18 +634,18 @@ impl NetworkInterfaceController { for info in info.values_mut() { info.ip_info = None; } - write_to.send_replace(info); + ip_info.send_replace(info); } Err(e) => { tracing::error!("Error loading network interface info: {e}"); tracing::debug!("{e:?}"); } }; - tokio::join!(watcher(write_to), async { + tokio::join!(watcher(ip_info.clone()), async { let res: Result<(), Error> = async { loop { if let Err(e) = async { - let ip_info = { read_from.borrow().clone() }; + let ip_info = ip_info.read(); Self::sync(&db, &ip_info).boxed().await?; Ok::<_, Error>(()) @@ -644,12 +656,7 @@ impl NetworkInterfaceController { tracing::debug!("{e:?}"); } - read_from.changed().await.map_err(|_| { - Error::new( - eyre!("NetworkManager watch thread exited"), - ErrorKind::Network, - ) - })?; + ip_info.changed().await; } } .await; @@ -678,9 +685,32 @@ impl NetworkInterfaceController { })?; Ok(NetworkInterfaceListener { _arc: arc, - ip_info: self.ip_info.subscribe(), + ip_info: self.ip_info.clone(), listeners: ListenerMap::new(port), - needs_update: true, + }) + } + + pub fn upgrade_listener( + &self, + listener: impl IntoIterator, + ) -> Result { + let listeners = ListenerMap::from_listener(listener)?; + let port = listeners.port; + let arc = Arc::new(()); + self.listeners.mutate(|l| { + if l.get(&port).filter(|w| w.strong_count() > 0).is_some() { + return Err(Error::new( + std::io::Error::from_raw_os_error(libc::EADDRINUSE), + ErrorKind::Network, + )); + } + l.insert(port, Arc::downgrade(&arc)); + Ok(()) + })?; + Ok(NetworkInterfaceListener { + _arc: arc, + ip_info: self.ip_info.clone(), + listeners, }) } @@ -757,7 +787,44 @@ impl NetworkInterfaceController { struct ListenerMap { prev_public: bool, port: u16, - listeners: BTreeMap<(IpAddr, u32), (TcpListener, bool, Option)>, + listeners: BTreeMap)>, +} +impl ListenerMap { + fn from_listener(listener: impl IntoIterator) -> Result { + let mut prev_public = false; + let mut port = 0; + let mut listeners = BTreeMap::)>::new(); + for listener in listener { + let local = listener.local_addr().with_kind(ErrorKind::Network)?; + if port != 0 && port != local.port() { + return Err(Error::new( + eyre!("Provided listeners are bound to different ports"), + ErrorKind::InvalidRequest, + )); + } + let public = match local.ip() { + IpAddr::V4(ip4) => { + !ip4.is_loopback() + && (!ip4.is_private() || ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations + && !ip4.is_link_local() + } + IpAddr::V6(ip6) => !ipv6_is_local(ip6), + }; + port = local.port(); + listeners.insert(local, (listener, true, None)); + } + if port == 0 { + return return Err(Error::new( + eyre!("Listener array cannot be empty"), + ErrorKind::InvalidRequest, + )); + } + Ok(Self { + prev_public: true, + port, + listeners, + }) + } } impl ListenerMap { fn new(port: u16) -> Self { @@ -769,12 +836,12 @@ impl ListenerMap { } #[instrument(skip(self))] - async fn update( + fn update( &mut self, ip_info: &BTreeMap, public: bool, ) -> Result<(), Error> { - let mut keep = BTreeSet::<(IpAddr, u32)>::new(); + let mut keep = BTreeSet::::new(); for info in ip_info.values().chain([&NetworkInterfaceInfo { public: Some(false), ip_info: Some(IpInfo { @@ -792,24 +859,27 @@ impl ListenerMap { if public || !info.public() { if let Some(ip_info) = &info.ip_info { for ipnet in &ip_info.subnets { - let key = (ipnet.addr(), ip_info.scope_id); - keep.insert(key); - if let Some((_, is_public, wan_ip)) = self.listeners.get_mut(&key) { + let addr = match ipnet.addr() { + IpAddr::V6(ip6) => { + SocketAddrV6::new(ip6, self.port, 0, ip_info.scope_id).into() + } + ip => SocketAddr::new(ip, self.port), + }; + keep.insert(addr); + if let Some((_, is_public, wan_ip)) = self.listeners.get_mut(&addr) { *is_public = info.public(); *wan_ip = info.ip_info.as_ref().and_then(|i| i.wan_ip); continue; } self.listeners.insert( - key, + addr, ( - TcpListener::bind(match ipnet.addr() { - IpAddr::V6(ip6) => { - SocketAddrV6::new(ip6, self.port, 0, ip_info.scope_id) - .into() - } - ip => SocketAddr::new(ip, self.port), - }) - .await?, + TcpListener::from_std( + mio::net::TcpListener::bind(addr) + .with_kind(ErrorKind::Network)? + .into(), + ) + .with_kind(ErrorKind::Network)?, info.public(), info.ip_info.as_ref().and_then(|i| i.wan_ip), ), @@ -823,19 +893,16 @@ impl ListenerMap { crate::dbg!(&self.listeners); Ok(()) } - fn accept(&mut self) -> ListenerMapFut { - ListenerMapFut(&mut self.listeners) - } -} -#[pin_project::pin_project] -struct ListenerMapFut<'a>(&'a mut BTreeMap<(IpAddr, u32), (TcpListener, bool, Option)>); -impl<'a> Future for ListenerMapFut<'a> { - type Output = Result<(IpAddr, bool, Option, TcpStream, SocketAddr), Error>; - fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { - let this = self.project(); - for ((ip, _), listener) in this.0.iter() { + fn poll_accept(&self, cx: &mut std::task::Context<'_>) -> Poll> { + for (bind_addr, listener) in self.listeners.iter() { if let Poll::Ready((stream, addr)) = listener.0.poll_accept(cx)? { - return Poll::Ready(Ok((*ip, listener.1, listener.2, stream, addr))); + return Poll::Ready(Ok(Accepted { + stream, + peer: addr, + is_public: listener.1, + wan_ip: listener.2, + bind: *bind_addr, + })); } } Poll::Pending @@ -843,8 +910,7 @@ impl<'a> Future for ListenerMapFut<'a> { } pub struct NetworkInterfaceListener { - needs_update: bool, - ip_info: watch::Receiver>, + ip_info: Watch>, listeners: ListenerMap, _arc: Arc<()>, } @@ -853,34 +919,29 @@ impl NetworkInterfaceListener { self.listeners.port } + pub fn poll_accept( + &mut self, + cx: &mut std::task::Context<'_>, + public: bool, + ) -> Poll> { + if self.ip_info.poll_changed(cx).is_ready() || public != self.listeners.prev_public { + self.ip_info + .peek(|ip_info| self.listeners.update(ip_info, public))?; + } + self.listeners.poll_accept(cx) + } + pub async fn accept(&mut self, public: bool) -> Result { - self.needs_update |= public != self.listeners.prev_public; - loop { - if self.needs_update { - let ip_info = self.ip_info.borrow().clone(); - self.listeners.update(&ip_info, public).await?; - self.needs_update = false; - } - tokio::select! { - accepted = self.listeners.accept() => { - let (ip, is_public, wan_ip, stream, peer) = accepted?; - return Ok(Accepted { - stream, - peer, - is_public, - wan_ip, - bind: (ip, self.listeners.port).into(), - }) - }, - res = self.ip_info.changed() => { - res.map_err(|_| Error::new( - eyre!("NetworkManager watch thread exited"), - ErrorKind::Network, - ))?; - self.needs_update = true; - } + #[pin_project::pin_project] + struct Accept<'a>(&'a mut NetworkInterfaceListener, bool); + impl<'a> Future for Accept<'a> { + type Output = Result; + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + let this = self.project(); + this.0.poll_accept(cx, *this.1) } } + Accept(self, public).await } } diff --git a/core/startos/src/net/static_server.rs b/core/startos/src/net/static_server.rs index b6961df93..386f64d7b 100644 --- a/core/startos/src/net/static_server.rs +++ b/core/startos/src/net/static_server.rs @@ -8,7 +8,7 @@ use std::time::UNIX_EPOCH; use async_compression::tokio::bufread::GzipEncoder; use axum::body::Body; use axum::extract::{self as x, Request}; -use axum::response::Response; +use axum::response::{Redirect, Response}; use axum::routing::{any, get}; use axum::Router; use base64::display::Base64Display; @@ -16,7 +16,7 @@ use digest::Digest; use futures::future::ready; use http::header::{ ACCEPT_ENCODING, ACCEPT_RANGES, CACHE_CONTROL, CONNECTION, CONTENT_ENCODING, CONTENT_LENGTH, - CONTENT_RANGE, CONTENT_TYPE, ETAG, RANGE, + CONTENT_RANGE, CONTENT_TYPE, ETAG, HOST, RANGE, }; use http::request::Parts as RequestParts; use http::{HeaderValue, Method, StatusCode}; @@ -229,6 +229,20 @@ pub fn refresher() -> Router { })) } +pub fn redirecter() -> Router { + Router::new().fallback(get(|request: Request| async move { + Redirect::temporary(&format!( + "https://{}{}", + request + .headers() + .get(HOST) + .and_then(|s| s.to_str().ok()) + .unwrap_or("localhost"), + request.uri() + )) + })) +} + async fn proxy_request(ctx: RpcContext, request: Request, url: String) -> Result { if_authorized(&ctx, request, |mut request| async { for header in PROXY_STRIP_HEADERS { diff --git a/core/startos/src/net/utils.rs b/core/startos/src/net/utils.rs index 2505611da..be65fb99b 100644 --- a/core/startos/src/net/utils.rs +++ b/core/startos/src/net/utils.rs @@ -1,19 +1,22 @@ -use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; use std::path::Path; use async_stream::try_stream; use color_eyre::eyre::eyre; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; -use ipnet::{Ipv4Net, Ipv6Net}; +use getifaddrs::if_nametoindex; +use ipnet::{IpNet, Ipv4Net, Ipv6Net}; use tokio::net::{TcpListener, TcpStream}; use tokio::process::Command; +use crate::prelude::*; use crate::util::Invoke; -use crate::Error; pub fn ipv6_is_local(addr: Ipv6Addr) -> bool { - (addr.segments()[0] & 0xfe00) == 0xfc00 || (addr.segments()[0] & 0xffc0) == 0xfe80 + addr.is_loopback() + || (addr.segments()[0] & 0xfe00) == 0xfc00 + || (addr.segments()[0] & 0xffc0) == 0xfe80 } fn parse_iface_ip(output: &str) -> Result, Error> { @@ -116,6 +119,52 @@ pub async fn find_eth_iface() -> Result { )) } +pub async fn all_socket_addrs_for(port: u16) -> Result, Error> { + let mut res = Vec::new(); + + let raw = String::from_utf8( + Command::new("ip") + .arg("-o") + .arg("addr") + .arg("show") + .invoke(ErrorKind::ParseSysInfo) + .await?, + )?; + let err = |item: &str, lineno: usize, line: &str| { + Error::new( + eyre!("failed to parse ip info ({item}[line:{lineno}]) from {line:?}"), + ErrorKind::ParseSysInfo, + ) + }; + for (idx, line) in raw + .lines() + .map(|l| l.trim()) + .enumerate() + .filter(|(_, l)| !l.is_empty()) + { + let mut split = line.split_whitespace(); + let _num = split.next(); + let ifname = split.next().ok_or_else(|| err("ifname", idx, line))?; + let _kind = split.next(); + let ipnet_str = split.next().ok_or_else(|| err("ipnet", idx, line))?; + let ipnet = ipnet_str + .parse::() + .with_ctx(|_| (ErrorKind::ParseSysInfo, err("ipnet", idx, ipnet_str)))?; + match ipnet.addr() { + IpAddr::V4(ip4) => res.push(SocketAddr::new(ip4.into(), port)), + IpAddr::V6(ip6) => res.push(SocketAddr::V6(SocketAddrV6::new( + ip6, + port, + 0, + if_nametoindex(ifname) + .with_ctx(|_| (ErrorKind::ParseSysInfo, "reading scope_id"))?, + ))), + } + } + + Ok(res) +} + pub struct TcpListeners { listeners: Vec, } diff --git a/core/startos/src/net/web_server.rs b/core/startos/src/net/web_server.rs index d1ad64d01..15dfa9810 100644 --- a/core/startos/src/net/web_server.rs +++ b/core/startos/src/net/web_server.rs @@ -1,134 +1,290 @@ -use std::convert::Infallible; +use std::future::Future; use std::net::SocketAddr; +use std::ops::Deref; +use std::pin::Pin; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, RwLock}; use std::task::Poll; use std::time::Duration; -use axum::extract::Request; use axum::Router; -use axum_server::Handle; -use bytes::Bytes; -use futures::future::{ready, BoxFuture}; -use futures::FutureExt; +use futures::future::Either; +use futures::{FutureExt, StreamExt}; use helpers::NonDetachingJoinHandle; +use hyper_util::rt::{TokioIo, TokioTimer}; +use hyper_util::service::TowerToHyperService; +use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{oneshot, watch}; use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; +use crate::net::network_interface::NetworkInterfaceListener; use crate::net::static_server::{ - diagnostic_ui_router, init_ui_router, install_ui_router, main_ui_router, refresher, + diagnostic_ui_router, init_ui_router, install_ui_router, main_ui_router, redirecter, refresher, setup_ui_router, }; use crate::prelude::*; +use crate::util::actor::background::BackgroundJobQueue; +use crate::util::sync::Watch; -#[derive(Clone)] -pub struct SwappableRouter(watch::Sender); -impl SwappableRouter { - pub fn new(router: Router) -> Self { - Self(watch::channel(router).0) - } - pub fn swap(&self, router: Router) { - let _ = self.0.send_replace(router); - } +pub struct Accepted { + pub https_redirect: bool, + pub stream: TcpStream, } -pub struct SwappableRouterService { - router: watch::Receiver, - changed: Option>, +pub trait Accept { + fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll>; } -impl SwappableRouterService { - fn router(&self) -> Router { - self.router.borrow().clone() - } - fn changed(&mut self, cx: &mut std::task::Context<'_>) -> Poll<()> { - let mut changed = if let Some(changed) = self.changed.take() { - changed - } else { - let mut router = self.router.clone(); - async move { - router.changed().await; + +impl Accept for Vec { + fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + for listener in &*self { + if let Poll::Ready((stream, _)) = listener.poll_accept(cx)? { + return Poll::Ready(Ok(Accepted { + https_redirect: false, + stream, + })); } - .boxed() - }; - if changed.poll_unpin(cx).is_ready() { - return Poll::Ready(()); } - self.changed = Some(changed); Poll::Pending } } -impl Clone for SwappableRouterService { - fn clone(&self) -> Self { - Self { - router: self.router.clone(), - changed: None, +impl Accept for NetworkInterfaceListener { + fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + NetworkInterfaceListener::poll_accept(self, cx, true).map(|res| { + res.map(|a| Accepted { + https_redirect: a.is_public, + stream: a.stream, + }) + }) + } +} + +impl Accept for Either { + fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + match self { + Either::Left(a) => a.poll_accept(cx), + Either::Right(b) => b.poll_accept(cx), } } } -impl tower_service::Service> for SwappableRouterService -where - B: axum::body::HttpBody + Send + 'static, - B::Error: Into, -{ - type Response = >>::Response; - type Error = >>::Error; - type Future = >>::Future; - #[inline] - fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - if self.changed(cx).is_ready() { - return Poll::Ready(Ok(())); +impl Accept for Option { + fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + match self { + None => Poll::Pending, + Some(a) => a.poll_accept(cx), } - tower_service::Service::>::poll_ready(&mut self.router(), cx) } - fn call(&mut self, req: Request) -> Self::Future { - self.router().call(req) +} + +#[pin_project::pin_project] +pub struct Acceptor { + acceptor: Watch, +} +impl Acceptor { + pub fn new(acceptor: A) -> Self { + Self { + acceptor: Watch::new(acceptor), + } + } + + fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.acceptor.poll_changed(cx); + let mut res = Poll::Pending; + self.acceptor.send_if_modified(|a| { + res = a.poll_accept(cx); + false + }); + res + } + + async fn accept(&mut self) -> Result { + #[pin_project::pin_project] + struct AcceptFut<'a, A: Accept>(&'a mut Acceptor); + impl<'a, A: Accept> Future for AcceptFut<'a, A> { + type Output = Result; + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + self.project().0.poll_accept(cx) + } + } + AcceptFut(self).await } } +impl Acceptor> { + pub async fn bind(listen: impl IntoIterator) -> Result { + Ok(Self::new( + futures::future::try_join_all(listen.into_iter().map(TcpListener::bind)).await?, + )) + } +} + +pub type UpgradableListener = Option, NetworkInterfaceListener>>; -impl tower_service::Service for SwappableRouter { - type Response = SwappableRouterService; - type Error = Infallible; - type Future = futures::future::Ready>; - #[inline] - fn poll_ready( - &mut self, - _: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - Poll::Ready(Ok(())) - } - fn call(&mut self, _: T) -> Self::Future { - ready(Ok(SwappableRouterService { - router: self.0.subscribe(), - changed: None, - })) +impl Acceptor { + pub async fn bind_upgradable( + listen: impl IntoIterator, + ) -> Result { + Ok(Self::new(Some(Either::Left( + futures::future::try_join_all(listen.into_iter().map(TcpListener::bind)).await?, + )))) } } -pub struct WebServer { +pub struct WebServerAcceptorSetter { + acceptor: Watch, +} +impl WebServerAcceptorSetter>> { + pub fn try_upgrade Result>(&self, f: F) -> Result<(), Error> { + let mut res = Ok(()); + self.acceptor.send_modify(|a| { + *a = match a.take() { + Some(Either::Left(a)) => match f(a) { + Ok(b) => Some(Either::Right(b)), + Err(e) => { + res = Err(e); + None + } + }, + x => x, + } + }); + res + } +} +impl Deref for WebServerAcceptorSetter { + type Target = Watch; + fn deref(&self) -> &Self::Target { + &self.acceptor + } +} + +pub struct WebServer { shutdown: oneshot::Sender<()>, - router: SwappableRouter, + router: watch::Sender>, + acceptor: Watch, thread: NonDetachingJoinHandle<()>, } -impl WebServer { - pub fn new(bind: SocketAddr) -> Self { - let router = SwappableRouter::new(refresher()); - let thread_router = router.clone(); +impl WebServer { + pub fn acceptor_setter(&self) -> WebServerAcceptorSetter { + WebServerAcceptorSetter { + acceptor: self.acceptor.clone(), + } + } + + pub fn new(mut acceptor: Acceptor) -> Self { + let acceptor_send = acceptor.acceptor.clone(); + let (router, service) = watch::channel::>(None); let (shutdown, shutdown_recv) = oneshot::channel(); let thread = NonDetachingJoinHandle::from(tokio::spawn(async move { - let handle = Handle::new(); - let mut server = axum_server::bind(bind).handle(handle.clone()); - server.http_builder().http1().preserve_header_case(true); - server.http_builder().http1().title_case_headers(true); - - if let (Err(e), _) = tokio::join!(server.serve(thread_router), async { - let _ = shutdown_recv.await; - handle.graceful_shutdown(Some(Duration::from_secs(0))); - }) { - tracing::error!("Spawning hyper server error: {}", e); + #[derive(Clone)] + struct QueueRunner { + queue: Arc>>, + } + impl hyper::rt::Executor for QueueRunner + where + Fut: Future + Send + 'static, + { + fn execute(&self, fut: Fut) { + if let Some(q) = &*self.queue.read().unwrap() { + q.add_job(fut); + } else { + tracing::warn!("job queued after shutdown"); + } + } + } + + let accept = AtomicBool::new(true); + let queue_cell = Arc::new(RwLock::new(None)); + let graceful = hyper_util::server::graceful::GracefulShutdown::new(); + let mut server = hyper_util::server::conn::auto::Builder::new(QueueRunner { + queue: queue_cell.clone(), + }); + server + .http1() + .timer(TokioTimer::new()) + .title_case_headers(true) + .preserve_header_case(true) + .http2() + .timer(TokioTimer::new()) + .enable_connect_protocol() + .keep_alive_interval(Duration::from_secs(60)) + .keep_alive_timeout(Duration::from_secs(300)); + let (queue, mut runner) = BackgroundJobQueue::new(); + *queue_cell.write().unwrap() = Some(queue.clone()); + + let handler = async { + loop { + if let Err(e) = async { + let accepted = acceptor.accept().await?; + if accepted.https_redirect { + queue.add_job( + graceful.watch( + server + .serve_connection_with_upgrades( + TokioIo::new(accepted.stream), + TowerToHyperService::new(redirecter().into_service()), + ) + .into_owned(), + ), + ); + } else { + let service = service.borrow().clone(); + if let Some(service) = service { + queue.add_job( + graceful.watch( + server + .serve_connection_with_upgrades( + TokioIo::new(accepted.stream), + TowerToHyperService::new(service.into_service()), + ) + .into_owned(), + ), + ); + } else { + queue.add_job( + graceful.watch( + server + .serve_connection_with_upgrades( + TokioIo::new(accepted.stream), + TowerToHyperService::new( + refresher().into_service(), + ), + ) + .into_owned(), + ), + ); + } + } + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error accepting HTTP connection: {e}"); + tracing::debug!("{e:?}"); + } + } + } + .boxed(); + + tokio::select! { + _ = shutdown_recv => (), + _ = handler => (), + _ = &mut runner => (), + } + + accept.store(false, std::sync::atomic::Ordering::SeqCst); + drop(queue); + drop(queue_cell.write().unwrap().take()); + + if !runner.is_empty() { + runner.await; } })); Self { shutdown, router, thread, + acceptor: acceptor_send, } } @@ -138,7 +294,7 @@ impl WebServer { } pub fn serve_router(&mut self, router: Router) { - self.router.swap(router) + self.router.send_replace(Some(router)); } pub fn serve_main(&mut self, ctx: RpcContext) { diff --git a/core/startos/src/registry/mod.rs b/core/startos/src/registry/mod.rs index 9bb605d92..b56cedce6 100644 --- a/core/startos/src/registry/mod.rs +++ b/core/startos/src/registry/mod.rs @@ -10,7 +10,7 @@ use ts_rs::TS; use crate::context::CliContext; use crate::middleware::cors::Cors; use crate::net::static_server::{bad_request, not_found, server_error}; -use crate::net::web_server::WebServer; +use crate::net::web_server::{Accept, WebServer}; use crate::prelude::*; use crate::registry::auth::Auth; use crate::registry::context::RegistryContext; @@ -143,7 +143,7 @@ pub fn registry_router(ctx: RegistryContext) -> Router { ) } -impl WebServer { +impl WebServer { pub fn serve_registry(&mut self, ctx: RegistryContext) { self.serve_router(registry_router(ctx)) } diff --git a/core/startos/src/setup.rs b/core/startos/src/setup.rs index 1319ffae4..f2aba4328 100644 --- a/core/startos/src/setup.rs +++ b/core/startos/src/setup.rs @@ -80,7 +80,7 @@ async fn setup_init( password: Option, init_phases: InitPhases, ) -> Result<(AccountInfo, PreInitNetController), Error> { - let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; + let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?; let account = net_ctrl .db @@ -167,7 +167,7 @@ pub async fn attach( let (account, net_ctrl) = setup_init(&setup_ctx, password, init_phases).await?; - let rpc_ctx = RpcContext::init(&setup_ctx.config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?; + let rpc_ctx = RpcContext::init(&setup_ctx.webserver, &setup_ctx.config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?; Ok(((&account).try_into()?, rpc_ctx)) })?; @@ -456,9 +456,16 @@ async fn fresh_setup( db.put(&ROOT, &Database::init(&account)?).await?; drop(db); - let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; + let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?; - let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?; + let rpc_ctx = RpcContext::init( + &ctx.webserver, + &ctx.config, + guid, + Some(net_ctrl), + rpc_ctx_phases, + ) + .await?; Ok(((&account).try_into()?, rpc_ctx)) } @@ -571,7 +578,14 @@ async fn migrate( let (account, net_ctrl) = setup_init(&ctx, Some(start_os_password), init_phases).await?; - let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?; + let rpc_ctx = RpcContext::init( + &ctx.webserver, + &ctx.config, + guid, + Some(net_ctrl), + rpc_ctx_phases, + ) + .await?; Ok(((&account).try_into()?, rpc_ctx)) } diff --git a/core/startos/src/util/actor/background.rs b/core/startos/src/util/actor/background.rs index f37e10c14..7666cbf04 100644 --- a/core/startos/src/util/actor/background.rs +++ b/core/startos/src/util/actor/background.rs @@ -15,8 +15,13 @@ impl BackgroundJobQueue { }, ) } - pub fn add_job(&self, fut: impl Future + Send + 'static) { - let _ = self.0.send(fut.boxed()); + pub fn add_job(&self, fut: impl Future + Send + 'static) { + let _ = self.0.send( + async { + fut.await; + } + .boxed(), + ); } } diff --git a/core/startos/src/util/future.rs b/core/startos/src/util/future.rs index c690f9754..2ef053fed 100644 --- a/core/startos/src/util/future.rs +++ b/core/startos/src/util/future.rs @@ -1,5 +1,6 @@ use std::pin::Pin; -use std::task::{Context, Poll}; +use std::sync::atomic::AtomicUsize; +use std::task::{Context, Poll, Waker}; use futures::future::{abortable, pending, BoxFuture, FusedFuture}; use futures::stream::{AbortHandle, Abortable, BoxStream}; @@ -7,6 +8,7 @@ use futures::{Future, FutureExt, Stream, StreamExt}; use tokio::sync::watch; use crate::prelude::*; +use crate::util::sync::SyncMutex; #[pin_project::pin_project(PinnedDrop)] pub struct DropSignaling { diff --git a/core/startos/src/util/sync.rs b/core/startos/src/util/sync.rs index 1edd21ce1..2bf1e767d 100644 --- a/core/startos/src/util/sync.rs +++ b/core/startos/src/util/sync.rs @@ -1,3 +1,10 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::AtomicUsize; +use std::sync::Arc; +use std::task::{Poll, Waker}; + +#[derive(Debug, Default)] pub struct SyncMutex(std::sync::Mutex); impl SyncMutex { pub fn new(t: T) -> Self { @@ -10,3 +17,122 @@ impl SyncMutex { f(&*self.0.lock().unwrap()) } } + +struct WatchData { + data: T, + wakers: Vec, +} + +struct Shared { + data: SyncMutex>, + version: AtomicUsize, +} + +pub struct Watch { + data: Arc>, + seen_version: usize, +} +impl Clone for Watch { + fn clone(&self) -> Self { + Self { + data: self.data.clone(), + seen_version: self.seen_version, + } + } +} +impl Watch { + pub fn new(init: T) -> Self { + Self { + data: Arc::new(Shared { + data: SyncMutex::new(WatchData { + data: init, + wakers: Vec::new(), + }), + version: AtomicUsize::new(1), + }), + seen_version: 1, + } + } + + pub fn poll_changed(&mut self, cx: &mut std::task::Context<'_>) -> Poll<()> { + let mut guard = self.data.data.0.lock().unwrap(); + let version = self.data.version.load(std::sync::atomic::Ordering::SeqCst); + if version > self.seen_version { + self.seen_version = version; + return Poll::Ready(()); + } + let waker = cx.waker(); + if !guard.wakers.iter().any(|w| w.will_wake(waker)) { + guard.wakers.push(waker.clone()); + } + Poll::Pending + } + + pub async fn changed(&mut self) { + #[pin_project::pin_project] + struct Changed<'a, T>(&'a mut Watch); + impl<'a, T> Future for Changed<'a, T> { + type Output = (); + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + let this = self.project(); + this.0.poll_changed(cx) + } + } + Changed(self).await + } + + pub fn mark_changed(&mut self) { + self.seen_version = 0; + } + + pub fn peek(&self, peek: F) -> U + where + F: FnOnce(&T) -> U, + { + self.data.data.peek(|d| peek(&d.data)) + } + + pub fn send_if_modified(&self, modify: F) -> bool + where + F: FnOnce(&mut T) -> bool, + { + let mut guard = self.data.data.0.lock().unwrap(); + let changed = modify(&mut guard.data); + if changed { + self.data + .version + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + for waker in guard.wakers.drain(..) { + waker.wake(); + } + } + changed + } + + pub fn send_modify(&self, modify: F) + where + F: FnOnce(&mut T), + { + self.send_if_modified(|x| { + modify(x); + true + }); + } + + pub fn send_replace(&self, mut value: T) -> T { + self.send_modify(|x| { + std::mem::swap(x, &mut value); + }); + value + } + + pub fn send(&self, value: T) { + self.send_replace(value); + } +} + +impl Watch { + pub fn read(&self) -> T { + self.data.data.0.lock().unwrap().data.clone() + } +} From 7546c361479f58e3c86f850409400b3099b325a0 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 2 Jan 2025 17:41:00 -0700 Subject: [PATCH 24/29] misc fixes --- core/Cargo.lock | 12 - core/startos/Cargo.toml | 2 +- core/startos/src/context/rpc.rs | 2 +- core/startos/src/db/model/public.rs | 2 +- core/startos/src/net/net_controller.rs | 1 + core/startos/src/net/network_interface.rs | 215 ++++++++++-------- core/startos/src/net/service_interface.rs | 2 + core/startos/src/net/utils.rs | 10 +- core/startos/src/net/web_server.rs | 13 +- sdk/base/lib/osBindings/IpHostname.ts | 8 +- sdk/base/lib/types.ts | 27 --- sdk/base/lib/util/getServiceInterface.ts | 4 +- .../app-interfaces/app-interfaces.page.ts | 68 +++--- .../server-specs/server-specs.page.html | 28 ++- .../server-specs/server-specs.page.ts | 1 + .../ui/src/app/services/api/api.fixures.ts | 6 +- .../ui/src/app/services/api/mock-patch.ts | 6 +- 17 files changed, 214 insertions(+), 193 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index b8cbe8d5b..26a839a78 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2224,17 +2224,6 @@ dependencies = [ "webpki-roots 0.26.7", ] -[[package]] -name = "getifaddrs" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba121d81ab5ea05b0cd5858516266800bf965531a794f7ac58e3eeb804f364f" -dependencies = [ - "bitflags 2.6.0", - "libc", - "windows-sys 0.59.0", -] - [[package]] name = "getrandom" version = "0.1.16" @@ -5660,7 +5649,6 @@ dependencies = [ "fd-lock-rs", "form_urlencoded", "futures", - "getifaddrs", "gpt", "helpers", "hex", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 2e9e49f91..29c781af2 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -95,7 +95,6 @@ exver = { version = "0.2.0", git = "https://github.com/Start9Labs/exver-rs.git", fd-lock-rs = "0.1.4" form_urlencoded = "1.2.1" futures = "0.3.28" -getifaddrs = "0.1.5" gpt = "3.1.0" helpers = { path = "../helpers" } hex = "0.4.3" @@ -142,6 +141,7 @@ new_mime_guess = "4" nix = { version = "0.29.0", features = [ "fs", "mount", + "net", "process", "sched", "signal", diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index 5b65f3d6d..12de59e05 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -161,7 +161,7 @@ impl RpcContext { account.tor_key.clone(), ) .await?; - webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a)); + webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?; net_ctrl }, config diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index f5ab2120c..92b9c8a88 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -174,7 +174,7 @@ impl NetworkInterfaceInfo { || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations || ip4.is_link_local() } - IpAddr::V6(ip6) => true, + IpAddr::V6(_) => true, } }) }) diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 518384925..cdf2607b7 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -473,6 +473,7 @@ impl NetService { public: public && !ipv6_is_local(net.addr()), hostname: IpHostname::Ipv6 { value: net.addr(), + scope_id: ip_info.scope_id, port: bind.net.assigned_port, ssl_port: bind.net.assigned_ssl_port, }, diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index bdb81edae..e732c1ae7 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -8,11 +8,11 @@ use std::time::Duration; use clap::Parser; use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; -use getifaddrs::if_nametoindex; use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use ipnet::IpNet; use itertools::Itertools; +use nix::net::if_::if_nametoindex; use patch_db::json_ptr::JsonPointer; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; @@ -29,7 +29,7 @@ use zbus::{proxy, Connection}; use crate::context::{CliContext, RpcContext}; use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; use crate::db::model::Database; -use crate::net::utils::ipv6_is_local; +use crate::net::utils::{ipv6_is_link_local, ipv6_is_local}; use crate::prelude::*; use crate::util::future::Until; use crate::util::io::open_file; @@ -396,7 +396,7 @@ async fn get_wan_ipv4(iface: &str) -> Result, Error> { .transpose()?) } -#[instrument(skip_all)] +#[instrument(skip(connection, device_proxy, write_to))] async fn watch_ip( connection: &Connection, device_proxy: DeviceProxy<'_>, @@ -434,6 +434,7 @@ async fn watch_ip( if &*dac == "/" { return Ok(()); } + let active_connection_proxy = active_connection::ActiveConnectionProxy::new(&connection, dac).await?; @@ -452,94 +453,100 @@ async fn watch_ip( .stub(), ); - until - .run(async { - let external = active_connection_proxy.state_flags().await? & 0x80 != 0; - if external { - return Ok(()); - } - - let dhcp4_config = active_connection_proxy.dhcp4_config().await?; - let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config).await?; - let ip6_proxy = Ip6ConfigProxy::new(&connection, ip6_config).await?; - let mut until = Until::new() - .with_stream(ip4_proxy.receive_address_data_changed().await.stub()) - .with_stream(ip6_proxy.receive_address_data_changed().await.stub()); - - let dhcp4_proxy = if &*dhcp4_config != "/" { - let dhcp4_proxy = - Dhcp4ConfigProxy::new(&connection, dhcp4_config).await?; - until = until - .with_stream(dhcp4_proxy.receive_options_changed().await.stub()); - Some(dhcp4_proxy) - } else { - None - }; + loop { + until + .run(async { + let external = active_connection_proxy.state_flags().await? & 0x80 != 0; + if external { + return Ok(()); + } - loop { - until - .run(async { - let addresses = ip4_proxy - .address_data() - .await? - .into_iter() - .chain(ip6_proxy.address_data().await?) - .collect_vec(); - let mut ntp_servers = BTreeSet::new(); - if let Some(dhcp4_proxy) = &dhcp4_proxy { - let dhcp = crate::dbg!(dhcp4_proxy.options().await?); - if let Some(ntp) = dhcp.ntp_servers { - ntp_servers.extend( - ntp.split_whitespace().map(InternedString::intern), - ); + let dhcp4_config = active_connection_proxy.dhcp4_config().await?; + let ip4_proxy = + Ip4ConfigProxy::new(&connection, ip4_config.clone()).await?; + let ip6_proxy = + Ip6ConfigProxy::new(&connection, ip6_config.clone()).await?; + let mut until = Until::new() + .with_stream(ip4_proxy.receive_address_data_changed().await.stub()) + .with_stream(ip6_proxy.receive_address_data_changed().await.stub()); + + let dhcp4_proxy = if &*dhcp4_config != "/" { + let dhcp4_proxy = + Dhcp4ConfigProxy::new(&connection, dhcp4_config).await?; + until = until.with_stream( + dhcp4_proxy.receive_options_changed().await.stub(), + ); + Some(dhcp4_proxy) + } else { + None + }; + + loop { + until + .run(async { + let addresses = ip4_proxy + .address_data() + .await? + .into_iter() + .chain(ip6_proxy.address_data().await?) + .collect_vec(); + let mut ntp_servers = BTreeSet::new(); + if let Some(dhcp4_proxy) = &dhcp4_proxy { + let dhcp = dhcp4_proxy.options().await?; + if let Some(ntp) = dhcp.ntp_servers { + ntp_servers.extend( + ntp.split_whitespace() + .map(InternedString::intern), + ); + } } - } - let scope_id = - if_nametoindex(&*iface).with_kind(ErrorKind::Network)?; - let subnets: BTreeSet = addresses - .into_iter() - .map(TryInto::try_into) - .try_collect()?; - let ip_info = if !subnets.is_empty() { - let wan_ip = match get_wan_ipv4(&*iface).await { - Ok(a) => a, - Err(e) => { - tracing::error!( + let scope_id = if_nametoindex(&*iface) + .with_kind(ErrorKind::Network)?; + let subnets: BTreeSet = addresses + .into_iter() + .map(TryInto::try_into) + .try_collect()?; + let ip_info = if !subnets.is_empty() { + let wan_ip = match get_wan_ipv4(&*iface).await { + Ok(a) => a, + Err(e) => { + tracing::error!( "Failed to determine WAN IP for {iface}: {e}" ); - tracing::debug!("{e:?}"); - None - } + tracing::debug!("{e:?}"); + None + } + }; + Some(IpInfo { + scope_id, + subnets, + wan_ip, + ntp_servers, + }) + } else { + None }; - Some(IpInfo { - scope_id, - subnets, - wan_ip, - ntp_servers, - }) - } else { - None - }; - - write_to.send_if_modified(|m| { - let public = m.get(&iface).map_or(None, |i| i.public); - m.insert( - iface.clone(), - NetworkInterfaceInfo { - public, - ip_info: ip_info.clone(), - }, - ) - .filter(|old| &old.ip_info == &ip_info) - .is_none() - }); - - Ok::<_, Error>(()) - }) - .await?; - } - }) - .await + + write_to.send_if_modified(|m| { + let public = m.get(&iface).map_or(None, |i| i.public); + m.insert( + iface.clone(), + NetworkInterfaceInfo { + public, + ip_info: ip_info.clone(), + }, + ) + .filter(|old| &old.ip_info == &ip_info) + .is_none() + }); + + Ok::<_, Error>(()) + }) + .await?; + } + }) + .await?; + } }) .await?; } @@ -795,7 +802,12 @@ impl ListenerMap { let mut port = 0; let mut listeners = BTreeMap::)>::new(); for listener in listener { - let local = listener.local_addr().with_kind(ErrorKind::Network)?; + let mut local = listener.local_addr().with_kind(ErrorKind::Network)?; + if let SocketAddr::V6(l) = &mut local { + if ipv6_is_link_local(*l.ip()) && l.scope_id() == 0 { + continue; // TODO determine scope id + } + } if port != 0 && port != local.port() { return Err(Error::new( eyre!("Provided listeners are bound to different ports"), @@ -810,17 +822,18 @@ impl ListenerMap { } IpAddr::V6(ip6) => !ipv6_is_local(ip6), }; + prev_public |= public; port = local.port(); - listeners.insert(local, (listener, true, None)); + listeners.insert(local, (listener, public, None)); } if port == 0 { - return return Err(Error::new( + return Err(Error::new( eyre!("Listener array cannot be empty"), ErrorKind::InvalidRequest, )); } Ok(Self { - prev_public: true, + prev_public, port, listeners, }) @@ -860,9 +873,17 @@ impl ListenerMap { if let Some(ip_info) = &info.ip_info { for ipnet in &ip_info.subnets { let addr = match ipnet.addr() { - IpAddr::V6(ip6) => { - SocketAddrV6::new(ip6, self.port, 0, ip_info.scope_id).into() - } + IpAddr::V6(ip6) => SocketAddrV6::new( + ip6, + self.port, + 0, + if ipv6_is_link_local(ip6) { + ip_info.scope_id + } else { + 0 + }, + ) + .into(), ip => SocketAddr::new(ip, self.port), }; keep.insert(addr); @@ -876,7 +897,12 @@ impl ListenerMap { ( TcpListener::from_std( mio::net::TcpListener::bind(addr) - .with_kind(ErrorKind::Network)? + .with_ctx(|_| { + ( + ErrorKind::Network, + lazy_format!("binding to {addr:?}"), + ) + })? .into(), ) .with_kind(ErrorKind::Network)?, @@ -890,7 +916,6 @@ impl ListenerMap { } self.listeners.retain(|key, _| keep.contains(key)); self.prev_public = public; - crate::dbg!(&self.listeners); Ok(()) } fn poll_accept(&self, cx: &mut std::task::Context<'_>) -> Poll> { diff --git a/core/startos/src/net/service_interface.rs b/core/startos/src/net/service_interface.rs index 95d7115cd..ad2900da7 100644 --- a/core/startos/src/net/service_interface.rs +++ b/core/startos/src/net/service_interface.rs @@ -44,6 +44,8 @@ pub enum IpHostname { }, Ipv6 { value: Ipv6Addr, + #[serde(default)] + scope_id: u32, port: Option, ssl_port: Option, }, diff --git a/core/startos/src/net/utils.rs b/core/startos/src/net/utils.rs index be65fb99b..d6dcdde15 100644 --- a/core/startos/src/net/utils.rs +++ b/core/startos/src/net/utils.rs @@ -5,18 +5,20 @@ use async_stream::try_stream; use color_eyre::eyre::eyre; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; -use getifaddrs::if_nametoindex; use ipnet::{IpNet, Ipv4Net, Ipv6Net}; +use nix::net::if_::if_nametoindex; use tokio::net::{TcpListener, TcpStream}; use tokio::process::Command; use crate::prelude::*; use crate::util::Invoke; +pub fn ipv6_is_link_local(addr: Ipv6Addr) -> bool { + (addr.segments()[0] & 0xffc0) == 0xfe80 +} + pub fn ipv6_is_local(addr: Ipv6Addr) -> bool { - addr.is_loopback() - || (addr.segments()[0] & 0xfe00) == 0xfc00 - || (addr.segments()[0] & 0xffc0) == 0xfe80 + addr.is_loopback() || (addr.segments()[0] & 0xfe00) == 0xfc00 || ipv6_is_link_local(addr) } fn parse_iface_ip(output: &str) -> Result, Error> { diff --git a/core/startos/src/net/web_server.rs b/core/startos/src/net/web_server.rs index 15dfa9810..113273b04 100644 --- a/core/startos/src/net/web_server.rs +++ b/core/startos/src/net/web_server.rs @@ -9,12 +9,12 @@ use std::time::Duration; use axum::Router; use futures::future::Either; -use futures::{FutureExt, StreamExt}; +use futures::FutureExt; use helpers::NonDetachingJoinHandle; use hyper_util::rt::{TokioIo, TokioTimer}; use hyper_util::service::TowerToHyperService; use tokio::net::{TcpListener, TcpStream}; -use tokio::sync::{oneshot, watch}; +use tokio::sync::oneshot; use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::net::network_interface::NetworkInterfaceListener; @@ -88,7 +88,7 @@ impl Acceptor { } fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - self.acceptor.poll_changed(cx); + let _ = self.acceptor.poll_changed(cx); let mut res = Poll::Pending; self.acceptor.send_if_modified(|a| { res = a.poll_accept(cx); @@ -159,7 +159,7 @@ impl Deref for WebServerAcceptorSetter { pub struct WebServer { shutdown: oneshot::Sender<()>, - router: watch::Sender>, + router: Watch>, acceptor: Watch, thread: NonDetachingJoinHandle<()>, } @@ -172,7 +172,8 @@ impl WebServer { pub fn new(mut acceptor: Acceptor) -> Self { let acceptor_send = acceptor.acceptor.clone(); - let (router, service) = watch::channel::>(None); + let router = Watch::>::new(None); + let service = router.clone(); let (shutdown, shutdown_recv) = oneshot::channel(); let thread = NonDetachingJoinHandle::from(tokio::spawn(async move { #[derive(Clone)] @@ -227,7 +228,7 @@ impl WebServer { ), ); } else { - let service = service.borrow().clone(); + let service = service.read(); if let Some(service) = service { queue.add_job( graceful.watch( diff --git a/sdk/base/lib/osBindings/IpHostname.ts b/sdk/base/lib/osBindings/IpHostname.ts index 4a6b5e87c..9b3ddd6d1 100644 --- a/sdk/base/lib/osBindings/IpHostname.ts +++ b/sdk/base/lib/osBindings/IpHostname.ts @@ -2,7 +2,13 @@ export type IpHostname = | { kind: "ipv4"; value: string; port: number | null; sslPort: number | null } - | { kind: "ipv6"; value: string; port: number | null; sslPort: number | null } + | { + kind: "ipv6" + value: string + scopeId: number + port: number | null + sslPort: number | null + } | { kind: "local" value: string diff --git a/sdk/base/lib/types.ts b/sdk/base/lib/types.ts index 36d4bd293..85a8c4404 100644 --- a/sdk/base/lib/types.ts +++ b/sdk/base/lib/types.ts @@ -138,33 +138,6 @@ export declare const hostName: unique symbol // asdflkjadsf.onion | 1.2.3.4 export type Hostname = string & { [hostName]: never } -export type HostnameInfoIp = { - kind: "ip" - networkInterfaceId: string - public: boolean - hostname: - | { - kind: "ipv4" | "ipv6" | "local" - value: string - port: number | null - sslPort: number | null - } - | { - kind: "domain" - domain: string - subdomain: string | null - port: number | null - sslPort: number | null - } -} - -export type HostnameInfoOnion = { - kind: "onion" - hostname: { value: string; port: number | null; sslPort: number | null } -} - -export type HostnameInfo = HostnameInfoIp | HostnameInfoOnion - export type ServiceInterfaceId = string export { ServiceInterface } diff --git a/sdk/base/lib/util/getServiceInterface.ts b/sdk/base/lib/util/getServiceInterface.ts index 393c4f9a7..2e81e5ee2 100644 --- a/sdk/base/lib/util/getServiceInterface.ts +++ b/sdk/base/lib/util/getServiceInterface.ts @@ -74,7 +74,9 @@ export const addressHostToUrl = ( if (host.hostname.kind === "domain") { hostname = `${host.hostname.subdomain ? `${host.hostname.subdomain}.` : ""}${host.hostname.domain}` } else if (host.hostname.kind === "ipv6") { - hostname = `[${host.hostname.value}]` + hostname = host.hostname.value.startsWith("fe80::") + ? `[${host.hostname.value}%${host.hostname.scopeId}]` + : `[${host.hostname.value}]` } else { hostname = host.hostname.value } diff --git a/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts b/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts index 0b31dce4b..d30d65c89 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts +++ b/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts @@ -124,39 +124,47 @@ function getAddresses( const hostnames = host.kind === 'multi' ? host.hostnameInfo[addressInfo.internalPort] : [] - const addressesWithNames = hostnames.flatMap(h => { - let name = '' - - if (h.kind === 'onion') { - name = `Tor` - } else { - const hostnameKind = h.hostname.kind + const addressesWithNames = hostnames + .filter( + h => + window.location.host === 'localhost' || + h.kind !== 'ip' || + h.hostname.kind !== 'ipv6' || + !h.hostname.value.startsWith('fe80::'), + ) + .flatMap(h => { + let name = '' + + if (h.kind === 'onion') { + name = `Tor` + } else { + const hostnameKind = h.hostname.kind + + if (hostnameKind === 'domain') { + name = 'Domain' + } else { + name = + hostnameKind === 'local' + ? 'Local' + : `${h.networkInterfaceId} (${hostnameKind})` + } + } - if (hostnameKind === 'domain') { - name = 'Domain' + const addresses = utils.addressHostToUrl(addressInfo, h) + if (addresses.length > 1) { + return utils.addressHostToUrl(addressInfo, h).map(url => ({ + name: `${name} (${new URL(url).protocol + .replace(':', '') + .toUpperCase()})`, + url, + })) } else { - name = - hostnameKind === 'local' - ? 'Local' - : `${h.networkInterfaceId} (${hostnameKind})` + return utils.addressHostToUrl(addressInfo, h).map(url => ({ + name, + url, + })) } - } - - const addresses = utils.addressHostToUrl(addressInfo, h) - if (addresses.length > 1) { - return utils.addressHostToUrl(addressInfo, h).map(url => ({ - name: `${name} (${new URL(url).protocol - .replace(':', '') - .toUpperCase()})`, - url, - })) - } else { - return utils.addressHostToUrl(addressInfo, h).map(url => ({ - name, - url, - })) - } - }) + }) return addressesWithNames.filter( (value, index, self) => index === self.findIndex(t => t.url === value.url), diff --git a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html index 855e757ef..c7bdf6a2a 100644 --- a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html +++ b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.html @@ -59,17 +59,25 @@

LAN

 - + - - -

{{ iface.key }} ({{ ipAddr.includes("::") ? "IPv6" : "IPv4" }})

-

{{ ipAddr.includes("::") ? "[" + ipAddr + "%" + iface.value.ipInfo.scopeId + "]" : ipAddr }}

- - - - - + + + +

{{ iface.key }} ({{ ipAddr.includes("::") ? "IPv6" : "IPv4" }})

+

{{ + ipAddr.includes("fe80::") + ? "[" + ipAddr + "%" + iface.value.ipInfo.scopeId + "]" + : ipAddr.includes("::") + ? "[" + ipAddr + "]" + : ipAddr + }}

+ + + + + + diff --git a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.ts b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.ts index 0055ff6c1..6c7b41399 100644 --- a/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.ts +++ b/web/projects/ui/src/app/pages/server-routes/server-specs/server-specs.page.ts @@ -14,6 +14,7 @@ import { DataModel } from 'src/app/services/patch-db/data-model' }) export class ServerSpecsPage { readonly server$ = this.patch.watch$('serverInfo') + readonly isLocalhost = window.location.host === 'localhost' constructor( private readonly toastCtrl: ToastController, diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts index 589a562db..a91976493 100644 --- a/web/projects/ui/src/app/services/api/api.fixures.ts +++ b/web/projects/ui/src/app/services/api/api.fixures.ts @@ -1827,7 +1827,8 @@ export module Mock { public: false, hostname: { kind: 'ipv6', - value: '[FE80:CD00:0000:0CDE:1257:0000:211E:729CD]', + value: '[fe80:cd00:0000:0cde:1257:0000:211e:72cd]', + scopeId: 2, port: null, sslPort: 1234, }, @@ -1838,7 +1839,8 @@ export module Mock { public: false, hostname: { kind: 'ipv6', - value: '[FE80:CD00:0000:0CDE:1257:0000:211E:1234]', + value: '[fe80:cd00:0000:0cde:1257:0000:211e:1234]', + scopeId: 3, port: null, sslPort: 1234, }, diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index 6a26955aa..02ed582f1 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -255,7 +255,8 @@ export const mockPatchData: DataModel = { public: false, hostname: { kind: 'ipv6', - value: '[FE80:CD00:0000:0CDE:1257:0000:211E:729CD]', + value: '[fe80:cd00:0000:0cde:1257:0000:211e:72cd]', + scopeId: 2, port: null, sslPort: 1234, }, @@ -266,7 +267,8 @@ export const mockPatchData: DataModel = { public: false, hostname: { kind: 'ipv6', - value: '[FE80:CD00:0000:0CDE:1257:0000:211E:1234]', + value: '[fe80:cd00:0000:0cde:1257:0000:211e:1234]', + scopeId: 3, port: null, sslPort: 1234, }, From b249269286418be226572eb8f159c1234aac19b6 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Fri, 3 Jan 2025 17:19:11 -0700 Subject: [PATCH 25/29] misc fixes --- Makefile | 3 +- core/startos/src/action.rs | 1 - core/startos/src/backup/restore.rs | 1 - core/startos/src/context/rpc.rs | 2 +- core/startos/src/diagnostic.rs | 1 - core/startos/src/init.rs | 5 +- core/startos/src/net/net_controller.rs | 1 - core/startos/src/net/network_interface.rs | 123 ++++++++++------- core/startos/src/net/web_server.rs | 62 +++++---- core/startos/src/notifications.rs | 2 +- core/startos/src/registry/mod.rs | 2 +- core/startos/src/s9pk/v2/manifest.rs | 1 - core/startos/src/service/effects/net/host.rs | 1 - core/startos/src/service/transition/backup.rs | 3 +- .../startos/src/service/transition/restore.rs | 3 +- core/startos/src/update/mod.rs | 2 +- core/startos/src/util/future.rs | 4 +- core/startos/src/util/io.rs | 1 - core/startos/src/util/rpc.rs | 1 - core/startos/src/util/sync.rs | 125 ------------------ 20 files changed, 123 insertions(+), 221 deletions(-) diff --git a/Makefile b/Makefile index e0f89f3f9..a00a5c1c9 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ GZIP_BIN := $(shell which pigz || which gzip) TAR_BIN := $(shell which gtar || which tar) COMPILED_TARGETS := core/target/$(ARCH)-unknown-linux-musl/release/startbox core/target/$(ARCH)-unknown-linux-musl/release/containerbox system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar container-runtime/rootfs.$(ARCH).squashfs ALL_TARGETS := $(STARTD_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE) $(COMPILED_TARGETS) cargo-deps/$(ARCH)-unknown-linux-musl/release/startos-backup-fs $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-musl/release/pi-beep; fi) $(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]; then echo cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console; fi') $(PLATFORM_FILE) +REBUILD_TYPES = 1 ifeq ($(REMOTE),) mkdir = mkdir -p $1 @@ -226,7 +227,7 @@ container-runtime/node_modules/.package-lock.json: container-runtime/package.jso npm --prefix container-runtime ci touch container-runtime/node_modules/.package-lock.json -sdk/base/lib/osBindings/index.ts: core/startos/bindings/index.ts +sdk/base/lib/osBindings/index.ts: $(shell if [ "$(REBUILD_TYPES)" -ne 0 ]; then echo core/startos/bindings/index.ts; fi) mkdir -p sdk/base/lib/osBindings rsync -ac --delete core/startos/bindings/ sdk/base/lib/osBindings/ touch sdk/base/lib/osBindings/index.ts diff --git a/core/startos/src/action.rs b/core/startos/src/action.rs index 801360a44..b29768d73 100644 --- a/core/startos/src/action.rs +++ b/core/startos/src/action.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeMap; use std::fmt; use clap::{CommandFactory, FromArgMatches, Parser}; diff --git a/core/startos/src/backup/restore.rs b/core/startos/src/backup/restore.rs index b22d9027d..e2e9e2158 100644 --- a/core/startos/src/backup/restore.rs +++ b/core/startos/src/backup/restore.rs @@ -19,7 +19,6 @@ use crate::disk::mount::backup::BackupMountGuard; use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; use crate::init::{init, InitResult}; -use crate::net::web_server::WebServer; use crate::prelude::*; use crate::s9pk::S9pk; use crate::service::service_map::DownloadInstallFuture; diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index 12de59e05..87245b4fa 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -31,7 +31,7 @@ use crate::init::check_time_is_synchronized; use crate::lxc::{ContainerId, LxcContainer, LxcManager}; use crate::net::net_controller::{NetController, PreInitNetController}; use crate::net::utils::{find_eth_iface, find_wifi_iface}; -use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter}; +use crate::net::web_server::{UpgradableListener, WebServerAcceptorSetter}; use crate::net::wifi::WpaCli; use crate::prelude::*; use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle}; diff --git a/core/startos/src/diagnostic.rs b/core/startos/src/diagnostic.rs index f0c142706..71f76c379 100644 --- a/core/startos/src/diagnostic.rs +++ b/core/startos/src/diagnostic.rs @@ -1,4 +1,3 @@ -use std::path::Path; use std::sync::Arc; use rpc_toolkit::yajrc::RpcError; diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index 92715683b..3652336dc 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -7,7 +7,6 @@ use std::time::{Duration, SystemTime}; use axum::extract::ws::{self}; use color_eyre::eyre::eyre; -use futures::future::Either; use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; use models::ResultExt; @@ -26,7 +25,7 @@ use crate::db::model::Database; use crate::disk::mount::util::unmount; use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH; use crate::net::net_controller::PreInitNetController; -use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter}; +use crate::net::web_server::{UpgradableListener, WebServerAcceptorSetter}; use crate::prelude::*; use crate::progress::{ FullProgress, FullProgressTracker, PhaseProgressTrackerHandle, PhasedProgressBar, @@ -359,7 +358,7 @@ pub async fn init( account.tor_key, ) .await?; - webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a)); + webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?; start_net.complete(); mount_logs.start(); diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index cdf2607b7..75c37eb4c 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -6,7 +6,6 @@ use color_eyre::eyre::eyre; use imbl::OrdMap; use imbl_value::InternedString; use ipnet::IpNet; -use itertools::Itertools; use models::{HostId, OptionExt, PackageId}; use torut::onion::{OnionAddressV3, TorSecretKeyV3}; use tracing::instrument; diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index e732c1ae7..b964af273 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize}; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::net::{TcpListener, TcpStream}; use tokio::process::Command; +use tokio::sync::watch; use ts_rs::TS; use zbus::proxy::{PropertyChanged, PropertyStream, SignalStream}; use zbus::zvariant::{ @@ -34,7 +35,7 @@ use crate::prelude::*; use crate::util::future::Until; use crate::util::io::open_file; use crate::util::serde::{display_serializable, HandlerExtSerde}; -use crate::util::sync::{SyncMutex, Watch}; +use crate::util::sync::SyncMutex; use crate::util::Invoke; pub fn network_interface_api() -> ParentHandler { @@ -108,7 +109,7 @@ pub fn network_interface_api() -> ParentHandler { async fn list_interfaces( ctx: RpcContext, ) -> Result, Error> { - Ok(ctx.net_controller.net_iface.ip_info.read()) + Ok(ctx.net_controller.net_iface.ip_info.borrow().clone()) } #[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)] @@ -167,13 +168,16 @@ async fn forget_iface( )] trait NetworkManager { #[zbus(property)] - fn devices(&self) -> Result, Error>; + fn all_devices(&self) -> Result, Error>; #[zbus(signal)] fn device_added(&self) -> Result<(), Error>; #[zbus(signal)] fn device_removed(&self) -> Result<(), Error>; + + #[zbus(signal)] + fn state_changed(&self) -> Result<(), Error>; } mod active_connection { @@ -256,31 +260,38 @@ impl TryFrom for Dhcp4Options { } } -#[proxy( - interface = "org.freedesktop.NetworkManager.Device", - default_service = "org.freedesktop.NetworkManager" -)] -trait Device { - #[zbus(property)] - fn ip_interface(&self) -> Result; +mod device { + use zbus::proxy; + use zbus::zvariant::OwnedObjectPath; - #[zbus(property)] - fn managed(&self) -> Result; + use crate::prelude::*; - #[zbus(property)] - fn active_connection(&self) -> Result; + #[proxy( + interface = "org.freedesktop.NetworkManager.Device", + default_service = "org.freedesktop.NetworkManager" + )] + pub trait Device { + #[zbus(property)] + fn ip_interface(&self) -> Result; - #[zbus(property)] - fn ip4_config(&self) -> Result; + #[zbus(property)] + fn managed(&self) -> Result; - #[zbus(property)] - fn ip6_config(&self) -> Result; + #[zbus(property)] + fn active_connection(&self) -> Result; - #[zbus(property, name = "State")] - fn _state(&self) -> Result; + #[zbus(property)] + fn ip4_config(&self) -> Result; - #[zbus(signal)] - fn state_changed(&self) -> Result<(), Error>; + #[zbus(property)] + fn ip6_config(&self) -> Result; + + #[zbus(property, name = "State")] + fn _state(&self) -> Result; + + #[zbus(signal)] + fn state_changed(&self) -> Result<(), Error>; + } } trait StubStream<'a> { @@ -305,7 +316,7 @@ impl<'a> StubStream<'a> for SignalStream<'a> { } #[instrument(skip_all)] -async fn watcher(write_to: Watch>) { +async fn watcher(write_to: watch::Sender>) { loop { let res: Result<(), Error> = async { let connection = Connection::system().await?; @@ -313,7 +324,7 @@ async fn watcher(write_to: Watch> let netman_proxy = NetworkManagerProxy::new(&connection).await?; let mut until = Until::new() - .with_stream(netman_proxy.receive_devices_changed().await.stub()) + .with_stream(netman_proxy.receive_all_devices_changed().await.stub()) .with_stream( netman_proxy .receive_device_added() @@ -327,17 +338,24 @@ async fn watcher(write_to: Watch> .await? .into_inner() .stub(), + ) + .with_stream( + netman_proxy + .receive_state_changed() + .await? + .into_inner() + .stub(), ); loop { until .run(async { - let devices = netman_proxy.devices().await?; + let devices = netman_proxy.all_devices().await?; let mut ifaces = BTreeSet::new(); let mut jobs = Vec::new(); for device in devices { let device_proxy = - DeviceProxy::new(&connection, device.clone()).await?; + device::DeviceProxy::new(&connection, device.clone()).await?; let iface = InternedString::intern(device_proxy.ip_interface().await?); if iface.is_empty() { continue; @@ -399,9 +417,9 @@ async fn get_wan_ipv4(iface: &str) -> Result, Error> { #[instrument(skip(connection, device_proxy, write_to))] async fn watch_ip( connection: &Connection, - device_proxy: DeviceProxy<'_>, + device_proxy: device::DeviceProxy<'_>, iface: InternedString, - write_to: &Watch>, + write_to: &watch::Sender>, ) -> Result<(), Error> { let mut until = Until::new() .with_stream( @@ -554,7 +572,7 @@ async fn watch_ip( pub struct NetworkInterfaceController { db: TypedPatchDb, - ip_info: Watch>, + ip_info: watch::Sender>, _watcher: NonDetachingJoinHandle<()>, listeners: SyncMutex>>, } @@ -624,7 +642,7 @@ impl NetworkInterfaceController { Ok(()) } pub fn new(db: TypedPatchDb) -> Self { - let mut ip_info = Watch::new(BTreeMap::new()); + let (ip_info, mut recv) = watch::channel(BTreeMap::new()); Self { db: db.clone(), ip_info: ip_info.clone(), @@ -652,7 +670,7 @@ impl NetworkInterfaceController { let res: Result<(), Error> = async { loop { if let Err(e) = async { - let ip_info = ip_info.read(); + let ip_info = { recv.borrow().clone() }; Self::sync(&db, &ip_info).boxed().await?; Ok::<_, Error>(()) @@ -663,7 +681,7 @@ impl NetworkInterfaceController { tracing::debug!("{e:?}"); } - ip_info.changed().await; + let _ = recv.changed().await; } } .await; @@ -692,7 +710,8 @@ impl NetworkInterfaceController { })?; Ok(NetworkInterfaceListener { _arc: arc, - ip_info: self.ip_info.clone(), + ip_info: self.ip_info.subscribe(), + changed: None, listeners: ListenerMap::new(port), }) } @@ -716,7 +735,8 @@ impl NetworkInterfaceController { })?; Ok(NetworkInterfaceListener { _arc: arc, - ip_info: self.ip_info.clone(), + ip_info: self.ip_info.subscribe(), + changed: None, listeners, }) } @@ -935,8 +955,9 @@ impl ListenerMap { } pub struct NetworkInterfaceListener { - ip_info: Watch>, + ip_info: watch::Receiver>, listeners: ListenerMap, + changed: Option + Send + Sync + 'static>>>, _arc: Arc<()>, } impl NetworkInterfaceListener { @@ -944,29 +965,35 @@ impl NetworkInterfaceListener { self.listeners.port } + fn poll_ip_info_changed(&mut self, cx: &mut std::task::Context<'_>) -> Poll<()> { + let mut changed = if let Some(changed) = self.changed.take() { + changed + } else { + let mut ip_info = self.ip_info.clone(); + Box::pin(async move { + let _ = ip_info.changed().await; + }) + }; + let res = changed.poll_unpin(cx); + if res.is_pending() { + self.changed = Some(changed); + } + res + } + pub fn poll_accept( &mut self, cx: &mut std::task::Context<'_>, public: bool, ) -> Poll> { - if self.ip_info.poll_changed(cx).is_ready() || public != self.listeners.prev_public { - self.ip_info - .peek(|ip_info| self.listeners.update(ip_info, public))?; + if self.poll_ip_info_changed(cx).is_ready() || public != self.listeners.prev_public { + self.listeners.update(&*self.ip_info.borrow(), public)?; } self.listeners.poll_accept(cx) } pub async fn accept(&mut self, public: bool) -> Result { - #[pin_project::pin_project] - struct Accept<'a>(&'a mut NetworkInterfaceListener, bool); - impl<'a> Future for Accept<'a> { - type Output = Result; - fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { - let this = self.project(); - this.0.poll_accept(cx, *this.1) - } - } - Accept(self, public).await + futures::future::poll_fn(|cx| self.poll_accept(cx, public)).await } } diff --git a/core/startos/src/net/web_server.rs b/core/startos/src/net/web_server.rs index 113273b04..b38a7ee56 100644 --- a/core/startos/src/net/web_server.rs +++ b/core/startos/src/net/web_server.rs @@ -1,20 +1,19 @@ use std::future::Future; use std::net::SocketAddr; use std::ops::Deref; -use std::pin::Pin; use std::sync::atomic::AtomicBool; use std::sync::{Arc, RwLock}; use std::task::Poll; use std::time::Duration; use axum::Router; -use futures::future::Either; +use futures::future::{BoxFuture, Either}; use futures::FutureExt; use helpers::NonDetachingJoinHandle; use hyper_util::rt::{TokioIo, TokioTimer}; use hyper_util::service::TowerToHyperService; use tokio::net::{TcpListener, TcpStream}; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, watch}; use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::net::network_interface::NetworkInterfaceListener; @@ -24,7 +23,6 @@ use crate::net::static_server::{ }; use crate::prelude::*; use crate::util::actor::background::BackgroundJobQueue; -use crate::util::sync::Watch; pub struct Accepted { pub https_redirect: bool, @@ -78,19 +76,38 @@ impl Accept for Option { #[pin_project::pin_project] pub struct Acceptor { - acceptor: Watch, + acceptor: (watch::Sender, watch::Receiver), + changed: Option>, } -impl Acceptor { +impl Acceptor { pub fn new(acceptor: A) -> Self { Self { - acceptor: Watch::new(acceptor), + acceptor: watch::channel(acceptor), + changed: None, } } + fn poll_changed(&mut self, cx: &mut std::task::Context<'_>) -> Poll<()> { + let mut changed = if let Some(changed) = self.changed.take() { + changed + } else { + let mut recv = self.acceptor.1.clone(); + async move { + let _ = recv.changed().await; + } + .boxed() + }; + let res = changed.poll_unpin(cx); + if res.is_pending() { + self.changed = Some(changed); + } + res + } + fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - let _ = self.acceptor.poll_changed(cx); + let _ = self.poll_changed(cx); let mut res = Poll::Pending; - self.acceptor.send_if_modified(|a| { + self.acceptor.0.send_if_modified(|a| { res = a.poll_accept(cx); false }); @@ -98,15 +115,7 @@ impl Acceptor { } async fn accept(&mut self) -> Result { - #[pin_project::pin_project] - struct AcceptFut<'a, A: Accept>(&'a mut Acceptor); - impl<'a, A: Accept> Future for AcceptFut<'a, A> { - type Output = Result; - fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { - self.project().0.poll_accept(cx) - } - } - AcceptFut(self).await + std::future::poll_fn(|cx| self.poll_accept(cx)).await } } impl Acceptor> { @@ -130,7 +139,7 @@ impl Acceptor { } pub struct WebServerAcceptorSetter { - acceptor: Watch, + acceptor: watch::Sender, } impl WebServerAcceptorSetter>> { pub fn try_upgrade Result>(&self, f: F) -> Result<(), Error> { @@ -151,7 +160,7 @@ impl WebServerAcceptorSetter>> { } } impl Deref for WebServerAcceptorSetter { - type Target = Watch; + type Target = watch::Sender; fn deref(&self) -> &Self::Target { &self.acceptor } @@ -159,11 +168,11 @@ impl Deref for WebServerAcceptorSetter { pub struct WebServer { shutdown: oneshot::Sender<()>, - router: Watch>, - acceptor: Watch, + router: watch::Sender>, + acceptor: watch::Sender, thread: NonDetachingJoinHandle<()>, } -impl WebServer { +impl WebServer { pub fn acceptor_setter(&self) -> WebServerAcceptorSetter { WebServerAcceptorSetter { acceptor: self.acceptor.clone(), @@ -171,9 +180,8 @@ impl WebServer { } pub fn new(mut acceptor: Acceptor) -> Self { - let acceptor_send = acceptor.acceptor.clone(); - let router = Watch::>::new(None); - let service = router.clone(); + let acceptor_send = acceptor.acceptor.0.clone(); + let (router, service) = watch::channel::>(None); let (shutdown, shutdown_recv) = oneshot::channel(); let thread = NonDetachingJoinHandle::from(tokio::spawn(async move { #[derive(Clone)] @@ -228,7 +236,7 @@ impl WebServer { ), ); } else { - let service = service.read(); + let service = { service.borrow().clone() }; if let Some(service) = service { queue.add_job( graceful.watch( diff --git a/core/startos/src/notifications.rs b/core/startos/src/notifications.rs index 4b45531a4..3ac09de0f 100644 --- a/core/startos/src/notifications.rs +++ b/core/startos/src/notifications.rs @@ -13,11 +13,11 @@ use serde::{Deserialize, Serialize}; use tracing::instrument; use ts_rs::TS; +use crate::backup::BackupReport; use crate::context::{CliContext, RpcContext}; use crate::db::model::DatabaseModel; use crate::prelude::*; use crate::util::serde::HandlerExtSerde; -use crate::{backup::BackupReport, db::model::Database}; // #[command(subcommands(list, delete, delete_before, create))] pub fn notification() -> ParentHandler { diff --git a/core/startos/src/registry/mod.rs b/core/startos/src/registry/mod.rs index b56cedce6..4e1411ea9 100644 --- a/core/startos/src/registry/mod.rs +++ b/core/startos/src/registry/mod.rs @@ -143,7 +143,7 @@ pub fn registry_router(ctx: RegistryContext) -> Router { ) } -impl WebServer { +impl WebServer { pub fn serve_registry(&mut self, ctx: RegistryContext) { self.serve_router(registry_router(ctx)) } diff --git a/core/startos/src/s9pk/v2/manifest.rs b/core/startos/src/s9pk/v2/manifest.rs index 187b2dede..11ea0d9af 100644 --- a/core/startos/src/s9pk/v2/manifest.rs +++ b/core/startos/src/s9pk/v2/manifest.rs @@ -3,7 +3,6 @@ use std::path::Path; use color_eyre::eyre::eyre; use exver::{Version, VersionRange}; -use helpers::const_true; use imbl_value::InternedString; pub use models::PackageId; use models::{mime, ImageId, VolumeId}; diff --git a/core/startos/src/service/effects/net/host.rs b/core/startos/src/service/effects/net/host.rs index 51f9eceec..570d5033d 100644 --- a/core/startos/src/service/effects/net/host.rs +++ b/core/startos/src/service/effects/net/host.rs @@ -1,6 +1,5 @@ use models::{HostId, PackageId}; -use crate::net::host::address::HostAddress; use crate::net::host::Host; use crate::service::effects::callbacks::CallbackHandler; use crate::service::effects::prelude::*; diff --git a/core/startos/src/service/transition/backup.rs b/core/startos/src/service/transition/backup.rs index 0d4116078..6205cdd61 100644 --- a/core/startos/src/service/transition/backup.rs +++ b/core/startos/src/service/transition/backup.rs @@ -15,6 +15,7 @@ use crate::service::ServiceActor; use crate::util::actor::background::BackgroundJobQueue; use crate::util::actor::{ConflictBuilder, Handler}; use crate::util::future::RemoteCancellable; +use crate::util::serde::NoOutput; pub(in crate::service) struct Backup { pub path: PathBuf, @@ -48,7 +49,7 @@ impl Handler for ServiceActor { .mount_backup(path, ReadWrite) .await?; seed.persistent_container - .execute(id, ProcedureName::CreateBackup, Value::Null, None) + .execute::(id, ProcedureName::CreateBackup, Value::Null, None) .await?; backup_guard.unmount(true).await?; diff --git a/core/startos/src/service/transition/restore.rs b/core/startos/src/service/transition/restore.rs index 08f3be942..7061b0c1e 100644 --- a/core/startos/src/service/transition/restore.rs +++ b/core/startos/src/service/transition/restore.rs @@ -11,6 +11,7 @@ use crate::service::ServiceActor; use crate::util::actor::background::BackgroundJobQueue; use crate::util::actor::{ConflictBuilder, Handler}; use crate::util::future::RemoteCancellable; +use crate::util::serde::NoOutput; pub(in crate::service) struct Restore { pub path: PathBuf, @@ -38,7 +39,7 @@ impl Handler for ServiceActor { .mount_backup(path, ReadOnly) .await?; seed.persistent_container - .execute(id, ProcedureName::RestoreBackup, Value::Null, None) + .execute::(id, ProcedureName::RestoreBackup, Value::Null, None) .await?; backup_guard.unmount(true).await?; diff --git a/core/startos/src/update/mod.rs b/core/startos/src/update/mod.rs index 51d8d77ae..d88838d4a 100644 --- a/core/startos/src/update/mod.rs +++ b/core/startos/src/update/mod.rs @@ -20,7 +20,7 @@ use ts_rs::TS; use crate::context::{CliContext, RpcContext}; use crate::disk::mount::filesystem::bind::Bind; use crate::disk::mount::filesystem::block_dev::BlockDev; -use crate::disk::mount::filesystem::efivarfs::{self, EfiVarFs}; +use crate::disk::mount::filesystem::efivarfs::{ EfiVarFs}; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; use crate::disk::mount::filesystem::MountType; use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard}; diff --git a/core/startos/src/util/future.rs b/core/startos/src/util/future.rs index 2ef053fed..c690f9754 100644 --- a/core/startos/src/util/future.rs +++ b/core/startos/src/util/future.rs @@ -1,6 +1,5 @@ use std::pin::Pin; -use std::sync::atomic::AtomicUsize; -use std::task::{Context, Poll, Waker}; +use std::task::{Context, Poll}; use futures::future::{abortable, pending, BoxFuture, FusedFuture}; use futures::stream::{AbortHandle, Abortable, BoxStream}; @@ -8,7 +7,6 @@ use futures::{Future, FutureExt, Stream, StreamExt}; use tokio::sync::watch; use crate::prelude::*; -use crate::util::sync::SyncMutex; #[pin_project::pin_project(PinnedDrop)] pub struct DropSignaling { diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs index 9018b3344..f0bae7a0a 100644 --- a/core/startos/src/util/io.rs +++ b/core/startos/src/util/io.rs @@ -541,7 +541,6 @@ impl std::io::Read for BackTrackingIO { } BTBuffer::NotBuffering => self.io.read(buf), BTBuffer::Rewound { read } => { - let mut ready = false; if (read.position() as usize) < read.get_ref().len() { let n = std::io::Read::read(read, buf)?; if n != 0 { diff --git a/core/startos/src/util/rpc.rs b/core/startos/src/util/rpc.rs index b2dea340e..f7c91eb82 100644 --- a/core/startos/src/util/rpc.rs +++ b/core/startos/src/util/rpc.rs @@ -3,7 +3,6 @@ use std::path::Path; use clap::Parser; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; -use url::Url; use crate::context::CliContext; use crate::prelude::*; diff --git a/core/startos/src/util/sync.rs b/core/startos/src/util/sync.rs index 2bf1e767d..2630858a9 100644 --- a/core/startos/src/util/sync.rs +++ b/core/startos/src/util/sync.rs @@ -1,9 +1,3 @@ -use std::future::Future; -use std::pin::Pin; -use std::sync::atomic::AtomicUsize; -use std::sync::Arc; -use std::task::{Poll, Waker}; - #[derive(Debug, Default)] pub struct SyncMutex(std::sync::Mutex); impl SyncMutex { @@ -17,122 +11,3 @@ impl SyncMutex { f(&*self.0.lock().unwrap()) } } - -struct WatchData { - data: T, - wakers: Vec, -} - -struct Shared { - data: SyncMutex>, - version: AtomicUsize, -} - -pub struct Watch { - data: Arc>, - seen_version: usize, -} -impl Clone for Watch { - fn clone(&self) -> Self { - Self { - data: self.data.clone(), - seen_version: self.seen_version, - } - } -} -impl Watch { - pub fn new(init: T) -> Self { - Self { - data: Arc::new(Shared { - data: SyncMutex::new(WatchData { - data: init, - wakers: Vec::new(), - }), - version: AtomicUsize::new(1), - }), - seen_version: 1, - } - } - - pub fn poll_changed(&mut self, cx: &mut std::task::Context<'_>) -> Poll<()> { - let mut guard = self.data.data.0.lock().unwrap(); - let version = self.data.version.load(std::sync::atomic::Ordering::SeqCst); - if version > self.seen_version { - self.seen_version = version; - return Poll::Ready(()); - } - let waker = cx.waker(); - if !guard.wakers.iter().any(|w| w.will_wake(waker)) { - guard.wakers.push(waker.clone()); - } - Poll::Pending - } - - pub async fn changed(&mut self) { - #[pin_project::pin_project] - struct Changed<'a, T>(&'a mut Watch); - impl<'a, T> Future for Changed<'a, T> { - type Output = (); - fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { - let this = self.project(); - this.0.poll_changed(cx) - } - } - Changed(self).await - } - - pub fn mark_changed(&mut self) { - self.seen_version = 0; - } - - pub fn peek(&self, peek: F) -> U - where - F: FnOnce(&T) -> U, - { - self.data.data.peek(|d| peek(&d.data)) - } - - pub fn send_if_modified(&self, modify: F) -> bool - where - F: FnOnce(&mut T) -> bool, - { - let mut guard = self.data.data.0.lock().unwrap(); - let changed = modify(&mut guard.data); - if changed { - self.data - .version - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - for waker in guard.wakers.drain(..) { - waker.wake(); - } - } - changed - } - - pub fn send_modify(&self, modify: F) - where - F: FnOnce(&mut T), - { - self.send_if_modified(|x| { - modify(x); - true - }); - } - - pub fn send_replace(&self, mut value: T) -> T { - self.send_modify(|x| { - std::mem::swap(x, &mut value); - }); - value - } - - pub fn send(&self, value: T) { - self.send_replace(value); - } -} - -impl Watch { - pub fn read(&self) -> T { - self.data.data.0.lock().unwrap().data.clone() - } -} From 181d9f12d2416d1b63061faefe32f2bb2f00c5f7 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 7 Jan 2025 10:55:58 -0700 Subject: [PATCH 26/29] refactor port forwarding --- CLEARNET.md | 40 -- core/startos/src/net/forward.rs | 371 ++++++++++++------ core/startos/src/net/net_controller.rs | 26 +- core/startos/src/net/network_interface.rs | 4 + .../app-interfaces/app-interfaces.page.ts | 91 +++-- 5 files changed, 321 insertions(+), 211 deletions(-) delete mode 100644 CLEARNET.md diff --git a/CLEARNET.md b/CLEARNET.md deleted file mode 100644 index 457a2e4f7..000000000 --- a/CLEARNET.md +++ /dev/null @@ -1,40 +0,0 @@ -# Setting up clearnet for a service interface - -NOTE: this guide is for HTTPS only! Other configurations may require a more bespoke setup depending on the service. Please consult the service documentation or the Start9 Community for help with non-HTTPS applications - -## Initialize ACME certificate generation - -The following command will register your device with an ACME certificate provider, such as letsencrypt - -This only needs to be done once. - -``` -start-cli net acme init --provider=letsencrypt --contact="mailto:me@drbonez.dev" -``` - -- `provider` can be `letsencrypt`, `letsencrypt-staging` (useful if you're doing a lot of testing and want to avoid being rate limited), or the url of any provider that supports the [RFC8555](https://datatracker.ietf.org/doc/html/rfc8555) ACME api -- `contact` can be any valid contact url, typically `mailto:` urls. it can be specified multiple times to set multiple contacts - -## Whitelist a domain for ACME certificate acquisition - -The following command will tell the OS to use ACME certificates instead of system signed ones for the provided url. In this example, `testing.drbonez.dev` - -This must be done for every domain you wish to host on clearnet. - -``` -start-cli net acme domain add "testing.drbonez.dev" -``` - -## Forward clearnet port - -Go into your router settings, and map port 443 on your router to port 5443 on your start-os device. This one port should cover most use cases - -## Add domain to service host - -The following command will tell the OS to route https requests from the WAN to the provided hostname to the specified service. In this example, we are adding `testing.drbonez.dev` to the host `ui-multi` on the package `hello-world`. To see a list of available host IDs for a given package, run `start-cli package host list` - -This must be done for every domain you wish to host on clearnet. - -``` -start-cli package host hello-world address ui-multi add testing.drbonez.dev -``` diff --git a/core/startos/src/net/forward.rs b/core/startos/src/net/forward.rs index e954bc36a..ba62945a0 100644 --- a/core/startos/src/net/forward.rs +++ b/core/startos/src/net/forward.rs @@ -1,12 +1,16 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::net::SocketAddr; use std::sync::{Arc, Weak}; +use futures::channel::oneshot; +use helpers::NonDetachingJoinHandle; use id_pool::IdPool; +use imbl_value::InternedString; use serde::{Deserialize, Serialize}; use tokio::process::Command; -use tokio::sync::Mutex; +use tokio::sync::{mpsc, watch}; +use crate::db::model::public::NetworkInterfaceInfo; use crate::prelude::*; use crate::util::Invoke; @@ -34,144 +38,269 @@ impl AvailablePorts { } } +#[derive(Debug)] +struct ForwardRequest { + public: bool, + target: SocketAddr, + rc: Weak<()>, +} + +#[derive(Debug, Default)] +struct ForwardState { + requested: BTreeMap, + current: BTreeMap>, +} +impl ForwardState { + async fn sync(&mut self, interfaces: &BTreeMap) -> Result<(), Error> { + let private_interfaces = interfaces + .iter() + .filter(|(_, public)| !*public) + .map(|(i, _)| i) + .collect::>(); + let all_interfaces = interfaces.keys().collect::>(); + self.requested.retain(|_, req| req.rc.strong_count() > 0); + for external in self + .requested + .keys() + .chain(self.current.keys()) + .copied() + .collect::>() + { + match ( + self.requested.get(&external), + self.current.get_mut(&external), + ) { + (Some(req), Some(cur)) => { + let expected = if req.public { + &all_interfaces + } else { + &private_interfaces + }; + let actual = cur.keys().collect::>(); + let mut to_rm = actual + .difference(expected) + .copied() + .cloned() + .collect::>(); + let mut to_add = expected + .difference(&actual) + .copied() + .cloned() + .collect::>(); + for interface in actual.intersection(expected).copied() { + if cur[interface] != req.target { + to_rm.insert(interface.clone()); + to_add.insert(interface.clone()); + } + } + for interface in to_rm { + unforward(external, &*interface, cur[&interface]).await?; + cur.remove(&interface); + } + for interface in to_add { + forward(external, &*interface, req.target).await?; + cur.insert(interface, req.target); + } + } + (Some(req), None) => { + let cur = self.current.entry(external).or_default(); + for interface in if req.public { + &all_interfaces + } else { + &private_interfaces + } + .into_iter() + .copied() + .cloned() + { + forward(external, &*interface, req.target).await?; + cur.insert(interface, req.target); + } + } + (None, Some(cur)) => { + let to_rm = cur.keys().cloned().collect::>(); + for interface in to_rm { + unforward(external, &*interface, cur[&interface]).await?; + cur.remove(&interface); + } + self.current.remove(&external); + } + _ => (), + } + } + Ok(()) + } +} + +fn err_has_exited(_: T) -> Error { + Error::new( + eyre!("PortForwardController thread has exited"), + ErrorKind::Unknown, + ) +} + pub struct LanPortForwardController { - forwards: Mutex>>>, + req: mpsc::UnboundedSender<( + Option<(u16, ForwardRequest)>, + oneshot::Sender>, + )>, + _thread: NonDetachingJoinHandle<()>, } impl LanPortForwardController { - pub fn new() -> Self { + pub fn new( + mut net_iface: watch::Receiver>, + ) -> Self { + let (req_send, mut req_recv) = mpsc::unbounded_channel(); + let thread = NonDetachingJoinHandle::from(tokio::spawn(async move { + let mut state = ForwardState::default(); + let mut interfaces = net_iface + .borrow_and_update() + .iter() + .map(|(iface, info)| (iface.clone(), info.public())) + .collect(); + let mut reply: Option>> = None; + loop { + tokio::select! { + msg = req_recv.recv() => { + if let Some((msg, re)) = msg { + if let Some((external, req)) = msg { + state.requested.insert(external, req); + } + reply = Some(re); + } else { + break; + } + } + _ = net_iface.changed() => { + interfaces = net_iface + .borrow() + .iter() + .map(|(iface, info)| (iface.clone(), info.public())) + .collect(); + } + } + let res = state.sync(&interfaces).await; + if let Err(e) = &res { + tracing::error!("Error in PortForwardController: {e}"); + tracing::debug!("{e:?}"); + } + if let Some(re) = reply.take() { + let _ = re.send(res); + } + } + })); Self { - forwards: Mutex::new(BTreeMap::new()), + req: req_send, + _thread: thread, } } - pub async fn add(&self, port: u16, addr: SocketAddr) -> Result, Error> { - let mut writable = self.forwards.lock().await; - let (prev, mut forward) = if let Some(forward) = writable.remove(&port) { - ( - forward.keys().next().cloned(), - forward - .into_iter() - .filter(|(_, rc)| rc.strong_count() > 0) - .collect(), - ) - } else { - (None, BTreeMap::new()) - }; + pub async fn add(&self, port: u16, public: bool, target: SocketAddr) -> Result, Error> { let rc = Arc::new(()); - forward.insert(addr, Arc::downgrade(&rc)); - let next = forward.keys().next().cloned(); - if !forward.is_empty() { - writable.insert(port, forward); - } - - update_forward(port, prev, next).await?; - Ok(rc) - } - pub async fn gc(&self, external: u16) -> Result<(), Error> { - let mut writable = self.forwards.lock().await; - let (prev, forward) = if let Some(forward) = writable.remove(&external) { - ( - forward.keys().next().cloned(), - forward - .into_iter() - .filter(|(_, rc)| rc.strong_count() > 0) - .collect(), - ) - } else { - (None, BTreeMap::new()) - }; - let next = forward.keys().next().cloned(); - if !forward.is_empty() { - writable.insert(external, forward); - } + let (send, recv) = oneshot::channel(); + self.req + .send(( + Some(( + port, + ForwardRequest { + public, + target, + rc: Arc::downgrade(&rc), + }, + )), + send, + )) + .map_err(err_has_exited)?; - update_forward(external, prev, next).await + recv.await.map_err(err_has_exited)?.map(|_| rc) } -} + pub async fn gc(&self) -> Result<(), Error> { + let (send, recv) = oneshot::channel(); + self.req.send((None, send)).map_err(err_has_exited)?; -async fn update_forward( - external: u16, - prev: Option, - next: Option, -) -> Result<(), Error> { - if prev != next { - if let Some(prev) = prev { - unforward(START9_BRIDGE_IFACE, external, prev).await?; - } - if let Some(next) = next { - forward(START9_BRIDGE_IFACE, external, next).await?; - } + recv.await.map_err(err_has_exited)? } - Ok(()) } // iptables -I FORWARD -o br-start9 -p tcp -d 172.18.0.2 --dport 8333 -j ACCEPT // iptables -t nat -I PREROUTING -p tcp --dport 32768 -j DNAT --to 172.18.0.2:8333 -async fn forward(iface: &str, external: u16, addr: SocketAddr) -> Result<(), Error> { - Command::new("iptables") - .arg("-I") - .arg("FORWARD") - .arg("-o") - .arg(iface) - .arg("-p") - .arg("tcp") - .arg("-d") - .arg(addr.ip().to_string()) - .arg("--dport") - .arg(addr.port().to_string()) - .arg("-j") - .arg("ACCEPT") - .invoke(crate::ErrorKind::Network) - .await?; - Command::new("iptables") - .arg("-t") - .arg("nat") - .arg("-I") - .arg("PREROUTING") - .arg("-p") - .arg("tcp") - .arg("--dport") - .arg(external.to_string()) - .arg("-j") - .arg("DNAT") - .arg("--to") - .arg(addr.to_string()) - .invoke(crate::ErrorKind::Network) - .await?; +async fn forward(external: u16, interface: &str, target: SocketAddr) -> Result<(), Error> { + for proto in ["tcp", "udp"] { + Command::new("iptables") + .arg("-I") + .arg("FORWARD") + .arg("-i") + .arg(interface) + .arg("-o") + .arg(START9_BRIDGE_IFACE) + .arg("-p") + .arg(proto) + .arg("-d") + .arg(target.ip().to_string()) + .arg("--dport") + .arg(target.port().to_string()) + .arg("-j") + .arg("ACCEPT") + .invoke(crate::ErrorKind::Network) + .await?; + Command::new("iptables") + .arg("-t") + .arg("nat") + .arg("-I") + .arg("PREROUTING") + .arg("-i") + .arg(interface) + .arg("-p") + .arg(proto) + .arg("--dport") + .arg(external.to_string()) + .arg("-j") + .arg("DNAT") + .arg("--to") + .arg(target.to_string()) + .invoke(crate::ErrorKind::Network) + .await?; + } Ok(()) } // iptables -D FORWARD -o br-start9 -p tcp -d 172.18.0.2 --dport 8333 -j ACCEPT // iptables -t nat -D PREROUTING -p tcp --dport 32768 -j DNAT --to 172.18.0.2:8333 -async fn unforward(iface: &str, external: u16, addr: SocketAddr) -> Result<(), Error> { - Command::new("iptables") - .arg("-D") - .arg("FORWARD") - .arg("-o") - .arg(iface) - .arg("-p") - .arg("tcp") - .arg("-d") - .arg(addr.ip().to_string()) - .arg("--dport") - .arg(addr.port().to_string()) - .arg("-j") - .arg("ACCEPT") - .invoke(crate::ErrorKind::Network) - .await?; - Command::new("iptables") - .arg("-t") - .arg("nat") - .arg("-D") - .arg("PREROUTING") - .arg("-p") - .arg("tcp") - .arg("--dport") - .arg(external.to_string()) - .arg("-j") - .arg("DNAT") - .arg("--to") - .arg(addr.to_string()) - .invoke(crate::ErrorKind::Network) - .await?; +async fn unforward(external: u16, interface: &str, target: SocketAddr) -> Result<(), Error> { + for proto in ["tcp", "udp"] { + Command::new("iptables") + .arg("-D") + .arg("FORWARD") + .arg("-i") + .arg(interface) + .arg("-o") + .arg(START9_BRIDGE_IFACE) + .arg("-p") + .arg(proto) + .arg("-d") + .arg(target.ip().to_string()) + .arg("--dport") + .arg(target.port().to_string()) + .arg("-j") + .arg("ACCEPT") + .invoke(crate::ErrorKind::Network) + .await?; + Command::new("iptables") + .arg("-t") + .arg("nat") + .arg("-D") + .arg("PREROUTING") + .arg("-i") + .arg(interface) + .arg("-p") + .arg(proto) + .arg("--dport") + .arg(external.to_string()) + .arg("-j") + .arg("DNAT") + .arg("--to") + .arg(target.to_string()) + .invoke(crate::ErrorKind::Network) + .await?; + } Ok(()) } diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 75c37eb4c..322e0dedc 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -141,9 +141,9 @@ impl NetController { db, tor, vhost, - net_iface, dns: DnsController::init(dns_bind).await?, - forward: LanPortForwardController::new(), + forward: LanPortForwardController::new(net_iface.subscribe()), + net_iface, os_bindings, server_hostnames, }; @@ -175,7 +175,7 @@ impl NetController { #[derive(Default, Debug)] struct HostBinds { - forwards: BTreeMap)>, + forwards: BTreeMap)>, vhosts: BTreeMap<(Option, u16), (TargetInfo, Arc<()>)>, tor: BTreeMap, Vec>)>, } @@ -262,7 +262,7 @@ impl NetService { pub async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> { let ctrl = self.net_controller()?; - let mut forwards: BTreeMap = BTreeMap::new(); + let mut forwards: BTreeMap = BTreeMap::new(); let mut vhosts: BTreeMap<(Option, u16), TargetInfo> = BTreeMap::new(); let mut tor: BTreeMap)> = BTreeMap::new(); @@ -372,7 +372,7 @@ impl NetService { // doesn't make sense to have 2 listening ports, both with ssl } else { let external = bind.net.assigned_port.or_not_found("assigned lan port")?; - forwards.insert(external, (self.ip, *port).into()); + forwards.insert(external, ((self.ip, *port).into(), bind.net.public)); } } let mut bind_hostname_info: Vec = @@ -554,23 +554,23 @@ impl NetService { .collect::>(); for external in all { let mut prev = binds.forwards.remove(&external); - if let Some(internal) = forwards.remove(&external) { - prev = prev.filter(|(i, _)| i == &internal); + if let Some((internal, public)) = forwards.remove(&external) { + prev = prev.filter(|(i, p, _)| i == &internal && *p == public); binds.forwards.insert( external, if let Some(prev) = prev { prev } else { - (internal, ctrl.forward.add(external, internal).await?) + ( + internal, + public, + ctrl.forward.add(external, public, internal).await?, + ) }, ); - } else { - if let Some((_, rc)) = prev { - drop(rc); - ctrl.forward.gc(external).await?; - } } } + ctrl.forward.gc().await?; let all = binds .vhosts diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index b964af273..ad8f7de4f 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -577,6 +577,10 @@ pub struct NetworkInterfaceController { listeners: SyncMutex>>, } impl NetworkInterfaceController { + pub fn subscribe(&self) -> watch::Receiver> { + self.ip_info.subscribe() + } + async fn sync( db: &TypedPatchDb, info: &BTreeMap, diff --git a/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts b/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts index d30d65c89..ca8645f18 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts +++ b/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts @@ -121,50 +121,67 @@ function getAddresses( ): MappedAddress[] { const addressInfo = serviceInterface.addressInfo - const hostnames = + let hostnames = host.kind === 'multi' ? host.hostnameInfo[addressInfo.internalPort] : [] - const addressesWithNames = hostnames - .filter( - h => - window.location.host === 'localhost' || - h.kind !== 'ip' || - h.hostname.kind !== 'ipv6' || - !h.hostname.value.startsWith('fe80::'), + hostnames = hostnames.filter( + h => + window.location.host === 'localhost' || + h.kind !== 'ip' || + h.hostname.kind !== 'ipv6' || + !h.hostname.value.startsWith('fe80::'), + ) + if (window.location.host === 'localhost') { + const local = hostnames.find( + h => h.kind === 'ip' && h.hostname.kind === 'local', ) - .flatMap(h => { - let name = '' + if (local) { + hostnames.unshift({ + kind: 'ip', + networkInterfaceId: 'lo', + public: false, + hostname: { + kind: 'local', + port: local.hostname.port, + sslPort: local.hostname.sslPort, + value: 'localhost', + }, + }) + } + } + const addressesWithNames = hostnames.flatMap(h => { + let name = '' - if (h.kind === 'onion') { - name = `Tor` - } else { - const hostnameKind = h.hostname.kind - - if (hostnameKind === 'domain') { - name = 'Domain' - } else { - name = - hostnameKind === 'local' - ? 'Local' - : `${h.networkInterfaceId} (${hostnameKind})` - } - } + if (h.kind === 'onion') { + name = `Tor` + } else { + const hostnameKind = h.hostname.kind - const addresses = utils.addressHostToUrl(addressInfo, h) - if (addresses.length > 1) { - return utils.addressHostToUrl(addressInfo, h).map(url => ({ - name: `${name} (${new URL(url).protocol - .replace(':', '') - .toUpperCase()})`, - url, - })) + if (hostnameKind === 'domain') { + name = 'Domain' } else { - return utils.addressHostToUrl(addressInfo, h).map(url => ({ - name, - url, - })) + name = + hostnameKind === 'local' + ? 'Local' + : `${h.networkInterfaceId} (${hostnameKind})` } - }) + } + + const addresses = utils.addressHostToUrl(addressInfo, h) + if (addresses.length > 1) { + return addresses.map(url => ({ + name: `${name} (${new URL(url).protocol + .replace(':', '') + .toUpperCase()})`, + url, + })) + } else { + return addresses.map(url => ({ + name, + url, + })) + } + }) return addressesWithNames.filter( (value, index, self) => index === self.findIndex(t => t.url === value.url), From d9a01e20338c527927e45fd80181b2a26ed393c4 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 7 Jan 2025 14:15:49 -0700 Subject: [PATCH 27/29] recheck interfaces every 5 min if no dbus event --- core/startos/src/net/network_interface.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index ad8f7de4f..d39a9861a 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -436,7 +436,14 @@ async fn watch_ip( .stub(), ) .with_stream(device_proxy.receive_ip4_config_changed().await.stub()) - .with_stream(device_proxy.receive_ip6_config_changed().await.stub()); + .with_stream(device_proxy.receive_ip6_config_changed().await.stub()) + .with_async_fn(|| { + async { + tokio::time::sleep(Duration::from_secs(300)).await; + Ok(()) + } + .fuse() + }); loop { until From 4b3ebdddb4eca0c722ebd37e3ed82c93362fcad7 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 9 Jan 2025 14:14:51 -0700 Subject: [PATCH 28/29] misc fixes and cleanup --- build/lib/scripts/enable-kiosk | 3 +- container-runtime/src/Adapters/RpcListener.ts | 26 +++-- .../src/Adapters/Systems/SystemForStartOs.ts | 9 +- core/Cargo.lock | 17 ++-- core/startos/Cargo.toml | 1 + core/startos/src/bins/registry.rs | 2 +- core/startos/src/bins/start_init.rs | 7 +- core/startos/src/context/config.rs | 13 +-- core/startos/src/context/diagnostic.rs | 5 +- core/startos/src/context/rpc.rs | 3 - core/startos/src/context/setup.rs | 11 +-- core/startos/src/db/model/public.rs | 11 ++- core/startos/src/diagnostic.rs | 5 +- core/startos/src/disk/mount/backup.rs | 1 - core/startos/src/disk/mount/filesystem/mod.rs | 1 - core/startos/src/init.rs | 25 ++--- core/startos/src/install/mod.rs | 3 - core/startos/src/lib.rs | 5 + core/startos/src/lxc/mod.rs | 1 - core/startos/src/net/mod.rs | 6 -- core/startos/src/net/network_interface.rs | 19 +++- core/startos/src/net/static_server.rs | 6 +- core/startos/src/os_install/gpt.rs | 2 +- .../startos/src/service/effects/dependency.rs | 4 +- core/startos/src/service/mod.rs | 15 +-- .../src/service/persistent_container.rs | 8 +- core/startos/src/service/service_actor.rs | 1 - core/startos/src/service/service_map.rs | 8 +- core/startos/src/setup.rs | 22 ++--- core/startos/src/shutdown.rs | 8 +- core/startos/src/status/mod.rs | 2 +- core/startos/src/system.rs | 9 +- core/startos/src/version/mod.rs | 4 +- core/startos/src/version/v0_3_6_alpha_0.rs | 16 +--- core/startos/src/version/v0_3_6_alpha_10.rs | 1 - core/startos/src/version/v0_3_6_alpha_6.rs | 2 +- core/startos/src/version/v0_3_6_alpha_7.rs | 2 +- core/startos/src/version/v0_3_6_alpha_8.rs | 5 +- core/startos/src/volume.rs | 7 +- debian/postinst | 12 +-- image-recipe/build.sh | 4 +- .../lib/dependencies/setupDependencies.ts | 20 ++-- sdk/base/lib/osBindings/IpInfo.ts | 2 + .../lib/osBindings/NetworkInterfaceType.ts | 3 + sdk/base/lib/osBindings/index.ts | 1 + sdk/base/lib/util/patterns.ts | 22 ++--- sdk/package/lib/StartSdk.ts | 12 +++ sdk/package/lib/mainFn/CommandController.ts | 94 ++++++++++--------- sdk/package/lib/mainFn/Daemon.ts | 2 + sdk/package/lib/mainFn/Daemons.ts | 59 +++++++----- sdk/package/lib/mainFn/HealthDaemon.ts | 13 +++ .../ui/src/app/services/api/mock-patch.ts | 2 + 52 files changed, 291 insertions(+), 251 deletions(-) create mode 100644 sdk/base/lib/osBindings/NetworkInterfaceType.ts diff --git a/build/lib/scripts/enable-kiosk b/build/lib/scripts/enable-kiosk index 40753af40..cd48fc032 100755 --- a/build/lib/scripts/enable-kiosk +++ b/build/lib/scripts/enable-kiosk @@ -83,8 +83,7 @@ user_pref("toolkit.telemetry.updatePing.enabled", false); user_pref("toolkit.telemetry.cachedClientID", ""); EOF -cp /usr/lib/firefox-esr/libnssckbi.so /usr/lib/firefox-esr/libnssckbi.so.bak -ln -sf /usr/lib/x86_64-linux-gnu/pkcs11/p11-kit-trust.so /usr/lib/firefox-esr/libnssckbi.so +ln -sf /usr/lib/$(uname -m)-linux-gnu/pkcs11/p11-kit-trust.so /usr/lib/firefox-esr/libnssckbi.so # create kiosk script cat > /home/kiosk/kiosk.sh << 'EOF' diff --git a/container-runtime/src/Adapters/RpcListener.ts b/container-runtime/src/Adapters/RpcListener.ts index 3e86e60d1..0585c2637 100644 --- a/container-runtime/src/Adapters/RpcListener.ts +++ b/container-runtime/src/Adapters/RpcListener.ts @@ -212,16 +212,22 @@ export class RpcListener { s.on("data", (a) => Promise.resolve(a) .then((b) => b.toString()) - .then(logData("dataIn")) - .then(jsonParse) - .then(captureId) - .then((x) => this.dealWithInput(x)) - .catch(mapError) - .then(logData("response")) - .then(writeDataToSocket) - .catch((e) => { - console.error(`Major error in socket handling: ${e}`) - console.debug(`Data in: ${a.toString()}`) + .then((buf) => { + for (let s in buf.split("\n")) { + if (s) + Promise.resolve(s) + .then(logData("dataIn")) + .then(jsonParse) + .then(captureId) + .then((x) => this.dealWithInput(x)) + .catch(mapError) + .then(logData("response")) + .then(writeDataToSocket) + .catch((e) => { + console.error(`Major error in socket handling: ${e}`) + console.debug(`Data in: ${a.toString()}`) + }) + } }), ) }) diff --git a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts index 334764a87..1d38c83e6 100644 --- a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts +++ b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts @@ -74,8 +74,8 @@ export class SystemForStartOs implements System { async exit(): Promise {} async start(effects: Effects): Promise { + if (this.runningMain) return effects.constRetry = utils.once(() => effects.restart()) - if (this.runningMain) await this.stop() let mainOnTerm: () => Promise | undefined const started = async (onTerm: () => Promise) => { await effects.setMainStatus({ status: "running" }) @@ -98,8 +98,11 @@ export class SystemForStartOs implements System { async stop(): Promise { if (this.runningMain) { - await this.runningMain.stop() - this.runningMain = undefined + try { + await this.runningMain.stop() + } finally { + this.runningMain = undefined + } } } } diff --git a/core/Cargo.lock b/core/Cargo.lock index 26a839a78..b4e8d701c 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1161,18 +1161,18 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const_format" -version = "0.2.31" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" dependencies = [ "const_format_proc_macros", ] [[package]] name = "const_format_proc_macros" -version = "0.2.31" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" dependencies = [ "proc-macro2", "quote", @@ -5540,9 +5540,9 @@ dependencies = [ [[package]] name = "sscanf" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a147d3cf7e723671ed11355b5b008c8019195f7fc902e213f5557d931e9f839d" +checksum = "c713ebd15ce561dd4a13ed62bc2a0368e16806fc30dcaf66ecf1256b2a3fdde6" dependencies = [ "const_format", "lazy_static", @@ -5552,9 +5552,9 @@ dependencies = [ [[package]] name = "sscanf_macro" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3a37bdf8e90e77cc60f74473edf28d922ae2eacdd595e67724ccd2381774cc" +checksum = "84955aa74a157e5834d58a07be11af7f0ab923f0194a0bb2ea6b3db8b5d1611d" dependencies = [ "convert_case 0.6.0", "proc-macro2", @@ -5637,6 +5637,7 @@ dependencies = [ "color-eyre", "console", "console-subscriber", + "const_format", "cookie", "cookie_store", "der", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 29c781af2..a62c9a49f 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -75,6 +75,7 @@ clap = "4.4.12" color-eyre = "0.6.2" console = "0.15.7" console-subscriber = { version = "0.3.0", optional = true } +const_format = "0.2.34" cookie = "0.18.0" cookie_store = "0.21.0" der = { version = "0.7.9", features = ["derive", "pem"] } diff --git a/core/startos/src/bins/registry.rs b/core/startos/src/bins/registry.rs index 8a52b2485..9c2cd2b92 100644 --- a/core/startos/src/bins/registry.rs +++ b/core/startos/src/bins/registry.rs @@ -1,7 +1,7 @@ use std::ffi::OsString; use clap::Parser; -use futures::{FutureExt, TryStreamExt}; +use futures::{FutureExt}; use tokio::signal::unix::signal; use tracing::instrument; diff --git a/core/startos/src/bins/start_init.rs b/core/startos/src/bins/start_init.rs index aad29c0c5..7ff903090 100644 --- a/core/startos/src/bins/start_init.rs +++ b/core/startos/src/bins/start_init.rs @@ -1,3 +1,4 @@ +use std::path::Path; use std::sync::Arc; use tokio::process::Command; @@ -16,7 +17,7 @@ use crate::prelude::*; use crate::progress::FullProgressTracker; use crate::shutdown::Shutdown; use crate::util::Invoke; -use crate::PLATFORM; +use crate::{DATA_DIR, PLATFORM}; #[instrument(skip_all)] async fn setup_or_init( @@ -156,7 +157,7 @@ async fn setup_or_init( let disk_guid = Arc::new(String::from(guid_string.trim())); let requires_reboot = crate::disk::main::import( &**disk_guid, - config.datadir(), + DATA_DIR, if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { @@ -182,7 +183,7 @@ async fn setup_or_init( let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1)); reboot_phase.start(); return Ok(Err(Shutdown { - export_args: Some((disk_guid, config.datadir().to_owned())), + export_args: Some((disk_guid, Path::new(DATA_DIR).to_owned())), restart: true, })); } diff --git a/core/startos/src/context/config.rs b/core/startos/src/context/config.rs index 811819479..3f631c1e4 100644 --- a/core/startos/src/context/config.rs +++ b/core/startos/src/context/config.rs @@ -13,6 +13,7 @@ use crate::disk::OsPartitionInfo; use crate::init::init_postgres; use crate::prelude::*; use crate::util::serde::IoFormat; +use crate::MAIN_DATA; pub const DEVICE_CONFIG_PATH: &str = "/media/startos/config/config.yaml"; // "/media/startos/config/config.yaml"; pub const CONFIG_PATH: &str = "/etc/startos/config.yaml"; @@ -110,8 +111,6 @@ pub struct ServerConfig { pub dns_bind: Option>, #[arg(long)] pub revision_cache_size: Option, - #[arg(short, long)] - pub datadir: Option, #[arg(long)] pub disable_encryption: Option, #[arg(long)] @@ -131,7 +130,6 @@ impl ContextConfig for ServerConfig { .revision_cache_size .take() .or(other.revision_cache_size); - self.datadir = self.datadir.take().or(other.datadir); self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption); self.multi_arch_s9pks = self.multi_arch_s9pks.take().or(other.multi_arch_s9pks); } @@ -145,13 +143,8 @@ impl ServerConfig { self.load_path_rec(Some(CONFIG_PATH))?; Ok(self) } - pub fn datadir(&self) -> &Path { - self.datadir - .as_deref() - .unwrap_or_else(|| Path::new("/embassy-data")) - } pub async fn db(&self) -> Result { - let db_path = self.datadir().join("main").join("embassy.db"); + let db_path = Path::new(MAIN_DATA).join("embassy.db"); let db = PatchDb::open(&db_path) .await .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?; @@ -160,7 +153,7 @@ impl ServerConfig { } #[instrument(skip_all)] pub async fn secret_store(&self) -> Result { - init_postgres(self.datadir()).await?; + init_postgres("/media/startos/data").await?; let secret_store = PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root")) .await?; diff --git a/core/startos/src/context/diagnostic.rs b/core/startos/src/context/diagnostic.rs index 0bf67e172..6acb21e30 100644 --- a/core/startos/src/context/diagnostic.rs +++ b/core/startos/src/context/diagnostic.rs @@ -1,5 +1,4 @@ use std::ops::Deref; -use std::path::PathBuf; use std::sync::Arc; use rpc_toolkit::yajrc::RpcError; @@ -13,7 +12,6 @@ use crate::shutdown::Shutdown; use crate::Error; pub struct DiagnosticContextSeed { - pub datadir: PathBuf, pub shutdown: Sender, pub error: Arc, pub disk_guid: Option>, @@ -25,7 +23,7 @@ pub struct DiagnosticContext(Arc); impl DiagnosticContext { #[instrument(skip_all)] pub fn init( - config: &ServerConfig, + _config: &ServerConfig, disk_guid: Option>, error: Error, ) -> Result { @@ -35,7 +33,6 @@ impl DiagnosticContext { let (shutdown, _) = tokio::sync::broadcast::channel(1); Ok(Self(Arc::new(DiagnosticContextSeed { - datadir: config.datadir().to_owned(), shutdown, disk_guid, error: Arc::new(error.into()), diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index 87245b4fa..a00b9d681 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::ops::Deref; -use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -48,7 +47,6 @@ pub struct RpcContextSeed { pub os_partitions: OsPartitionInfo, pub wifi_interface: Option, pub ethernet_interface: String, - pub datadir: PathBuf, pub disk_guid: Arc, pub ephemeral_sessions: SyncMutex, pub db: TypedPatchDb, @@ -214,7 +212,6 @@ impl RpcContext { let seed = Arc::new(RpcContextSeed { is_closed: AtomicBool::new(false), - datadir: config.datadir().to_path_buf(), os_partitions: config.os_partitions.clone().ok_or_else(|| { Error::new( eyre!("OS Partition Information Missing"), diff --git a/core/startos/src/context/setup.rs b/core/startos/src/context/setup.rs index 2db5668b9..30ca1ad39 100644 --- a/core/startos/src/context/setup.rs +++ b/core/startos/src/context/setup.rs @@ -1,5 +1,5 @@ use std::ops::Deref; -use std::path::PathBuf; +use std::path::{Path}; use std::sync::Arc; use std::time::Duration; @@ -10,8 +10,6 @@ use josekit::jwk::Jwk; use patch_db::PatchDb; use rpc_toolkit::Context; use serde::{Deserialize, Serialize}; -use sqlx::postgres::PgConnectOptions; -use sqlx::PgPool; use tokio::sync::broadcast::Sender; use tokio::sync::OnceCell; use tracing::instrument; @@ -22,13 +20,13 @@ use crate::context::config::ServerConfig; use crate::context::RpcContext; use crate::disk::OsPartitionInfo; use crate::hostname::Hostname; -use crate::init::init_postgres; use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter}; use crate::prelude::*; use crate::progress::FullProgressTracker; use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations}; use crate::setup::SetupProgress; use crate::util::net::WebSocketExt; +use crate::MAIN_DATA; lazy_static::lazy_static! { pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| { @@ -70,7 +68,6 @@ pub struct SetupContextSeed { pub task: OnceCell>, pub result: OnceCell>, pub shutdown: Sender<()>, - pub datadir: PathBuf, pub rpc_continuations: RpcContinuations, } @@ -83,7 +80,6 @@ impl SetupContext { config: &ServerConfig, ) -> Result { let (shutdown, _) = tokio::sync::broadcast::channel(1); - let datadir = config.datadir().to_owned(); Ok(Self(Arc::new(SetupContextSeed { webserver: webserver.acceptor_setter(), config: config.clone(), @@ -98,13 +94,12 @@ impl SetupContext { task: OnceCell::new(), result: OnceCell::new(), shutdown, - datadir, rpc_continuations: RpcContinuations::new(), }))) } #[instrument(skip_all)] pub async fn db(&self) -> Result { - let db_path = self.datadir.join("main").join("embassy.db"); + let db_path = Path::new(MAIN_DATA).join("embassy.db"); let db = PatchDb::open(&db_path) .await .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?; diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 92b9c8a88..90df469ba 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -18,7 +18,6 @@ use ts_rs::TS; use crate::account::AccountInfo; use crate::db::model::package::AllPackageData; use crate::net::acme::AcmeProvider; -use crate::net::utils::ipv6_is_local; use crate::prelude::*; use crate::progress::FullProgress; use crate::system::SmtpValue; @@ -187,6 +186,7 @@ impl NetworkInterfaceInfo { #[serde(rename_all = "camelCase")] pub struct IpInfo { pub scope_id: u32, + pub device_type: Option, #[ts(type = "string[]")] pub subnets: BTreeSet, pub wan_ip: Option, @@ -194,6 +194,15 @@ pub struct IpInfo { pub ntp_servers: BTreeSet, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, TS)] +#[ts(export)] +#[serde(rename_all = "kebab-case")] +pub enum NetworkInterfaceType { + Ethernet, + Wireless, + Wireguard, +} + #[derive(Debug, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] #[model = "Model"] diff --git a/core/startos/src/diagnostic.rs b/core/startos/src/diagnostic.rs index 71f76c379..73ef93125 100644 --- a/core/startos/src/diagnostic.rs +++ b/core/startos/src/diagnostic.rs @@ -1,3 +1,4 @@ +use std::path::Path; use std::sync::Arc; use rpc_toolkit::yajrc::RpcError; @@ -9,7 +10,7 @@ use crate::context::{CliContext, DiagnosticContext, RpcContext}; use crate::init::SYSTEM_REBUILD_PATH; use crate::shutdown::Shutdown; use crate::util::io::delete_file; -use crate::Error; +use crate::{Error, DATA_DIR}; pub fn diagnostic() -> ParentHandler { ParentHandler::new() @@ -70,7 +71,7 @@ pub fn restart(ctx: DiagnosticContext) -> Result<(), Error> { export_args: ctx .disk_guid .clone() - .map(|guid| (guid, ctx.datadir.clone())), + .map(|guid| (guid, Path::new(DATA_DIR).to_owned())), restart: true, }) .expect("receiver dropped"); diff --git a/core/startos/src/disk/mount/backup.rs b/core/startos/src/disk/mount/backup.rs index 2c322c284..3c6f97d91 100644 --- a/core/startos/src/disk/mount/backup.rs +++ b/core/startos/src/disk/mount/backup.rs @@ -7,7 +7,6 @@ use models::PackageId; use tokio::io::AsyncWriteExt; use tracing::instrument; -use super::filesystem::ecryptfs::EcryptFS; use super::guard::{GenericMountGuard, TmpMountGuard}; use crate::auth::check_password; use crate::backup::target::BackupInfo; diff --git a/core/startos/src/disk/mount/filesystem/mod.rs b/core/startos/src/disk/mount/filesystem/mod.rs index 818549a0a..80bfcc903 100644 --- a/core/startos/src/disk/mount/filesystem/mod.rs +++ b/core/startos/src/disk/mount/filesystem/mod.rs @@ -1,7 +1,6 @@ use std::ffi::OsStr; use std::fmt::{Display, Write}; use std::path::Path; -use std::time::Duration; use digest::generic_array::GenericArray; use digest::OutputSizeUser; diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index 3652336dc..21d23ab5a 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -7,6 +7,7 @@ use std::time::{Duration, SystemTime}; use axum::extract::ws::{self}; use color_eyre::eyre::eyre; +use const_format::formatcp; use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; use models::ResultExt; @@ -38,7 +39,7 @@ use crate::util::io::{create_file, IOHook}; use crate::util::lshw::lshw; use crate::util::net::WebSocketExt; use crate::util::{cpupower, Invoke}; -use crate::Error; +use crate::{Error, MAIN_DATA, PACKAGE_DATA}; pub const SYSTEM_REBUILD_PATH: &str = "/media/startos/config/system-rebuild"; pub const STANDBY_MODE_PATH: &str = "/media/startos/config/standby"; @@ -319,7 +320,7 @@ pub async fn init( })?; tokio::fs::set_permissions(LOCAL_AUTH_COOKIE_PATH, Permissions::from_mode(0o046)).await?; Command::new("chown") - .arg("root:embassy") + .arg("root:startos") .arg(LOCAL_AUTH_COOKIE_PATH) .invoke(crate::ErrorKind::Filesystem) .await?; @@ -362,7 +363,7 @@ pub async fn init( start_net.complete(); mount_logs.start(); - let log_dir = cfg.datadir().join("main/logs"); + let log_dir = Path::new(MAIN_DATA).join("logs"); if tokio::fs::metadata(&log_dir).await.is_err() { tokio::fs::create_dir_all(&log_dir).await?; } @@ -422,36 +423,28 @@ pub async fn init( load_ca_cert.complete(); load_wifi.start(); - crate::net::wifi::synchronize_network_manager( - &cfg.datadir().join("main"), - &mut server_info.wifi, - ) - .await?; + crate::net::wifi::synchronize_network_manager(MAIN_DATA, &mut server_info.wifi).await?; load_wifi.complete(); tracing::info!("Synchronized WiFi"); init_tmp.start(); - let tmp_dir = cfg.datadir().join("package-data/tmp"); + let tmp_dir = Path::new(PACKAGE_DATA).join("tmp"); if tokio::fs::metadata(&tmp_dir).await.is_ok() { tokio::fs::remove_dir_all(&tmp_dir).await?; } if tokio::fs::metadata(&tmp_dir).await.is_err() { tokio::fs::create_dir_all(&tmp_dir).await?; } - let tmp_var = cfg.datadir().join(format!("package-data/tmp/var")); + let tmp_var = Path::new(PACKAGE_DATA).join("tmp/var"); if tokio::fs::metadata(&tmp_var).await.is_ok() { tokio::fs::remove_dir_all(&tmp_var).await?; } crate::disk::mount::util::bind(&tmp_var, "/var/tmp", false).await?; - let downloading = cfg - .datadir() - .join(format!("package-data/archive/downloading")); + let downloading = Path::new(PACKAGE_DATA).join("archive/downloading"); if tokio::fs::metadata(&downloading).await.is_ok() { tokio::fs::remove_dir_all(&downloading).await?; } - let tmp_docker = cfg - .datadir() - .join(format!("package-data/tmp/{CONTAINER_TOOL}")); + let tmp_docker = Path::new(PACKAGE_DATA).join(formatcp!("tmp/{CONTAINER_TOOL}")); crate::disk::mount::util::bind(&tmp_docker, CONTAINER_DATADIR, false).await?; init_tmp.complete(); diff --git a/core/startos/src/install/mod.rs b/core/startos/src/install/mod.rs index f153c88ad..bc971da37 100644 --- a/core/startos/src/install/mod.rs +++ b/core/startos/src/install/mod.rs @@ -202,9 +202,6 @@ pub async fn sideload( use axum::extract::ws::Message; async move { if let Err(e) = async { - type RpcResponse = rpc_toolkit::yajrc::RpcResponse< - GenericRpcMethod<&'static str, (), FullProgress>, - >; tokio::select! { res = async { while let Some(progress) = progress_listener.next().await { diff --git a/core/startos/src/lib.rs b/core/startos/src/lib.rs index 1285b4811..6d05c43f4 100644 --- a/core/startos/src/lib.rs +++ b/core/startos/src/lib.rs @@ -1,3 +1,8 @@ +use const_format::formatcp; + +pub const DATA_DIR: &str = "/media/startos/data"; +pub const MAIN_DATA: &str = formatcp!("{DATA_DIR}/main"); +pub const PACKAGE_DATA: &str = formatcp!("{DATA_DIR}/package-data"); pub const DEFAULT_REGISTRY: &str = "https://registry.start9.com"; // pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com"; pub const HOST_IP: [u8; 4] = [10, 0, 3, 1]; diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index 60f9f4301..ce5d28970 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -1,5 +1,4 @@ use std::collections::BTreeSet; -use std::ffi::OsString; use std::net::Ipv4Addr; use std::path::Path; use std::sync::{Arc, Weak}; diff --git a/core/startos/src/net/mod.rs b/core/startos/src/net/mod.rs index fc0236300..49d3560ef 100644 --- a/core/startos/src/net/mod.rs +++ b/core/startos/src/net/mod.rs @@ -17,18 +17,12 @@ pub mod vhost; pub mod web_server; pub mod wifi; -pub const PACKAGE_CERT_PATH: &str = "/var/lib/embassy/ssl"; - pub fn net() -> ParentHandler { ParentHandler::new() .subcommand( "tor", tor::tor::().with_about("Tor commands such as list-services, logs, and reset"), ) - // .subcommand( - // "dhcp", - // network_interface::dhcp::().with_about("Command to update IP assigned from dhcp"), - // ) .subcommand( "acme", acme::acme::().with_about("Setup automatic clearnet certificate acquisition"), diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index d39a9861a..2de082d21 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -28,7 +28,7 @@ use zbus::zvariant::{ use zbus::{proxy, Connection}; use crate::context::{CliContext, RpcContext}; -use crate::db::model::public::{IpInfo, NetworkInterfaceInfo}; +use crate::db::model::public::{IpInfo, NetworkInterfaceInfo, NetworkInterfaceType}; use crate::db::model::Database; use crate::net::utils::{ipv6_is_link_local, ipv6_is_local}; use crate::prelude::*; @@ -52,10 +52,13 @@ pub fn network_interface_api() -> ParentHandler { } let mut table = Table::new(); - table.add_row(row![bc => "INTERFACE", "PUBLIC", "ADDRESSES", "WAN IP"]); + table.add_row(row![bc => "INTERFACE", "TYPE", "PUBLIC", "ADDRESSES", "WAN IP"]); for (iface, info) in res { table.add_row(row![ iface, + info.ip_info.as_ref() + .and_then(|ip_info| ip_info.device_type) + .map_or_else(|| "UNKNOWN".to_owned(), |ty| format!("{ty:?}")), info.public(), info.ip_info.as_ref().map_or_else( || "".to_owned(), @@ -289,6 +292,9 @@ mod device { #[zbus(property, name = "State")] fn _state(&self) -> Result; + #[zbus(property)] + fn device_type(&self) -> Result; + #[zbus(signal)] fn state_changed(&self) -> Result<(), Error>; } @@ -486,6 +492,13 @@ async fn watch_ip( return Ok(()); } + let device_type = match device_proxy.device_type().await? { + 1 => Some(NetworkInterfaceType::Ethernet), + 2 => Some(NetworkInterfaceType::Wireless), + 29 => Some(NetworkInterfaceType::Wireguard), + _ => None, + }; + let dhcp4_config = active_connection_proxy.dhcp4_config().await?; let ip4_proxy = Ip4ConfigProxy::new(&connection, ip4_config.clone()).await?; @@ -544,6 +557,7 @@ async fn watch_ip( }; Some(IpInfo { scope_id, + device_type, subnets, wan_ip, ntp_servers, @@ -890,6 +904,7 @@ impl ListenerMap { public: Some(false), ip_info: Some(IpInfo { scope_id: 1, + device_type: None, subnets: [ IpNet::new(Ipv4Addr::LOCALHOST.into(), 8).unwrap(), IpNet::new(Ipv6Addr::LOCALHOST.into(), 128).unwrap(), diff --git a/core/startos/src/net/static_server.rs b/core/startos/src/net/static_server.rs index 386f64d7b..d1070f9e1 100644 --- a/core/startos/src/net/static_server.rs +++ b/core/startos/src/net/static_server.rs @@ -46,7 +46,7 @@ use crate::s9pk::S9pk; use crate::util::io::open_file; use crate::util::net::SyncBody; use crate::util::serde::BASE64; -use crate::{diagnostic_api, init_api, install_api, main_api, setup_api}; +use crate::{diagnostic_api, init_api, install_api, main_api, setup_api, DATA_DIR}; const NOT_FOUND: &[u8] = b"Not Found"; const METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed"; @@ -266,7 +266,7 @@ fn s9pk_router(ctx: RpcContext) -> Router { let (parts, _) = request.into_parts(); match FileData::from_path( &parts, - &ctx.datadir + &Path::new(DATA_DIR) .join(PKG_ARCHIVE_DIR) .join("installed") .join(s9pk), @@ -292,7 +292,7 @@ fn s9pk_router(ctx: RpcContext) -> Router { let s9pk = S9pk::deserialize( &MultiCursorFile::from( open_file( - ctx.datadir + Path::new(DATA_DIR) .join(PKG_ARCHIVE_DIR) .join("installed") .join(s9pk), diff --git a/core/startos/src/os_install/gpt.rs b/core/startos/src/os_install/gpt.rs index 01703083b..4833f4ea7 100644 --- a/core/startos/src/os_install/gpt.rs +++ b/core/startos/src/os_install/gpt.rs @@ -50,7 +50,7 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result| async move { for volume_id in &s9pk.as_manifest().volumes { - let path = data_dir(&ctx.datadir, &s9pk.as_manifest().id, volume_id); + let path = data_dir(DATA_DIR, &s9pk.as_manifest().id, volume_id); if tokio::fs::metadata(&path).await.is_err() { tokio::fs::create_dir_all(&path).await?; } @@ -291,7 +292,7 @@ impl Service { Self::new(ctx, s9pk, start_stop).await.map(Some) } }; - let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed"); // TODO: make this based on hash + let s9pk_dir = Path::new(DATA_DIR).join(PKG_ARCHIVE_DIR).join("installed"); // TODO: make this based on hash let s9pk_path = s9pk_dir.join(id).with_extension("s9pk"); let Some(entry) = ctx .db diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index c99e1cac6..d99d9bb49 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -39,7 +39,7 @@ use crate::util::io::create_file; use crate::util::rpc_client::UnixRpcClient; use crate::util::Invoke; use crate::volume::data_dir; -use crate::ARCH; +use crate::{ARCH, DATA_DIR, PACKAGE_DATA}; const RPC_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); @@ -121,8 +121,8 @@ impl PersistentContainer { .lxc_manager .create( Some( - &ctx.datadir - .join("package-data/logs") + &Path::new(PACKAGE_DATA) + .join("logs") .join(&s9pk.as_manifest().id), ), LxcConfig::default(), @@ -157,7 +157,7 @@ impl PersistentContainer { .await?; let mount = MountGuard::mount( &IdMapped::new( - Bind::new(data_dir(&ctx.datadir, &s9pk.as_manifest().id, volume)), + Bind::new(data_dir(DATA_DIR, &s9pk.as_manifest().id, volume)), 0, 100000, 65536, diff --git a/core/startos/src/service/service_actor.rs b/core/startos/src/service/service_actor.rs index 712baaf1e..922d4e26b 100644 --- a/core/startos/src/service/service_actor.rs +++ b/core/startos/src/service/service_actor.rs @@ -92,7 +92,6 @@ async fn service_actor_loop( .. } => MainStatus::Stopped, }; - let previous = i.as_status().de()?; i.as_status_mut().ser(&main_status)?; return Ok(previous .major_changes(&main_status) diff --git a/core/startos/src/service/service_map.rs b/core/startos/src/service/service_map.rs index 0faab8dd5..104e191ba 100644 --- a/core/startos/src/service/service_map.rs +++ b/core/startos/src/service/service_map.rs @@ -1,3 +1,4 @@ +use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -27,6 +28,7 @@ use crate::service::start_stop::StartStop; use crate::service::{LoadDisposition, Service, ServiceRef}; use crate::status::MainStatus; use crate::util::serde::Pem; +use crate::DATA_DIR; pub type DownloadInstallFuture = BoxFuture<'static, Result>; pub type InstallFuture = BoxFuture<'static, Result<(), Error>>; @@ -220,8 +222,7 @@ impl ServiceMap { Ok(async move { let (installed_path, sync_progress_task) = reload_guard .handle(async { - let download_path = ctx - .datadir + let download_path = Path::new(DATA_DIR) .join(PKG_ARCHIVE_DIR) .join("downloading") .join(&id) @@ -251,8 +252,7 @@ impl ServiceMap { file.sync_all().await?; download_progress.complete(); - let installed_path = ctx - .datadir + let installed_path = Path::new(DATA_DIR) .join(PKG_ARCHIVE_DIR) .join("installed") .join(&id) diff --git a/core/startos/src/setup.rs b/core/startos/src/setup.rs index f2aba4328..e9109cabe 100644 --- a/core/startos/src/setup.rs +++ b/core/startos/src/setup.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use std::time::Duration; use color_eyre::eyre::eyre; +use const_format::formatcp; use josekit::jwk::Jwk; use patch_db::json_ptr::ROOT; use rpc_toolkit::yajrc::RpcError; @@ -38,7 +39,7 @@ use crate::rpc_continuations::Guid; use crate::util::crypto::EncryptedWire; use crate::util::io::{create_file, dir_copy, dir_size, Counter}; use crate::util::Invoke; -use crate::{Error, ErrorKind, ResultExt}; +use crate::{Error, ErrorKind, ResultExt, DATA_DIR, MAIN_DATA, PACKAGE_DATA}; pub fn setup() -> ParentHandler { ParentHandler::new() @@ -140,7 +141,7 @@ pub async fn attach( disk_phase.start(); let requires_reboot = crate::disk::main::import( &*disk_guid, - &setup_ctx.datadir, + DATA_DIR, if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { @@ -155,7 +156,7 @@ pub async fn attach( .with_ctx(|_| (ErrorKind::Filesystem, REPAIR_DISK_PATH))?; } if requires_reboot.0 { - crate::disk::main::export(&*disk_guid, &setup_ctx.datadir).await?; + crate::disk::main::export(&*disk_guid, DATA_DIR).await?; return Err(Error::new( eyre!( "Errors were corrected with your disk, but the server must be restarted in order to proceed" @@ -391,18 +392,13 @@ pub async fn execute_inner( crate::disk::main::create( &[start_os_logicalname], &pvscan().await?, - &ctx.datadir, + DATA_DIR, encryption_password, ) .await?, ); - let _ = crate::disk::main::import( - &*guid, - &ctx.datadir, - RepairStrategy::Preen, - encryption_password, - ) - .await?; + let _ = crate::disk::main::import(&*guid, DATA_DIR, RepairStrategy::Preen, encryption_password) + .await?; disk_phase.complete(); let progress = SetupExecuteProgress { @@ -520,10 +516,10 @@ async fn migrate( ) .await?; - let main_transfer_args = ("/media/startos/migrate/main/", "/embassy-data/main/"); + let main_transfer_args = ("/media/startos/migrate/main/", formatcp!("{MAIN_DATA}/")); let package_data_transfer_args = ( "/media/startos/migrate/package-data/", - "/embassy-data/package-data/", + formatcp!("{PACKAGE_DATA}/"), ); let tmpdir = Path::new(package_data_transfer_args.0).join("tmp"); diff --git a/core/startos/src/shutdown.rs b/core/startos/src/shutdown.rs index f6a984897..4e45f74c0 100644 --- a/core/startos/src/shutdown.rs +++ b/core/startos/src/shutdown.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::context::RpcContext; @@ -7,7 +7,7 @@ use crate::init::{STANDBY_MODE_PATH, SYSTEM_REBUILD_PATH}; use crate::prelude::*; use crate::sound::SHUTDOWN; use crate::util::Invoke; -use crate::PLATFORM; +use crate::{DATA_DIR, PLATFORM}; #[derive(Debug, Clone)] pub struct Shutdown { @@ -87,7 +87,7 @@ pub async fn shutdown(ctx: RpcContext) -> Result<(), Error> { .await?; ctx.shutdown .send(Some(Shutdown { - export_args: Some((ctx.disk_guid.clone(), ctx.datadir.clone())), + export_args: Some((ctx.disk_guid.clone(), Path::new(DATA_DIR).to_owned())), restart: false, })) .map_err(|_| ()) @@ -107,7 +107,7 @@ pub async fn restart(ctx: RpcContext) -> Result<(), Error> { .await?; ctx.shutdown .send(Some(Shutdown { - export_args: Some((ctx.disk_guid.clone(), ctx.datadir.clone())), + export_args: Some((ctx.disk_guid.clone(), Path::new(DATA_DIR).to_owned())), restart: true, })) .map_err(|_| ()) diff --git a/core/startos/src/status/mod.rs b/core/startos/src/status/mod.rs index 398797a5a..cf1ca0658 100644 --- a/core/startos/src/status/mod.rs +++ b/core/startos/src/status/mod.rs @@ -80,7 +80,7 @@ impl MainStatus { } } - pub fn backing_up(self) -> Self { + pub fn backing_up(&self) -> Self { MainStatus::BackingUp { on_complete: if self.running() { StartStop::Start diff --git a/core/startos/src/system.rs b/core/startos/src/system.rs index e64f30e98..02507b076 100644 --- a/core/startos/src/system.rs +++ b/core/startos/src/system.rs @@ -24,6 +24,7 @@ use crate::util::cpupower::{get_available_governors, set_governor, Governor}; use crate::util::io::open_file; use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat}; use crate::util::Invoke; +use crate::{MAIN_DATA, PACKAGE_DATA}; pub fn experimental() -> ParentHandler { ParentHandler::new() @@ -802,10 +803,10 @@ pub async fn get_mem_info() -> Result { #[instrument(skip_all)] async fn get_disk_info() -> Result { - let package_used_task = get_used("/embassy-data/package-data"); - let package_available_task = get_available("/embassy-data/package-data"); - let os_used_task = get_used("/embassy-data/main"); - let os_available_task = get_available("/embassy-data/main"); + let package_used_task = get_used(PACKAGE_DATA); + let package_available_task = get_available(PACKAGE_DATA); + let os_used_task = get_used(MAIN_DATA); + let os_available_task = get_available(MAIN_DATA); let (package_used, package_available, os_used, os_available) = futures::try_join!( package_used_task, diff --git a/core/startos/src/version/mod.rs b/core/startos/src/version/mod.rs index 2fd3e492b..e8d5f2249 100644 --- a/core/startos/src/version/mod.rs +++ b/core/startos/src/version/mod.rs @@ -7,12 +7,10 @@ use futures::future::BoxFuture; use futures::{Future, FutureExt}; use imbl::Vector; use imbl_value::{to_value, InternedString}; -use patch_db::json_ptr::{JsonPointer, ROOT}; +use patch_db::json_ptr::{ ROOT}; use crate::context::RpcContext; -use crate::db::model::Database; use crate::prelude::*; -use crate::progress::PhaseProgressTrackerHandle; use crate::Error; mod v0_3_5; diff --git a/core/startos/src/version/v0_3_6_alpha_0.rs b/core/startos/src/version/v0_3_6_alpha_0.rs index 64c7d2e12..7c594a9f9 100644 --- a/core/startos/src/version/v0_3_6_alpha_0.rs +++ b/core/startos/src/version/v0_3_6_alpha_0.rs @@ -1,19 +1,16 @@ use std::collections::BTreeMap; -use std::future::Future; use std::path::Path; use chrono::{DateTime, Utc}; +use const_format::formatcp; use ed25519_dalek::SigningKey; use exver::{PreReleaseSegment, VersionRange}; use imbl_value::{json, InternedString}; -use itertools::Itertools; use models::PackageId; -use openssl::pkey::{PKey, Private}; +use openssl::pkey::PKey; use openssl::x509::X509; -use patch_db::ModelExt; use sqlx::postgres::PgConnectOptions; use sqlx::{PgPool, Row}; -use ssh_key::Fingerprint; use tokio::process::Command; use torut::onion::TorSecretKeyV3; @@ -23,15 +20,11 @@ use crate::account::AccountInfo; use crate::auth::Sessions; use crate::backup::target::cifs::CifsTargets; use crate::context::RpcContext; -use crate::db::model::Database; use crate::disk::mount::filesystem::cifs::Cifs; use crate::disk::mount::util::unmount; use crate::hostname::Hostname; use crate::net::forward::AvailablePorts; use crate::net::keys::KeyStore; -use crate::net::ssl::CertStore; -use crate::net::tor; -use crate::net::tor::OnionStore; use crate::notifications::{Notification, Notifications}; use crate::prelude::*; use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; @@ -39,6 +32,7 @@ use crate::ssh::{SshKeys, SshPubKey}; use crate::util::crypto::ed25519_expand_key; use crate::util::serde::{Pem, PemEncoding}; use crate::util::Invoke; +use crate::{DATA_DIR, PACKAGE_DATA}; lazy_static::lazy_static! { static ref V0_3_6_alpha_0: exver::Version = exver::Version::new( @@ -207,7 +201,7 @@ impl VersionT for Version { &V0_3_0_COMPAT } async fn pre_up(self) -> Result { - let pg = init_postgres("/embassy-data").await?; + let pg = init_postgres(DATA_DIR).await?; let account = previous_account_info(&pg).await?; let ssh_keys = previous_ssh_keys(&pg).await?; @@ -327,7 +321,7 @@ impl VersionT for Version { #[instrument(skip(self, ctx))] /// MUST be idempotent, and is run after *all* db migrations async fn post_up(self, ctx: &RpcContext) -> Result<(), Error> { - let path = Path::new("/embassy-data/package-data/archive/"); + let path = Path::new(formatcp!("{PACKAGE_DATA}/archive/")); if !path.is_dir() { return Err(Error::new( eyre!( diff --git a/core/startos/src/version/v0_3_6_alpha_10.rs b/core/startos/src/version/v0_3_6_alpha_10.rs index f65479488..d81fc91ca 100644 --- a/core/startos/src/version/v0_3_6_alpha_10.rs +++ b/core/startos/src/version/v0_3_6_alpha_10.rs @@ -7,7 +7,6 @@ use torut::onion::OnionAddressV3; use super::v0_3_5::V0_3_0_COMPAT; use super::{v0_3_6_alpha_9, VersionT}; -use crate::db::model::Database; use crate::net::host::address::DomainConfig; use crate::prelude::*; diff --git a/core/startos/src/version/v0_3_6_alpha_6.rs b/core/startos/src/version/v0_3_6_alpha_6.rs index d91caa82b..7d62773ea 100644 --- a/core/startos/src/version/v0_3_6_alpha_6.rs +++ b/core/startos/src/version/v0_3_6_alpha_6.rs @@ -27,7 +27,7 @@ impl VersionT for Version { async fn pre_up(self) -> Result { Ok(()) } - fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> { + fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> { Ok(()) } async fn post_up<'a>(self, ctx: &'a crate::context::RpcContext) -> Result<(), Error> { diff --git a/core/startos/src/version/v0_3_6_alpha_7.rs b/core/startos/src/version/v0_3_6_alpha_7.rs index bbf9468ff..baf7aaa35 100644 --- a/core/startos/src/version/v0_3_6_alpha_7.rs +++ b/core/startos/src/version/v0_3_6_alpha_7.rs @@ -1,5 +1,5 @@ use exver::{PreReleaseSegment, VersionRange}; -use imbl_value::{json, InOMap}; +use imbl_value::json; use tokio::process::Command; use super::v0_3_5::V0_3_0_COMPAT; diff --git a/core/startos/src/version/v0_3_6_alpha_8.rs b/core/startos/src/version/v0_3_6_alpha_8.rs index fcef96a69..99920c767 100644 --- a/core/startos/src/version/v0_3_6_alpha_8.rs +++ b/core/startos/src/version/v0_3_6_alpha_8.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use exver::{PreReleaseSegment, VersionRange}; use tokio::fs::File; @@ -12,6 +14,7 @@ use crate::s9pk::v2::SIG_CONTEXT; use crate::s9pk::S9pk; use crate::service::LoadDisposition; use crate::util::io::create_file; +use crate::DATA_DIR; lazy_static::lazy_static! { static ref V0_3_6_alpha_8: exver::Version = exver::Version::new( @@ -40,7 +43,7 @@ impl VersionT for Version { Ok(()) } async fn post_up(self, ctx: &crate::context::RpcContext) -> Result<(), Error> { - let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed"); + let s9pk_dir = Path::new(DATA_DIR).join(PKG_ARCHIVE_DIR).join("installed"); if tokio::fs::metadata(&s9pk_dir).await.is_ok() { let mut read_dir = tokio::fs::read_dir(&s9pk_dir).await?; diff --git a/core/startos/src/volume.rs b/core/startos/src/volume.rs index c7165b202..113802286 100644 --- a/core/startos/src/volume.rs +++ b/core/startos/src/volume.rs @@ -1,10 +1,9 @@ use std::path::{Path, PathBuf}; pub use helpers::script_dir; +use models::PackageId; pub use models::VolumeId; -use models::{HostId, PackageId}; -use crate::net::PACKAGE_CERT_PATH; use crate::prelude::*; use crate::util::VersionString; @@ -36,7 +35,3 @@ pub fn asset_dir>( pub fn backup_dir(pkg_id: &PackageId) -> PathBuf { Path::new(BACKUP_DIR).join(pkg_id).join("data") } - -pub fn cert_dir(pkg_id: &PackageId, host_id: &HostId) -> PathBuf { - Path::new(PACKAGE_CERT_PATH).join(pkg_id).join(host_id) -} diff --git a/debian/postinst b/debian/postinst index 176bdb6b2..eeacfcbe1 100755 --- a/debian/postinst +++ b/debian/postinst @@ -105,7 +105,7 @@ rm -rf /var/lib/tor/* ln -sf /usr/lib/startos/scripts/tor-check.sh /usr/bin/tor-check ln -sf /usr/lib/startos/scripts/gather_debug_info.sh /usr/bin/gather-debug -echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/97-embassy.conf +echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/97-startos.conf # Old pi was set with this locale, because of pg we are now stuck with including that locale locale-gen en_GB en_GB.UTF-8 @@ -114,16 +114,14 @@ update-locale LANGUAGE rm -f "/etc/locale.gen" dpkg-reconfigure --frontend noninteractive locales -if ! getent group | grep '^embassy:'; then - groupadd embassy +if ! getent group | grep '^startos:'; then + groupadd startos fi -ln -sf /usr/lib/startos/scripts/dhclient-exit-hook /etc/dhcp/dhclient-exit-hooks.d/embassy - rm -f /etc/motd -ln -sf /usr/lib/startos/motd /etc/update-motd.d/00-embassy +ln -sf /usr/lib/startos/motd /etc/update-motd.d/00-startos chmod -x /etc/update-motd.d/* -chmod +x /etc/update-motd.d/00-embassy +chmod +x /etc/update-motd.d/00-startos # LXC cat /etc/subuid | grep -v '^root:' > /etc/subuid.tmp || true diff --git a/image-recipe/build.sh b/image-recipe/build.sh index 4ce2d6dac..9e61999e6 100755 --- a/image-recipe/build.sh +++ b/image-recipe/build.sh @@ -166,7 +166,7 @@ echo "deb [arch=${IB_TARGET_ARCH} signed-by=/etc/apt/trusted.gpg.d/docker.key.gp # Dependencies ## Base dependencies -dpkg-deb --fsys-tarfile $base_dir/deb/${IMAGE_BASENAME}.deb | tar --to-stdout -xvf - ./usr/lib/startos/depends > config/package-lists/embassy-depends.list.chroot +dpkg-deb --fsys-tarfile $base_dir/deb/${IMAGE_BASENAME}.deb | tar --to-stdout -xvf - ./usr/lib/startos/depends > config/package-lists/startos-depends.list.chroot ## Firmware if [ "$NON_FREE" = 1 ]; then @@ -210,7 +210,7 @@ if [ "${IB_TARGET_PLATFORM}" = "raspberrypi" ]; then mkinitramfs -c gzip -o /boot/initramfs_2712 6.6.62-v8-16k+ fi -useradd --shell /bin/bash -G embassy -m start9 +useradd --shell /bin/bash -G startos -m start9 echo start9:embassy | chpasswd usermod -aG sudo start9 diff --git a/sdk/base/lib/dependencies/setupDependencies.ts b/sdk/base/lib/dependencies/setupDependencies.ts index 710ec96ed..4583ae749 100644 --- a/sdk/base/lib/dependencies/setupDependencies.ts +++ b/sdk/base/lib/dependencies/setupDependencies.ts @@ -31,7 +31,7 @@ export type CurrentDependenciesResult = { [K in RequiredDependenciesOf]: DependencyRequirement } & { [K in OptionalDependenciesOf]?: DependencyRequirement -} & Record +} export function setupDependencies( fn: (options: { @@ -48,14 +48,16 @@ export function setupDependencies( } const dependencyType = await fn(options) return await options.effects.setDependencies({ - dependencies: Object.entries(dependencyType).map( - ([id, { versionRange, ...x }, ,]) => - ({ - id, - ...x, - versionRange: versionRange.toString(), - }) as T.DependencyRequirement, - ), + dependencies: Object.entries(dependencyType) + .map(([k, v]) => [k, v as DependencyRequirement] as const) + .map( + ([id, { versionRange, ...x }]) => + ({ + id, + ...x, + versionRange: versionRange.toString(), + }) as T.DependencyRequirement, + ), }) } return cell.updater diff --git a/sdk/base/lib/osBindings/IpInfo.ts b/sdk/base/lib/osBindings/IpInfo.ts index e9b0c9fb3..260add9e6 100644 --- a/sdk/base/lib/osBindings/IpInfo.ts +++ b/sdk/base/lib/osBindings/IpInfo.ts @@ -1,7 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { NetworkInterfaceType } from "./NetworkInterfaceType" export type IpInfo = { scopeId: number + deviceType: NetworkInterfaceType | null subnets: string[] wanIp: string | null ntpServers: string[] diff --git a/sdk/base/lib/osBindings/NetworkInterfaceType.ts b/sdk/base/lib/osBindings/NetworkInterfaceType.ts new file mode 100644 index 000000000..e20067dcc --- /dev/null +++ b/sdk/base/lib/osBindings/NetworkInterfaceType.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type NetworkInterfaceType = "ethernet" | "wireless" | "wireguard" diff --git a/sdk/base/lib/osBindings/index.ts b/sdk/base/lib/osBindings/index.ts index f3fec0b71..6eff86872 100644 --- a/sdk/base/lib/osBindings/index.ts +++ b/sdk/base/lib/osBindings/index.ts @@ -131,6 +131,7 @@ export { NamedHealthCheckResult } from "./NamedHealthCheckResult" export { NamedProgress } from "./NamedProgress" export { NetInfo } from "./NetInfo" export { NetworkInterfaceInfo } from "./NetworkInterfaceInfo" +export { NetworkInterfaceType } from "./NetworkInterfaceType" export { OnionHostname } from "./OnionHostname" export { OsIndex } from "./OsIndex" export { OsVersionInfoMap } from "./OsVersionInfoMap" diff --git a/sdk/base/lib/util/patterns.ts b/sdk/base/lib/util/patterns.ts index 2c9c7010d..c117c89e5 100644 --- a/sdk/base/lib/util/patterns.ts +++ b/sdk/base/lib/util/patterns.ts @@ -2,58 +2,58 @@ import { Pattern } from "../actions/input/inputSpecTypes" import * as regexes from "./regexes" export const ipv6: Pattern = { - regex: regexes.ipv6.toString(), + regex: regexes.ipv6.source, description: "Must be a valid IPv6 address", } export const ipv4: Pattern = { - regex: regexes.ipv4.toString(), + regex: regexes.ipv4.source, description: "Must be a valid IPv4 address", } export const hostname: Pattern = { - regex: regexes.hostname.toString(), + regex: regexes.hostname.source, description: "Must be a valid hostname", } export const localHostname: Pattern = { - regex: regexes.localHostname.toString(), + regex: regexes.localHostname.source, description: 'Must be a valid ".local" hostname', } export const torHostname: Pattern = { - regex: regexes.torHostname.toString(), + regex: regexes.torHostname.source, description: 'Must be a valid Tor (".onion") hostname', } export const url: Pattern = { - regex: regexes.url.toString(), + regex: regexes.url.source, description: "Must be a valid URL", } export const localUrl: Pattern = { - regex: regexes.localUrl.toString(), + regex: regexes.localUrl.source, description: 'Must be a valid ".local" URL', } export const torUrl: Pattern = { - regex: regexes.torUrl.toString(), + regex: regexes.torUrl.source, description: 'Must be a valid Tor (".onion") URL', } export const ascii: Pattern = { - regex: regexes.ascii.toString(), + regex: regexes.ascii.source, description: "May only contain ASCII characters. See https://www.w3schools.com/charsets/ref_html_ascii.asp", } export const email: Pattern = { - regex: regexes.email.toString(), + regex: regexes.email.source, description: "Must be a valid email address", } export const base64: Pattern = { - regex: regexes.base64.toString(), + regex: regexes.base64.source, description: "May only contain base64 characters. See https://base64.guru/learn/base64-characters", } diff --git a/sdk/package/lib/StartSdk.ts b/sdk/package/lib/StartSdk.ts index 390a5fe31..65ef23883 100644 --- a/sdk/package/lib/StartSdk.ts +++ b/sdk/package/lib/StartSdk.ts @@ -681,6 +681,18 @@ export class StartSdk { return Daemons.of({ effects, started, healthReceipts }) }, }, + SubContainer: { + of( + effects: Effects, + image: { + id: T.ImageId & keyof Manifest["images"] + sharedRun?: boolean + }, + name: string, + ) { + return SubContainer.of(effects, image, name) + }, + }, List: { /** * @description Create a list of text inputs. diff --git a/sdk/package/lib/mainFn/CommandController.ts b/sdk/package/lib/mainFn/CommandController.ts index 3b2285adb..498de2843 100644 --- a/sdk/package/lib/mainFn/CommandController.ts +++ b/sdk/package/lib/mainFn/CommandController.ts @@ -60,51 +60,59 @@ export class CommandController { } return subc })() - let childProcess: cp.ChildProcess - if (options.runAsInit) { - childProcess = await subc.launch(commands, { - env: options.env, - }) - } else { - childProcess = await subc.spawn(commands, { - env: options.env, - stdio: options.onStdout || options.onStderr ? "pipe" : "inherit", - }) - } - if (options.onStdout) childProcess.stdout?.on("data", options.onStdout) - if (options.onStderr) childProcess.stderr?.on("data", options.onStderr) + try { + let childProcess: cp.ChildProcess + if (options.runAsInit) { + childProcess = await subc.launch(commands, { + env: options.env, + }) + } else { + childProcess = await subc.spawn(commands, { + env: options.env, + stdio: options.onStdout || options.onStderr ? "pipe" : "inherit", + }) + } - const state = { exited: false } - const answer = new Promise((resolve, reject) => { - childProcess.on("exit", (code) => { - state.exited = true - if ( - code === 0 || - code === 143 || - (code === null && childProcess.signalCode == "SIGTERM") - ) { - return resolve(null) - } - if (code) { - return reject(new Error(`${commands[0]} exited with code ${code}`)) - } else { - return reject( - new Error( - `${commands[0]} exited with signal ${childProcess.signalCode}`, - ), - ) - } + if (options.onStdout) childProcess.stdout?.on("data", options.onStdout) + if (options.onStderr) childProcess.stderr?.on("data", options.onStderr) + + const state = { exited: false } + const answer = new Promise((resolve, reject) => { + childProcess.on("exit", (code) => { + state.exited = true + if ( + code === 0 || + code === 143 || + (code === null && childProcess.signalCode == "SIGTERM") + ) { + return resolve(null) + } + if (code) { + return reject( + new Error(`${commands[0]} exited with code ${code}`), + ) + } else { + return reject( + new Error( + `${commands[0]} exited with signal ${childProcess.signalCode}`, + ), + ) + } + }) }) - }) - return new CommandController( - answer, - state, - subc, - childProcess, - options.sigtermTimeout, - ) + return new CommandController( + answer, + state, + subc, + childProcess, + options.sigtermTimeout, + ) + } catch (e) { + await subc.destroy() + throw e + } } } get subContainerHandle() { @@ -121,7 +129,7 @@ export class CommandController { if (!this.state.exited) { this.process.kill("SIGKILL") } - await this.subcontainer.destroy?.().catch((_) => {}) + await this.subcontainer.destroy().catch((_) => {}) } } async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) { @@ -141,7 +149,7 @@ export class CommandController { await this.runningAnswer } finally { - await this.subcontainer.destroy?.() + await this.subcontainer.destroy() } } } diff --git a/sdk/package/lib/mainFn/Daemon.ts b/sdk/package/lib/mainFn/Daemon.ts index d9db89b6a..0cbc9cd94 100644 --- a/sdk/package/lib/mainFn/Daemon.ts +++ b/sdk/package/lib/mainFn/Daemon.ts @@ -60,6 +60,8 @@ export class Daemon { let timeoutCounter = 0 new Promise(async () => { while (this.shouldBeRunning) { + if (this.commandController) + await this.commandController.term().catch((err) => console.error(err)) this.commandController = await this.startCommand() await this.commandController.wait().catch((err) => console.error(err)) await new Promise((resolve) => setTimeout(resolve, timeoutCounter)) diff --git a/sdk/package/lib/mainFn/Daemons.ts b/sdk/package/lib/mainFn/Daemons.ts index 495e6c527..e18fa2314 100644 --- a/sdk/package/lib/mainFn/Daemons.ts +++ b/sdk/package/lib/mainFn/Daemons.ts @@ -5,7 +5,7 @@ import { HealthCheckResult } from "../health/checkFns" import { Trigger } from "../trigger" import * as T from "../../../base/lib/types" import { Mounts } from "./Mounts" -import { ExecSpawnable, MountOptions } from "../util/SubContainer" +import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer" import { promisify } from "node:util" import * as CP from "node:child_process" @@ -49,16 +49,18 @@ type DaemonsParams< > = { /** The command line command to start the daemon */ command: T.CommandType - /** Information about the image in which the daemon runs */ - image: { - /** The ID of the image. Must be one of the image IDs declared in the manifest */ - id: keyof Manifest["images"] & T.ImageId - /** - * Whether or not to share the `/run` directory with the parent container. - * This is useful if you are trying to connect to a service that exposes a unix domain socket or auth cookie via the `/run` directory - */ - sharedRun?: boolean - } + /** Information about the subcontainer in which the daemon runs */ + subcontainer: + | { + /** The ID of the image. Must be one of the image IDs declared in the manifest */ + id: keyof Manifest["images"] & T.ImageId + /** + * Whether or not to share the `/run` directory with the parent container. + * This is useful if you are trying to connect to a service that exposes a unix domain socket or auth cookie via the `/run` directory + */ + sharedRun?: boolean + } + | SubContainer /** For mounting the necessary volumes. Syntax: sdk.Mounts.of().addVolume() */ mounts: Mounts env?: Record @@ -147,11 +149,16 @@ export class Daemons options: DaemonsParams, ) { const daemonIndex = this.daemons.length - const daemon = Daemon.of()(this.effects, options.image, options.command, { - ...options, - mounts: options.mounts.build(), - subcontainerName: id, - }) + const daemon = Daemon.of()( + this.effects, + options.subcontainer, + options.command, + { + ...options, + mounts: options.mounts.build(), + subcontainerName: id, + }, + ) const healthDaemon = new HealthDaemon( daemon, daemonIndex, @@ -178,14 +185,18 @@ export class Daemons } async build() { - this.updateMainHealth() - this.healthDaemons.forEach((x) => - x.addWatcher(() => this.updateMainHealth()), - ) const built = { - term: async (options?: { signal?: Signals; timeout?: number }) => { + term: async () => { try { - await Promise.all(this.healthDaemons.map((x) => x.term(options))) + for (let result of await Promise.allSettled( + this.healthDaemons.map((x) => + x.term({ timeout: x.sigtermTimeout }), + ), + )) { + if (result.status === "rejected") { + console.error(result.reason) + } + } } finally { this.effects.setMainStatus({ status: "stopped" }) } @@ -194,8 +205,4 @@ export class Daemons this.started(() => built.term()) return built } - - private updateMainHealth() { - this.effects.setMainStatus({ status: "running" }) - } } diff --git a/sdk/package/lib/mainFn/HealthDaemon.ts b/sdk/package/lib/mainFn/HealthDaemon.ts index b66e3e406..ac459f08c 100644 --- a/sdk/package/lib/mainFn/HealthDaemon.ts +++ b/sdk/package/lib/mainFn/HealthDaemon.ts @@ -25,6 +25,8 @@ export class HealthDaemon { private _health: HealthCheckResult = { result: "starting", message: null } private healthWatchers: Array<() => unknown> = [] private running = false + private resolveReady: (() => void) | undefined + private readyPromise: Promise constructor( private readonly daemon: Promise, readonly daemonIndex: number, @@ -35,6 +37,7 @@ export class HealthDaemon { readonly effects: Effects, readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT, ) { + this.readyPromise = new Promise((resolve) => (this.resolveReady = resolve)) this.updateStatus() this.dependencies.forEach((d) => d.addWatcher(() => this.updateStatus())) } @@ -112,6 +115,12 @@ export class HealthDaemon { message: "message" in err ? err.message : String(err), } }) + if ( + this.resolveReady && + (response.result === "success" || response.result === "disabled") + ) { + this.resolveReady() + } await this.setHealth(response) } else { await this.setHealth({ @@ -129,6 +138,10 @@ export class HealthDaemon { } } + onReady() { + return this.readyPromise + } + private async setHealth(health: HealthCheckResult) { this._health = health this.healthWatchers.forEach((watcher) => watcher()) diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index 02ed582f1..4e7928581 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -46,6 +46,7 @@ export const mockPatchData: DataModel = { public: false, ipInfo: { scopeId: 1, + deviceType: 'ethernet', subnets: ['10.0.0.1/24'], wanIp: null, ntpServers: [], @@ -55,6 +56,7 @@ export const mockPatchData: DataModel = { public: false, ipInfo: { scopeId: 2, + deviceType: 'wireless', subnets: [ '10.0.90.12/24', 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64', From ab3cdd3bf8c7d05267eaad06ab230144f189cb8c Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 9 Jan 2025 15:00:33 -0700 Subject: [PATCH 29/29] misc fixes --- container-runtime/src/Adapters/RpcListener.ts | 4 +-- core/startos/src/net/net_controller.rs | 2 +- core/startos/src/net/network_interface.rs | 8 +++-- core/startos/src/service/mod.rs | 9 ++--- core/startos/src/service/service_actor.rs | 36 ++++++++++++++++++- sdk/package/package-lock.json | 4 +-- sdk/package/package.json | 2 +- 7 files changed, 49 insertions(+), 16 deletions(-) diff --git a/container-runtime/src/Adapters/RpcListener.ts b/container-runtime/src/Adapters/RpcListener.ts index 0585c2637..c2dc8bafe 100644 --- a/container-runtime/src/Adapters/RpcListener.ts +++ b/container-runtime/src/Adapters/RpcListener.ts @@ -213,7 +213,7 @@ export class RpcListener { Promise.resolve(a) .then((b) => b.toString()) .then((buf) => { - for (let s in buf.split("\n")) { + for (let s of buf.split("\n")) { if (s) Promise.resolve(s) .then(logData("dataIn")) @@ -396,7 +396,7 @@ export class RpcListener { .defaultToLazy(() => { console.warn( - `Coudln't parse the following input ${JSON.stringify(input)}`, + `Couldn't parse the following input ${JSON.stringify(input)}`, ) return { jsonrpc, diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 322e0dedc..7ff707fbe 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -118,7 +118,7 @@ pub struct NetController { db: TypedPatchDb, pub(super) tor: TorController, pub(super) vhost: VHostController, - pub(super) net_iface: Arc, + pub net_iface: Arc, pub(super) dns: DnsController, pub(super) forward: LanPortForwardController, pub(super) os_bindings: Vec>, diff --git a/core/startos/src/net/network_interface.rs b/core/startos/src/net/network_interface.rs index 2de082d21..991765c86 100644 --- a/core/startos/src/net/network_interface.rs +++ b/core/startos/src/net/network_interface.rs @@ -733,9 +733,11 @@ impl NetworkInterfaceController { l.insert(port, Arc::downgrade(&arc)); Ok(()) })?; + let mut ip_info = self.ip_info.subscribe(); + ip_info.mark_changed(); Ok(NetworkInterfaceListener { _arc: arc, - ip_info: self.ip_info.subscribe(), + ip_info, changed: None, listeners: ListenerMap::new(port), }) @@ -758,9 +760,11 @@ impl NetworkInterfaceController { l.insert(port, Arc::downgrade(&arc)); Ok(()) })?; + let mut ip_info = self.ip_info.subscribe(); + ip_info.mark_changed(); Ok(NetworkInterfaceListener { _arc: arc, - ip_info: self.ip_info.subscribe(), + ip_info, changed: None, listeners, }) diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 0054869bb..5bc7fd999 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -606,6 +606,7 @@ impl Service { } pub async fn update_host(&self, host_id: HostId) -> Result<(), Error> { + let mut service = self.seed.persistent_container.net_service.lock().await; let host = self .seed .ctx @@ -620,13 +621,7 @@ impl Service { .as_idx(&host_id) .or_not_found(&host_id)? .de()?; - self.seed - .persistent_container - .net_service - .lock() - .await - .update(host_id, host) - .await + service.update(host_id, host).await } } diff --git a/core/startos/src/service/service_actor.rs b/core/startos/src/service/service_actor.rs index 922d4e26b..b88136650 100644 --- a/core/startos/src/service/service_actor.rs +++ b/core/startos/src/service/service_actor.rs @@ -36,7 +36,41 @@ impl Actor for ServiceActor { ServiceActorLoopNext::DontWait => (), } } - }) + }); + let seed = self.0.clone(); + let mut ip_info = seed.ctx.net_controller.net_iface.subscribe(); + jobs.add_job(async move { + loop { + if let Err(e) = async { + let mut service = seed.persistent_container.net_service.lock().await; + let hosts = seed + .ctx + .db + .peek() + .await + .as_public() + .as_package_data() + .as_idx(&seed.id) + .or_not_found(&seed.id)? + .as_hosts() + .de()?; + for (host_id, host) in hosts.0 { + service.update(host_id, host).await?; + } + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error syncronizing net host after network change: {e}"); + tracing::debug!("{e:?}"); + } + + if ip_info.changed().await.is_err() { + break; + }; + } + }); } } diff --git a/sdk/package/package-lock.json b/sdk/package/package-lock.json index 50efc6b57..58a37e357 100644 --- a/sdk/package/package-lock.json +++ b/sdk/package/package-lock.json @@ -1,12 +1,12 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.2", + "version": "0.3.6-beta.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.2", + "version": "0.3.6-beta.3", "license": "MIT", "dependencies": { "@iarna/toml": "^2.2.5", diff --git a/sdk/package/package.json b/sdk/package/package.json index 6750ad372..1a68b9d1b 100644 --- a/sdk/package/package.json +++ b/sdk/package/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.2", + "version": "0.3.6-beta.3", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./package/lib/index.js", "types": "./package/lib/index.d.ts",