Merge pull request #5251 from systeminit/si-fs

feat: a fuse filesystem for si

Cargo.toml

@@ -11,6 +11,7 @@ members = [
     "bin/pinga",
     "bin/rebaser",
     "bin/sdf",
+    "bin/si-fs",
     "bin/veritech",
     "lib/asset-sprayer",
     "lib/audit-database",
@@ -53,6 +54,7 @@ members = [
     "lib/si-data-pg",
     "lib/si-data-spicedb",
     "lib/si-events-rs",
+    "lib/si-filesystem",
     "lib/si-firecracker",
     "lib/si-frontend-types-rs",
     "lib/si-hash",
@@ -121,10 +123,11 @@ directories = "5.0.1"
 dyn-clone = "1.0.17"
 fastrace = "0.7.4"
 flate2 = "1.0.35"
-futures = "0.3.31"
-futures-lite = "2.5.0"
 foyer = { version = "0.13.1", features = ["tracing", "opentelemetry_0_26"] }
 fs4 = "0.12.0"
+fuser = { version = "0.15.1", default-features = false }
+futures = "0.3.31"
+futures-lite = "2.5.0"
 glob = "0.3.1"
 hex = "0.4.3"
 http = "0.2.12" # todo: upgrade this alongside hyper/axum/tokio-tungstenite/tower-http
@@ -151,8 +154,10 @@ mime_guess = { version = "=2.0.4" } # TODO(fnichol): 2.0.5 sets an env var in bu
 miniz_oxide = { version = "0.8.0", features = ["simd"] }
 monostate = "0.1.13"
 names = { version = "0.14.0", default-features = false }
-nix = { version = "0.26.0", features = [
+nix = { version = "0.29.0", features = [
+    "poll",
     "fs",
+    "ioctl",
     "mount",
     "process",
     "signal",

bin/si-fs/BUCK

@@ -0,0 +1,20 @@
+load(
+    "@prelude-si//:macros.bzl",
+    "docker_image",
+    "nix_omnibus_pkg",
+    "rust_binary",
+)
+
+rust_binary(
+    name = "si-fs",
+    deps = [
+        "//lib/si-filesystem:si-filesystem",
+        "//third-party/rust:clap",
+        "//third-party/rust:color-eyre",
+        "//third-party/rust:nix",
+        "//third-party/rust:tokio",
+    ],
+    srcs = glob(["src/**/*.rs"]),
+    env = {"CARGO_BIN_NAME": "si-fs"},
+)
+

bin/si-fs/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "si-fs"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+publish.workspace = true
+
+[dependencies]
+clap = { workspace = true }
+color-eyre = { workspace = true }
+si-filesystem = { path = "../../lib/si-filesystem" }
+tokio = { workspace = true }
+nix = { workspace = true }

bin/si-fs/src/main.rs

@@ -0,0 +1,78 @@
+use std::str::FromStr;
+
+use clap::Parser;
+use color_eyre::Result;
+use nix::{fcntl::OFlag, sys::stat::Mode, unistd::ForkResult};
+use si_filesystem::{mount, WorkspaceId};
+use tokio::runtime::Runtime;
+
+#[derive(Parser, Debug)]
+struct Args {
+    #[arg(long, short = 'w')]
+    workspace_id: String,
+    #[arg(long, short = 'e')]
+    endpoint: String,
+    #[arg(long, short = 't', env = "SI_BEARER_TOKEN")]
+    token: String,
+    #[arg(long, short = 'f')]
+    foreground: bool,
+    #[arg(value_name = "MOUNTPOINT")]
+    mount_point: String,
+}
+
+fn redirect_to_dev_null() -> Result<()> {
+    let dev_null_fd = nix::fcntl::open("/dev/null", OFlag::O_RDWR, Mode::empty())?;
+
+    nix::unistd::dup2(dev_null_fd, nix::libc::STDOUT_FILENO)?;
+    nix::unistd::dup2(dev_null_fd, nix::libc::STDIN_FILENO)?;
+    nix::unistd::dup2(dev_null_fd, nix::libc::STDERR_FILENO)?;
+
+    nix::unistd::close(dev_null_fd)?;
+
+    Ok(())
+}
+
+fn daemonize() -> Result<()> {
+    match unsafe { nix::unistd::fork() }? {
+        ForkResult::Parent { .. } => {
+            std::process::exit(0);
+        }
+        ForkResult::Child => {
+            nix::unistd::setsid()?;
+
+            match unsafe { nix::unistd::fork() }? {
+                ForkResult::Parent { .. } => {
+                    std::process::exit(0);
+                }
+                ForkResult::Child => {
+                    redirect_to_dev_null()?;
+
+                    Ok(())
+                }
+            }
+        }
+    }
+}
+
+fn main() -> Result<()> {
+    color_eyre::install()?;
+    let args = Args::parse();
+    let workspace_id = WorkspaceId::from_str(&args.workspace_id)?;
+
+    if !args.foreground {
+        daemonize()?;
+    }
+
+    let rt = Runtime::new()?;
+
+    mount(
+        args.token.clone(),
+        args.endpoint.clone(),
+        workspace_id,
+        &args.mount_point,
+        rt.handle().clone(),
+        None,
+    );
+
+    Ok(())
+}

lib/dal/src/change_set.rs

@@ -155,7 +155,6 @@ pub struct ChangeSet {
     pub id: ChangeSetId,
     pub created_at: DateTime<Utc>,
     pub updated_at: DateTime<Utc>,
-
     pub name: String,
     pub status: ChangeSetStatus,
     pub base_change_set_id: Option<ChangeSetId>,

lib/sdf-server/src/service/change_set/create_change_set.rs

@@ -2,27 +2,15 @@ use axum::extract::{Host, OriginalUri};
 use axum::Json;
 use dal::change_set::ChangeSet;
 use dal::WsEvent;
-use serde::{Deserialize, Serialize};
 use si_events::audit_log::AuditLogKind;
+use si_frontend_types::{CreateChangeSetRequest, CreateChangeSetResponse};
 
 use super::ChangeSetResult;
 use crate::{
     extract::{v1::AccessBuilder, HandlerContext, PosthogClient},
     track,
 };
 
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct CreateChangeSetRequest {
-    pub change_set_name: String,
-}
-
-#[derive(Deserialize, Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct CreateChangeSetResponse {
-    pub change_set: ChangeSet,
-}
-
 pub async fn create_change_set(
     HandlerContext(builder): HandlerContext,
     AccessBuilder(access_builder): AccessBuilder,
@@ -58,6 +46,7 @@ pub async fn create_change_set(
         .publish_on_commit(&ctx)
         .await?;
 
+    let change_set = change_set.into_frontend_type(&ctx).await?;
     ctx.commit_no_rebase().await?;
 
     Ok(Json(CreateChangeSetResponse { change_set }))

lib/sdf-server/src/service/v2/variant/list_variants.rs

@@ -4,10 +4,9 @@ use axum::{
     extract::{Host, OriginalUri, Path},
     Json,
 };
+
 use dal::{cached_module::CachedModule, ChangeSetId, SchemaVariant, WorkspacePk};
-use frontend_types::{SchemaVariant as FrontendVariant, UninstalledVariant};
-use serde::{Deserialize, Serialize};
-use si_frontend_types as frontend_types;
+use si_frontend_types::ListVariantsResponse;
 
 use crate::{
     extract::{HandlerContext, PosthogClient},
@@ -16,13 +15,6 @@ use crate::{
     track,
 };
 
-#[derive(Debug, Deserialize, Serialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ListVariantsResponse {
-    installed: Vec<FrontendVariant>,
-    uninstalled: Vec<UninstalledVariant>,
-}
-
 pub async fn list_variants(
     HandlerContext(builder): HandlerContext,
     AccessBuilder(access_builder): AccessBuilder,

lib/si-filesystem/BUCK

@@ -0,0 +1,18 @@
+load("@prelude-si//:macros.bzl", "rust_library")
+
+rust_library(
+    name = "si-filesystem",
+    deps = [
+        "//lib/si-id:si-id",
+        "//lib/si-frontend-types-rs:si-frontend-types",
+
+        "//third-party/rust:fuser",
+        "//third-party/rust:nix",
+        "//third-party/rust:reqwest",
+        "//third-party/rust:thiserror",
+        "//third-party/rust:tokio",
+    ],
+    srcs = glob([
+        "src/**/*.rs",
+    ]),
+)

lib/si-filesystem/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "si-filesystem"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+publish.workspace = true
+
+[dependencies]
+si-id = { path = "../si-id" }
+si-frontend-types = { path = "../si-frontend-types-rs" }
+
+fuser = { workspace = true }
+nix = { workspace = true }
+reqwest = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }