Create Ceph cluster

modules/proxmox-ve/ceph.nix

@@ -88,6 +88,9 @@ let
       description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
       partOf = [ "ceph.target" ];
       wantedBy = [ "ceph.target" ];
+      wants = lib.mkIf (daemonType == "osd") (
+        map (daemonId: "ceph-volume@${daemonId}.service") cfg.osd.daemons
+      );
       before = [ "ceph.target" ];
       unitConfig.StopWhenUnneeded = true;
     };
@@ -212,9 +215,9 @@ in
 
     networking.firewall = lib.mkIf config.services.proxmox-ve.openFirewall {
       allowedTCPPorts = lib.optionals cfg.mon.enable [
-          3300
-          6789
-        ];
+        3300
+        6789
+      ];
       allowedTCPPortRanges = lib.optionals (cfg.osd.enable || cfg.msd.enable || cfg.mgr.enable) [
         {
           from = 6800;
@@ -240,6 +243,20 @@ in
           lib.optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
           ++ lib.optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
           ++ lib.optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
+          ++ lib.optional cfg.osd.enable {
+            "ceph-volume@" = {
+              description = "Ceph Volume activation: %i";
+              after = [ "local-fs.target" ];
+              wants = [ "local-fs.target" ];
+              environment.CEPH_VOLUME_TIMEOUT = "10000";
+              serviceConfig = {
+                Type = "oneshot";
+                KillMode = "none";
+                ExecStart = "${pkgs.ceph}/bin/ceph-volume-systemd %i";
+                TimeoutSec = 0;
+              };
+            };
+          }
           ++ lib.optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
           ++ lib.optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
       in

tests/ceph.nix

@@ -28,7 +28,10 @@
           hashedPasswordFile = null;
         };
 
-        virtualisation.emptyDiskImages = [ 1024 ];
+        virtualisation = {
+          emptyDiskImages = [ 1024 ];
+          memorySize = 2048;
+        };
       };
 
     pve2 = {
@@ -45,7 +48,10 @@
         };
       };
 
-      virtualisation.emptyDiskImages = [ 1024 ];
+      virtualisation = {
+        emptyDiskImages = [ 1024 ];
+        memorySize = 2048;
+      };
     };
 
     pve3 = {
@@ -62,7 +68,10 @@
         };
       };
 
-      virtualisation.emptyDiskImages = [ 1024 ];
+      virtualisation = {
+        emptyDiskImages = [ 1024 ];
+        memorySize = 2048;
+      };
     };
   };
 
@@ -87,10 +96,18 @@
     time.sleep(10)
     pve2.succeed(f"pvesh create /cluster/config/join --hostname 192.168.1.1 --fingerprint {fingerprint.strip()} --password 'mypassword'")
 
+    pve3.wait_for_unit("multi-user.target")
+    time.sleep(10)
+    pve3.succeed(f"pvesh create /cluster/config/join --hostname 192.168.1.1 --fingerprint {fingerprint.strip()} --password 'mypassword'")
+
+    assert "pve2" in pve1.succeed("pvecm nodes")
+    assert "pve3" in pve1.succeed("pvecm nodes")
+    assert "Yes" in pve1.succeed("pvecm status | grep Quorate")
+
     pve1.succeed(
       "pveceph init",
-      "pveceph mon create",
       "pveceph mgr create",
+      "pveceph mon create",
       "ceph-volume lvm create --osd-id 1 --data /dev/vdb --no-systemd"
     )
 
@@ -105,5 +122,12 @@
       "pveceph mon create",
       "ceph-volume lvm create --osd-id 3 --data /dev/vdb --no-systemd"
     )
+
+    pve1.wait_for_unit("ceph.target")
+    pve2.wait_for_unit("ceph.target")
+    pve3.wait_for_unit("ceph.target")
+
+    time.sleep(10)
+    assert "HEALTH_OK" in pve1.succeed("ceph status")
   '';
 }

tests/default.nix

@@ -13,7 +13,7 @@ let
 in
 {
   test-pve-basic = runTest ./basic.nix;
-  test-pve-ceph = runTest ./vm.nix;
+  test-pve-ceph = runTest ./ceph.nix;
   test-pve-cluster = runTest ./cluster.nix;
   # Disable this test until drdb gets unbroken in 24.11
   # test-pve-linstor = runTest ./linstor.nix;