Skip to content

Commit

Permalink
cmd/microcloud: refactoring to respectively use OVNGeneveNetwork, `…
Browse files Browse the repository at this point in the history 
…MicroCephPublicNetwork` and `MicroCephInternalNetwork` instead of `OVNGeneveAddr`, `MicroCephPublicNetworkSubnet` and `MicroCephInternalNetworkSubnet`

Signed-off-by: Gabriel Mougard <[email protected]>
  • Loading branch information
@gabrielmougard
gabrielmougard committed Jan 9, 2025
1 parent 8d728e8 commit f0fa2c6
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 38 deletions.
70 changes: 53 additions & 17 deletions cmd/microcloud/ask.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -920,11 +920,11 @@ type Network struct {
Interface string
// IP is the IP address of the network. An example of why this is useful in MicroCloud is when we want to store
// a member OVN underlay network IP address.
IP net.IP
IP net.IP
// Subnet is the subnet of the network. An example of why this is useful in MicroCloud is when we want to check
// that we don't have subnet collisions between different network types (OVN, Ceph, etc) in a cluster.
// For example, we don't want a member using 'subnet A' for OVN and an other member using 'subnet A' for Ceph.
Subnet *net.IPNet
Subnet *net.IPNet
}

func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
Expand Down Expand Up @@ -1149,7 +1149,7 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
}
}

var ovnUnderlaySelectedIPs map[string]string
var ovnUnderlaySelectedNets map[string]*Network
ovnUnderlayData := [][]string{}
for peer, system := range c.systems {
// skip any systems that have already been clustered, but are available for other configuration.
Expand All @@ -1175,28 +1175,29 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {

if wantsDedicatedUnderlay {
header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"}
ovnUnderlaySelectedIPs = map[string]string{}
err = c.askRetry("Retry selecting underlay network interfaces?", func() error {
table := tui.NewSelectableTable(header, ovnUnderlayData)
answers, err := table.Render(context.Background(), c.asker, "Select exactly one network interface from each cluster member:")
if err != nil {
return err
}

ovnUnderlaySelectedIPs = map[string]string{}
ovnUnderlaySelectedNets = make(map[string]*Network)
for _, answer := range answers {
target := answer["LOCATION"]
ipAddr := answer["IP ADDRESS (CIDR)"]
if ovnUnderlaySelectedIPs[target] != "" {
ifaceName := answer["IFACE"]

if ovnUnderlaySelectedNets[target] != nil {
return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target)
}

ip, _, err := net.ParseCIDR(ipAddr)
ip, ipNet, err := net.ParseCIDR(ipAddr)
if err != nil {
return err
}

ovnUnderlaySelectedIPs[target] = ip.String()
ovnUnderlaySelectedNets[target] = &Network{Interface: ifaceName, IP: ip, Subnet: ipNet}
}

return nil
Expand All @@ -1207,11 +1208,11 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
}
}

if len(ovnUnderlaySelectedIPs) > 0 {
if len(ovnUnderlaySelectedNets) > 0 {
for peer := range askSystems {
underlayIP, ok := ovnUnderlaySelectedIPs[peer]
underlayNetwork, ok := ovnUnderlaySelectedNets[peer]
if ok {
fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayIP, peer)
fmt.Printf(" Using %q for OVN underlay traffic on %q\n", underlayNetwork.IP.String(), peer)
}
}

Expand Down Expand Up @@ -1248,10 +1249,10 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error {
system.Networks = append(system.Networks, finalConfigs...)
}

if ovnUnderlaySelectedIPs != nil {
ovnUnderlayIpAddr, ok := ovnUnderlaySelectedIPs[peer]
if ovnUnderlaySelectedNets != nil {
ovnUnderlayNet, ok := ovnUnderlaySelectedNets[peer]
if ok {
system.OVNGeneveAddr = ovnUnderlayIpAddr
system.OVNGeneveNetwork = ovnUnderlayNet
}
}

Expand Down Expand Up @@ -1409,14 +1410,34 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
return err
}

// Also register the MicroCloud internal network in the bootstrap system information.
microcloudNetworkInterface, err := lxd.FindInterfaceForSubnet(microCloudInternalNetworkAddrCIDR)
if err != nil {
return fmt.Errorf("Failed to find interface for subnet %q: %w", microCloudInternalNetworkAddrCIDR, err)
}

bootstrapSystem := c.systems[sh.Name]
bootstrapSystem.MicroCloudInternalNetwork = &Network{Interface: microcloudNetworkInterface.Name, Subnet: c.lookupSubnet, IP: microCloudInternalNetworkAddr}
c.systems[sh.Name] = bootstrapSystem

internalCephNetworkInterface, err := lxd.FindInterfaceForSubnet(internalCephSubnet)
if err != nil {
return fmt.Errorf("Failed to find interface for subnet %q: %w", internalCephSubnet, err)
}

if internalCephSubnet != microCloudInternalNetworkAddrCIDR {
err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, internalCephSubnet)
if err != nil {
return err
}

internalCephIP, internalCephNet, err := net.ParseCIDR(internalCephSubnet)
if err != nil {
return fmt.Errorf("Failed to parse the internal Ceph network: %w", err)
}

bootstrapSystem := c.systems[sh.Name]
bootstrapSystem.MicroCephInternalNetworkSubnet = internalCephSubnet
bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: internalCephNetworkInterface.Name, Subnet: internalCephNet, IP: internalCephIP}
c.systems[sh.Name] = bootstrapSystem
}

Expand All @@ -1425,6 +1446,11 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
return err
}

publicCephNetworkInterface, err := lxd.FindInterfaceForSubnet(publicCephSubnet)
if err != nil {
return fmt.Errorf("Failed to find interface for subnet %q: %w", publicCephSubnet, err)
}

if publicCephSubnet != internalCephSubnet {
err = c.validateCephInterfacesForSubnet(lxd, availableCephNetworkInterfaces, publicCephSubnet)
if err != nil {
Expand All @@ -1433,15 +1459,25 @@ func (c *initConfig) askCephNetwork(sh *service.Handler) error {
}

if publicCephSubnet != microCloudInternalNetworkAddrCIDR {
publicCephIP, publicCephNet, err := net.ParseCIDR(publicCephSubnet)
if err != nil {
return fmt.Errorf("Failed to parse the public Ceph network: %w", err)
}

bootstrapSystem := c.systems[sh.Name]
bootstrapSystem.MicroCephPublicNetworkSubnet = publicCephSubnet
bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: publicCephNetworkInterface.Name, Subnet: publicCephNet, IP: publicCephIP}
c.systems[sh.Name] = bootstrapSystem

// This is to avoid the situation where the internal network for Ceph has been skipped, but the public network has been set.
// Ceph will automatically set the internal network to the public Ceph network if the internal network is not set, which is not what we want.
// Instead, we still want to keep the internal Ceph network to use the MicroCloud internal network as a default.
if internalCephSubnet == microCloudInternalNetworkAddrCIDR {
bootstrapSystem.MicroCephInternalNetworkSubnet = microCloudInternalNetworkAddrCIDR
microcloudInternalIP, microcloudNet, err := net.ParseCIDR(microCloudInternalNetworkAddrCIDR)
if err != nil {
return fmt.Errorf("Failed to parse the internal MicroCloud network: %w", err)
}

bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: microcloudNetworkInterface.Name, Subnet: microcloudNet, IP: microcloudInternalIP}
c.systems[sh.Name] = bootstrapSystem
}
}
Expand Down
40 changes: 22 additions & 18 deletions cmd/microcloud/main_init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,24 @@ type InitSystem struct {
AvailableDisks []lxdAPI.ResourcesStorageDisk
// MicroCephDisks contains the disks intended to be passed to MicroCeph.
MicroCephDisks []cephTypes.DisksPost
// MicroCephPublicNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network.
MicroCephPublicNetworkSubnet string
// MicroCephClusterNetworkSubnet is an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network.
MicroCephInternalNetworkSubnet string
// MicroCephPublicNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph public network and
// the network interface name to use for the Ceph public network and its IP address within the subnet.
MicroCephPublicNetwork *Network
// MicroCephInternalNetwork contains an optional subnet (IPv4/IPv6 CIDR notation) for the Ceph cluster network and
// the network interface name to use for the Ceph cluster network and its IP address within the subnet.
MicroCephInternalNetwork *Network
// MicroCloudInternalNetwork contains the network configuration for the MicroCloud internal network.
MicroCloudInternalNetwork *Network
// TargetNetworks contains the network configuration for the target system.
TargetNetworks []lxdAPI.NetworksPost
// TargetStoragePools contains the storage pool configuration for the target system.
TargetStoragePools []lxdAPI.StoragePoolsPost
// Networks is the cluster-wide network configuration.
Networks []lxdAPI.NetworksPost
// OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system.
// OVNGeneveNetwork contains an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system.
// If left empty, the system will choose to route the Geneve traffic through the management network.
OVNGeneveAddr string
// It also contains the network interface name to use for the OVN Geneve tunnel and the network subnet.
OVNGeneveNetwork *Network
// StoragePools is the cluster-wide storage pool configuration.
StoragePools []lxdAPI.StoragePoolsPost
// StorageVolumes is the cluster-wide storage volume configuration.
Expand Down Expand Up @@ -354,9 +359,9 @@ func (c *initConfig) addPeers(sh *service.Handler) (revert.Hook, error) {
CephConfig: info.MicroCephDisks,
}

if info.OVNGeneveAddr != "" {
if info.OVNGeneveNetwork != nil {
p := joinConfig[peer]
p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveAddr}
p.OVNConfig = map[string]string{"ovn-encap-ip": info.OVNGeneveNetwork.IP.String()}
joinConfig[peer] = p
}
}
Expand Down Expand Up @@ -513,7 +518,7 @@ func validateGatewayNet(config map[string]string, ipPrefix string, cidrValidator

func (c *initConfig) validateSystems(s *service.Handler) (err error) {
for _, sys := range c.systems {
if sys.MicroCephInternalNetworkSubnet == "" || sys.OVNGeneveAddr == "" {
if sys.MicroCephInternalNetwork == nil || sys.OVNGeneveNetwork == nil {
continue
}

Expand All @@ -527,9 +532,8 @@ func (c *initConfig) validateSystems(s *service.Handler) (err error) {
return fmt.Errorf("OVN underlay IP %q is invalid", sys.OVNGeneveAddr)
}

if subnet.Contains(underlayIP) {
tui.PrintWarning(fmt.Sprintf("OVN underlay IP (%s) is shared with the Ceph cluster network (%s)\n", underlayIP.String(), subnet.String()))

if sys.MicroCephInternalNetwork.Subnet.Contains(sys.OVNGeneveNetwork.IP) {
tui.PrintWarning(fmt.Sprintf("OVN underlay IP (%s) is shared with the Ceph cluster network (%s)\n", sys.OVNGeneveNetwork.IP.String(), sys.MicroCephInternalNetwork.Subnet.String()))
break
}
}
Expand Down Expand Up @@ -660,12 +664,12 @@ func (c *initConfig) setupCluster(s *service.Handler) error {

if s.Type() == types.MicroCeph {
microCephBootstrapConf := make(map[string]string)
if bootstrapSystem.MicroCephInternalNetworkSubnet != "" {
microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetworkSubnet
if bootstrapSystem.MicroCephInternalNetwork != nil {
microCephBootstrapConf["ClusterNet"] = bootstrapSystem.MicroCephInternalNetwork.Subnet.String()
}

if bootstrapSystem.MicroCephPublicNetworkSubnet != "" {
microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetworkSubnet
if bootstrapSystem.MicroCephPublicNetwork != nil {
microCephBootstrapConf["PublicNet"] = bootstrapSystem.MicroCephPublicNetwork.Subnet.String()
}

if len(microCephBootstrapConf) > 0 {
Expand All @@ -675,8 +679,8 @@ func (c *initConfig) setupCluster(s *service.Handler) error {

if s.Type() == types.MicroOVN {
microOvnBootstrapConf := make(map[string]string)
if bootstrapSystem.OVNGeneveAddr != "" {
microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveAddr
if bootstrapSystem.OVNGeneveNetwork != nil {
microOvnBootstrapConf["ovn-encap-ip"] = bootstrapSystem.OVNGeneveNetwork.IP.String()
}

if len(microOvnBootstrapConf) > 0 {
Expand Down
48 changes: 45 additions & 3 deletions cmd/microcloud/preseed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,8 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map
return nil, fmt.Errorf("Failed to parse supplied underlay IP %q", sys.UnderlayIP)
}

ovnUnderlayIfaceByPeer := make(map[string]string)
ovnUnderlaySubnetByPeer := make(map[string]*net.IPNet)
for _, iface := range addressedInterfaces[sys.Name] {
for _, cidr := range iface.Addresses {
_, subnet, err := net.ParseCIDR(cidr)
Expand All @@ -803,6 +805,8 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map

if subnet.Contains(underlayIP) {
assignedSystems[sys.Name] = true
ovnUnderlayIfaceByPeer[sys.Name] = iface.Network.Name
ovnUnderlaySubnetByPeer[sys.Name] = subnet
break
}
}
Expand All @@ -812,8 +816,13 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map
return nil, fmt.Errorf("No available interface found for OVN underlay IP %q", sys.UnderlayIP)
}

ifaceName, ok := ovnUnderlayIfaceByPeer[sys.Name]
if !ok {
return nil, fmt.Errorf("Failed to find OVN underlay interface name for system %q", sys.Name)
}

system := c.systems[sys.Name]
system.OVNGeneveAddr = sys.UnderlayIP
system.OVNGeneveNetwork = &Network{Interface: ifaceName, Subnet: ovnUnderlaySubnetByPeer[sys.Name], IP: underlayIP}
c.systems[sys.Name] = system
}
}
Expand Down Expand Up @@ -1095,14 +1104,37 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map
internalCephNetwork = customTargetCephInternalNetwork
}

// Also register the MicroCloud internal network in the bootstrap system information.
microCloudInternalNetworkAddr := c.lookupSubnet.IP.Mask(c.lookupSubnet.Mask)
ones, _ := c.lookupSubnet.Mask.Size()
microCloudInternalNetworkAddrCIDR := fmt.Sprintf("%s/%d", microCloudInternalNetworkAddr.String(), ones)
microcloudNetworkInterface, err := lxd.FindInterfaceForSubnet(microCloudInternalNetworkAddrCIDR)
if err != nil {
return nil, fmt.Errorf("Failed to find interface for MicroCloud internal subnet %q: %w", microCloudInternalNetworkAddrCIDR, err)
}

bootstrapSystem := c.systems[s.Name]
bootstrapSystem.MicroCloudInternalNetwork = &Network{Interface: microcloudNetworkInterface.Name, Subnet: c.lookupSubnet, IP: microCloudInternalNetworkAddr}
c.systems[s.Name] = bootstrapSystem

if internalCephNetwork != "" {
err = c.validateCephInterfacesForSubnet(lxd, addressedInterfaces, internalCephNetwork)
if err != nil {
return nil, err
}

iface, err := lxd.FindInterfaceForSubnet(internalCephNetwork)
if err != nil {
return nil, err
}

_, internalCephSubnet, err := net.ParseCIDR(internalCephNetwork)
if err != nil {
return nil, fmt.Errorf("Invalid CIDR for internal Ceph subnet: %v", err)
}

bootstrapSystem := c.systems[s.Name]
bootstrapSystem.MicroCephInternalNetworkSubnet = internalCephNetwork
bootstrapSystem.MicroCephInternalNetwork = &Network{Interface: iface.Name, Subnet: internalCephSubnet, IP: nil}
c.systems[s.Name] = bootstrapSystem
}

Expand All @@ -1119,8 +1151,18 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map
return nil, err
}

iface, err := lxd.FindInterfaceForSubnet(publicCephNetwork)
if err != nil {
return nil, err
}

_, publicCephSubnet, err := net.ParseCIDR(publicCephNetwork)
if err != nil {
return nil, fmt.Errorf("Invalid CIDR for public Ceph subnet: %w", err)
}

bootstrapSystem := c.systems[s.Name]
bootstrapSystem.MicroCephPublicNetworkSubnet = publicCephNetwork
bootstrapSystem.MicroCephPublicNetwork = &Network{Interface: iface.Name, Subnet: publicCephSubnet, IP: nil}
c.systems[s.Name] = bootstrapSystem
}
} else {
Expand Down

0 comments on commit f0fa2c6

Please sign in to comment.