From cedfe90846c9f8c4ebb563aa67d9f12069f8cd6d Mon Sep 17 00:00:00 2001 From: Lisa Rashidi-Ranjbar Date: Fri, 6 Dec 2024 09:28:57 -0700 Subject: [PATCH] ARO-9395: Update fixssh to not filter on Master subnet instead of NIC name --- pkg/cluster/fixssh.go | 166 +++++++++++++++++- .../azuresdk/armnetwork/interfaces.go | 2 + 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/fixssh.go b/pkg/cluster/fixssh.go index f693afdef16..bef543cdb30 100644 --- a/pkg/cluster/fixssh.go +++ b/pkg/cluster/fixssh.go @@ -6,8 +6,10 @@ package cluster import ( "context" "fmt" + "regexp" "strings" + armnetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/go-autorest/autorest/to" @@ -15,6 +17,8 @@ import ( "github.com/Azure/ARO-RP/pkg/util/stringutils" ) +var nicNameRegex = regexp.MustCompile(`.*([0-2])\-nic`) + func (m *manager) fixSSH(ctx context.Context) error { infraID := m.doc.OpenShiftCluster.Properties.InfraID if infraID == "" { @@ -39,7 +43,8 @@ func (m *manager) fixSSH(ctx context.Context) error { return err } - err = m.checkandUpdateNIC(ctx, resourceGroup, infraID, lb) + // err = m.checkandUpdateNIC(ctx, resourceGroup, infraID, lb) + err = m.checkAndUpdateNICsInResourceGroup(ctx, resourceGroup, infraID, &lb) if err != nil { m.log.Warnf("Failed checking and Updating Network Interface with error: %s", err) return err @@ -48,6 +53,76 @@ func (m *manager) fixSSH(ctx context.Context) error { return nil } +func (m *manager) checkAndUpdateNICsInResourceGroup(ctx context.Context, resourceGroup string, infraID string, lb *mgmtnetwork.LoadBalancer) (err error) { + + opts := &armnetwork.InterfacesClientListAllOptions{} + pager := m.armInterfaces.NewListAllPager(opts) + + masterSubnetID := m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID + // r, err := arm.ParseResourceID(masterSubnetID) + // if err != nil { + // return err + // } + // masterSubnet, err := m.armSubnets.Get(ctx, resourceGroup, r.Parent.Name, r.Name, nil) + masterSubnetFound := false + + for pager.More() { + nextResult, err := pager.NextPage(ctx) + if err != nil { + m.log.Warnf("failed to advance page: %v", err) + return err + } + for _, nic := range nextResult.Value { + + // Look for master subnet + for _, ipc := range nic.Properties.IPConfigurations { + if *ipc.Properties.Subnet.ID == masterSubnetID { + masterSubnetFound = true + } + } + // If NIC is not associated with the master subnet continue to next NIC + if !masterSubnetFound { + continue + } + + if nic.Properties != nil && nic.Properties.VirtualMachine == nil { + err = m.removeBackendPoolsFromNICv2(ctx, resourceGroup, nic, masterSubnetID) + if err != nil { + m.log.Warnf("Removing BackendPools from NIC %s has failed with err %s", *nic.Name, err) + return err + } + } + + err = m.updateILBAddressPoolv2(ctx, nic, lb, resourceGroup, infraID, masterSubnetID) + if err != nil { + return err + } + + if m.doc.OpenShiftCluster.Properties.NetworkProfile.OutboundType == api.OutboundTypeUserDefinedRouting { + continue + } + + elbName := infraID + if m.doc.OpenShiftCluster.Properties.ArchitectureVersion == api.ArchitectureVersionV1 { + elbName = infraID + "-public-lb" + } + + elb, err := m.loadBalancers.Get(ctx, resourceGroup, elbName, "") + if err != nil { + return err + } + + err = m.updateELBAddressPoolv2(ctx, nic, &elb, resourceGroup, infraID, masterSubnetID) + if err != nil { + return err + } + } + + } + + return nil +} + func (m *manager) checkAndUpdateLB(ctx context.Context, resourceGroup string, lbName string) (lb mgmtnetwork.LoadBalancer, err error) { lb, err = m.loadBalancers.Get(ctx, resourceGroup, lbName, "") if err != nil { @@ -146,6 +221,24 @@ func (m *manager) removeBackendPoolsFromNIC(ctx context.Context, resourceGroup, return nil } +func (m *manager) removeBackendPoolsFromNICv2(ctx context.Context, resourceGroup string, nic *armnetwork.Interface, masterSubnetID string) error { + opts := &armnetwork.InterfacesClientBeginCreateOrUpdateOptions{ResumeToken: ""} + + if nic.Properties.IPConfigurations == nil { + return fmt.Errorf("unable to remove Backend Address Pools from NIC as there are no IP configurations for %s in resource group %s", *nic.Name, resourceGroup) + } + + for _, ipc := range nic.Properties.IPConfigurations { + if *ipc.Properties.Subnet.ID == masterSubnetID && ipc.Properties.LoadBalancerBackendAddressPools != nil { + m.log.Printf("Removing Load balancer Backend Address Pools from NIC %s with no VMs attached", *nic.Name) + ipc.Properties.LoadBalancerBackendAddressPools = []*armnetwork.BackendAddressPool{} + return m.armInterfaces.CreateOrUpdateAndWait(ctx, resourceGroup, *nic.Name, *nic, opts) + } + } + + return nil +} + func (m *manager) updateILBAddressPool(ctx context.Context, nic *mgmtnetwork.Interface, nicName string, lb *mgmtnetwork.LoadBalancer, i int, resourceGroup string, infraID string) error { if nic.InterfacePropertiesFormat.IPConfigurations == nil || len(*nic.InterfacePropertiesFormat.IPConfigurations) == 0 { return fmt.Errorf("unable to update NIC as there are no IP configurations for %s", nicName) @@ -198,6 +291,46 @@ func (m *manager) updateILBAddressPool(ctx context.Context, nic *mgmtnetwork.Int return nil } +func (m *manager) updateILBAddressPoolv2(ctx context.Context, nic *armnetwork.Interface, lb *mgmtnetwork.LoadBalancer, resourceGroup string, infraID string, masterSubnetID string) error { + opts := &armnetwork.InterfacesClientBeginCreateOrUpdateOptions{ResumeToken: ""} + + if nic.Properties.IPConfigurations == nil { + return fmt.Errorf("unable to update NIC as there are no IP configurations for %s", *nic.Name) + } + + ilbBackendPool := infraID + if m.doc.OpenShiftCluster.Properties.ArchitectureVersion == api.ArchitectureVersionV1 { + ilbBackendPool = infraID + "-internal-controlplane-v4" + } + + for _, ipc := range nic.Properties.IPConfigurations { + if *ipc.Properties.Subnet.ID == masterSubnetID && ipc.Properties.LoadBalancerBackendAddressPools == nil { + ipc.Properties.LoadBalancerBackendAddressPools = []*armnetwork.BackendAddressPool{} + } else if *ipc.Properties.Subnet.ID == masterSubnetID { + ilbBackendPoolID := fmt.Sprintf("%s/backendAddressPools/%s", *lb.ID, ilbBackendPool) + + m.log.Printf("Adding NIC %s to Internal Load Balancer API Address Pool %s", *nic.Name, ilbBackendPoolID) + ipc.Properties.LoadBalancerBackendAddressPools = append(ipc.Properties.LoadBalancerBackendAddressPools, &armnetwork.BackendAddressPool{ + ID: &ilbBackendPoolID, + }) + + // Find the index of the NIC we are working with + // nicNameRegex `.*([0-2])\-nic` + nicIndex := nicNameRegex.FindString(*nic.Name) + + sshBackendPoolID := fmt.Sprintf("%s/backendAddressPools/ssh-%s", *lb.ID, nicIndex) + m.log.Printf("Adding NIC %s to Internal Load Balancer API Address Pool %s", *nic.Name, sshBackendPoolID) + ipc.Properties.LoadBalancerBackendAddressPools = append(ipc.Properties.LoadBalancerBackendAddressPools, &armnetwork.BackendAddressPool{ + ID: &sshBackendPoolID, + }) + } + } + + m.log.Printf("updating Network Interface %s", *nic.Name) + return m.armInterfaces.CreateOrUpdateAndWait(ctx, resourceGroup, *nic.Name, *nic, opts) + +} + func (m *manager) updateV1ELBAddressPool(ctx context.Context, nic *mgmtnetwork.Interface, nicName string, resourceGroup string, infraID string) error { if nic.InterfacePropertiesFormat.IPConfigurations == nil || len(*nic.InterfacePropertiesFormat.IPConfigurations) == 0 { return fmt.Errorf("unable to update NIC as there are no IP configurations for %s", nicName) @@ -258,6 +391,37 @@ func (m *manager) updateELBAddressPool(ctx context.Context, nic *mgmtnetwork.Int return nil } +func (m *manager) updateELBAddressPoolv2(ctx context.Context, nic *armnetwork.Interface, lb *mgmtnetwork.LoadBalancer, resourceGroup string, infraID string, masterSubnetID string) error { + opts := &armnetwork.InterfacesClientBeginCreateOrUpdateOptions{ResumeToken: ""} + + if nic.Properties.IPConfigurations == nil { + return fmt.Errorf("unable to update NIC as there are no IP configurations for %s", *nic.Name) + } + + elbBackendPool := infraID + if m.doc.OpenShiftCluster.Properties.ArchitectureVersion == api.ArchitectureVersionV1 { + elbBackendPool = infraID + "-public-lb-control-plane-v4" + } + + elbBackendPoolID := fmt.Sprintf("%s/backendAddressPools/%s", *lb.ID, elbBackendPool) + + for _, ipc := range nic.Properties.IPConfigurations { + if *ipc.Properties.Subnet.ID == masterSubnetID && ipc.Properties.LoadBalancerBackendAddressPools == nil { + ipc.Properties.LoadBalancerBackendAddressPools = []*armnetwork.BackendAddressPool{} + } else if *ipc.Properties.Subnet.ID == masterSubnetID { + + m.log.Printf("Adding NIC %s to Public Load Balancer API Address Pool %s", *nic.Name, elbBackendPoolID) + ipc.Properties.LoadBalancerBackendAddressPools = append(ipc.Properties.LoadBalancerBackendAddressPools, &armnetwork.BackendAddressPool{ + ID: &elbBackendPoolID, + }) + + } + } + + m.log.Printf("updating Network Interface %s", *nic.Name) + return m.armInterfaces.CreateOrUpdateAndWait(ctx, resourceGroup, *nic.Name, *nic, opts) +} + func (m *manager) updateLB(ctx context.Context, lb *mgmtnetwork.LoadBalancer, lbName string) (changed bool) { backendAddressPools: for i := 0; i < 3; i++ { diff --git a/pkg/util/azureclient/azuresdk/armnetwork/interfaces.go b/pkg/util/azureclient/azuresdk/armnetwork/interfaces.go index 418fdccf2fe..b45e3aa14e6 100644 --- a/pkg/util/azureclient/azuresdk/armnetwork/interfaces.go +++ b/pkg/util/azureclient/azuresdk/armnetwork/interfaces.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" ) @@ -15,6 +16,7 @@ import ( type InterfacesClient interface { InterfacesClientAddons Get(ctx context.Context, resourceGroupName string, networkInterfaceName string, options *armnetwork.InterfacesClientGetOptions) (result armnetwork.InterfacesClientGetResponse, err error) + NewListAllPager(options *armnetwork.InterfacesClientListAllOptions) *runtime.Pager[armnetwork.InterfacesClientListAllResponse] } type interfacesClient struct {