Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Add Equinix Metal Load Balancer support #740

Merged
merged 16 commits into from
May 28, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .golangci.yml
Original file line number Diff line number Diff line change
@@ -164,7 +164,6 @@ linters-settings:
alias: infraexpv1
nolintlint:
allow-unused: false
allow-leading-space: false
require-specific: true
revive:
rules:
@@ -314,3 +313,7 @@ issues:
- gocritic
text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop"
path: _test\.go
- linters:
- bodyclose
path: .*(internal)/emlb/emlb.go
text: "response body must be closed"
6 changes: 3 additions & 3 deletions api/v1beta1/packetcluster_types.go
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ const (
NetworkInfrastructureReadyCondition clusterv1.ConditionType = "NetworkInfrastructureReady"
)

// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip.
// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip or Equinix Metal Load Balancer.
type VIPManagerType string

// PacketClusterSpec defines the desired state of PacketCluster.
@@ -46,9 +46,9 @@ type PacketClusterSpec struct {
// +optional
ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`

// VIPManager represents whether this cluster uses CPEM or kube-vip to
// VIPManager represents whether this cluster uses CPEM or kube-vip or Equinix Metal Load Balancer to
// manage its vip for the api server IP
// +kubebuilder:validation:Enum=CPEM;KUBE_VIP
// +kubebuilder:validation:Enum=CPEM;KUBE_VIP;EMLB
// +kubebuilder:default:=CPEM
VIPManager VIPManagerType `json:"vipManager"`
}
2 changes: 1 addition & 1 deletion api/v1beta1/packetcluster_webhook.go
Original file line number Diff line number Diff line change
@@ -94,7 +94,7 @@ func (c *PacketCluster) ValidateUpdate(oldRaw runtime.Object) (admission.Warning
}

// Must have only one of Metro or Facility
if len(c.Spec.Facility) > 0 && len(c.Spec.Metro) > 0 {
if c.Spec.Facility != "" && c.Spec.Metro != "" {
allErrs = append(allErrs,
field.Invalid(field.NewPath("spec", "Facility"),
c.Spec.Facility, "Metro and Facility are mutually exclusive, Metro is recommended"),
2 changes: 1 addition & 1 deletion api/v1beta1/packetmachine_webhook.go
Original file line number Diff line number Diff line change
@@ -53,7 +53,7 @@ func (m *PacketMachine) ValidateUpdate(old runtime.Object) (admission.Warnings,
var allErrs field.ErrorList

// Must have only one of Metro or Facility specified
if len(m.Spec.Facility) > 0 && len(m.Spec.Metro) > 0 {
if m.Spec.Facility != "" && m.Spec.Metro != "" {
allErrs = append(allErrs,
field.Invalid(field.NewPath("spec", "Facility"),
m.Spec.Facility, "Metro and Facility field are mutually exclusive"),
2 changes: 1 addition & 1 deletion clusterctl-settings.json
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
"name": "infrastructure-packet",
"config": {
"componentsFile": "infrastructure-components.yaml",
"nextVersion": "v0.6.99"
"nextVersion": "v0.8.99"
}
}

2 changes: 1 addition & 1 deletion cmd/ci-clean/main.go
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ func main() {
rootCmd := &cobra.Command{ //nolint:exhaustivestruct
Use: "ci-clean",
Short: "Clean up any stray resources in CI",
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
metalAuthToken := os.Getenv(authTokenEnvVar)
if metalAuthToken == "" {
return fmt.Errorf("%s: %w", authTokenEnvVar, errMissingRequiredEnvVar)
Original file line number Diff line number Diff line change
@@ -80,11 +80,12 @@ spec:
vipManager:
default: CPEM
description: |-
VIPManager represents whether this cluster uses CPEM or kube-vip to
VIPManager represents whether this cluster uses CPEM or kube-vip or Equinix Metal Load Balancer to
manage its vip for the api server IP
enum:
- CPEM
- KUBE_VIP
- EMLB
type: string
required:
- projectID
4 changes: 2 additions & 2 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
@@ -42,10 +42,10 @@ spec:
port: healthz
resources:
limits:
memory: 200Mi
memory: 300Mi
requests:
cpu: 100m
memory: 200Mi
memory: 300Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
81 changes: 48 additions & 33 deletions controllers/packetcluster_controller.go
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/handler"

infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
)
@@ -114,49 +115,63 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc

packetCluster := clusterScope.PacketCluster

ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
switch {
case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
// Parse metro and facility from the cluster spec
var metro, facility string

facility = packetCluster.Spec.Facility
metro = packetCluster.Spec.Metro

// If both specified, metro takes precedence over facility
if metro != "" {
facility = ""
case packetCluster.Spec.VIPManager == emlb.EMLBVIPID:
if !packetCluster.Spec.ControlPlaneEndpoint.IsValid() {
// Create new EMLB object
lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], packetCluster.Spec.ProjectID, packetCluster.Spec.Metro)

if err := lb.ReconcileLoadBalancer(ctx, clusterScope); err != nil {
log.Error(err, "Error Reconciling EMLB")
return err
}
}

// There is not an ElasticIP with the right tags, at this point we can create one
ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
if err != nil {
log.Error(err, "error reserving an ip")
case packetCluster.Spec.VIPManager == "KUBE_VIP":
log.Info("KUBE_VIP VIPManager Detected")
if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
log.Error(err, "error enabling bgp for project")
return err
}
clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
Host: ip.To4().String(),
Port: 6443,
}
case err != nil:
log.Error(err, "error getting cluster IP")
return err
default:
// If there is an ElasticIP with the right tag just use it again
clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
Host: ipReserv.GetAddress(),
Port: 6443,
}
}

if clusterScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" {
if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
log.Error(err, "error enabling bgp for project")
if packetCluster.Spec.VIPManager != emlb.EMLBVIPID {
ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
switch {
case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
// Parse metro and facility from the cluster spec
var metro, facility string

facility = packetCluster.Spec.Facility
metro = packetCluster.Spec.Metro

// If both specified, metro takes precedence over facility
if metro != "" {
facility = ""
}

// There is not an ElasticIP with the right tags, at this point we can create one
ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
if err != nil {
log.Error(err, "error reserving an ip")
return err
}
packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
Host: ip.To4().String(),
Port: 6443,
}
case err != nil:
log.Error(err, "error getting cluster IP")
return err
default:
// If there is an ElasticIP with the right tag just use it again
packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
Host: ipReserv.GetAddress(),
Port: 6443,
}
}
}

clusterScope.PacketCluster.Status.Ready = true
packetCluster.Status.Ready = true
conditions.MarkTrue(packetCluster, infrav1.NetworkInfrastructureReadyCondition)

return nil
39 changes: 30 additions & 9 deletions controllers/packetmachine_controller.go
Original file line number Diff line number Diff line change
@@ -43,6 +43,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"

infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
clog "sigs.k8s.io/cluster-api/util/log"
@@ -348,23 +349,33 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
// when a node is a control plane node we need the elastic IP
// to template out the kube-vip deployment
if machineScope.IsControlPlane() {
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
ctx,
machineScope.Cluster.Namespace,
machineScope.Cluster.Name,
machineScope.PacketCluster.Spec.ProjectID)
if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
var controlPlaneEndpointAddress string
var cpemLBConfig string
var emlbID string
switch {
case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
ctx,
machineScope.Cluster.Namespace,
machineScope.Cluster.Name,
machineScope.PacketCluster.Spec.ProjectID)
if len(controlPlaneEndpoint.Assignments) == 0 {
a := corev1.NodeAddress{
Type: corev1.NodeExternalIP,
Address: controlPlaneEndpoint.GetAddress(),
}
addrs = append(addrs, a)
}
controlPlaneEndpointAddress = controlPlaneEndpoint.GetAddress()
case machineScope.PacketCluster.Spec.VIPManager == emlb.EMLBVIPID:
controlPlaneEndpointAddress = machineScope.Cluster.Spec.ControlPlaneEndpoint.Host
cpemLBConfig = "emlb:///" + machineScope.PacketCluster.Spec.Metro
emlbID = machineScope.PacketCluster.Annotations["equinix.com/loadbalancerID"]
}
createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpoint.GetAddress()
createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpointAddress
createDeviceReq.CPEMLBConfig = cpemLBConfig
createDeviceReq.EMLBID = emlbID
}

dev, err = r.PacketClient.NewDevice(ctx, createDeviceReq)

switch {
@@ -413,7 +424,8 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
case infrav1.PacketResourceStatusRunning:
log.Info("Machine instance is active", "instance-id", machineScope.ProviderID())

if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
switch {
case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
ctx,
machineScope.Cluster.Namespace,
@@ -428,6 +440,15 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
return ctrl.Result{RequeueAfter: time.Second * 20}, nil
}
}
case machineScope.PacketCluster.Spec.VIPManager == emlb.EMLBVIPID:
if machineScope.IsControlPlane() {
// Create new EMLB object
lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, machineScope.PacketCluster.Spec.Metro)

if err := lb.ReconcileVIPOrigin(ctx, machineScope, deviceAddr); err != nil {
return ctrl.Result{}, err
}
}
}

machineScope.SetReady()
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -10,6 +10,8 @@ require (
github.com/pkg/errors v0.9.1
github.com/spf13/cobra v1.8.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.9.0
golang.org/x/oauth2 v0.18.0
k8s.io/api v0.29.3
k8s.io/apimachinery v0.29.3
k8s.io/client-go v0.29.3
@@ -57,6 +59,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.18.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
@@ -73,7 +76,6 @@ require (
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
Loading