Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[NET-6938] Create workloads in Consul for mesh gateway pods #3382

Merged
merged 5 commits into from
Dec 18, 2023
Merged

[NET-6938] Create workloads in Consul for mesh gateway pods #3382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e3eb693
Create workload in Consul for gateway pods
nathancoleman Dec 15, 2023
32dacd1
Log when pod controller creates workload in Consul
nathancoleman Dec 15, 2023
e9b547b
Disable t-proxy probe overwrite for mesh gateway pods
nathancoleman Dec 15, 2023
66739e7
Update test assertions
nathancoleman Dec 15, 2023
e518860
Add test case for gateway pod reconciliation
nathancoleman Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions control-plane/connect-inject/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,16 @@ func HasBeenMeshInjected(pod corev1.Pod) bool {
return false
}

func IsGateway(pod corev1.Pod) bool {
if pod.Annotations == nil {
return false
}
if anno, ok := pod.Annotations[constants.AnnotationGatewayKind]; ok && anno != "" {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This utilizes a known annotation added to all v2 mesh gateway (and api/terminating in the future) pods

consul-k8s/control-plane/gateways/deployment.go

Line 65 in c8e5d9e

constants.AnnotationGatewayKind: meshGatewayAnnotationKind,

return true
}
return false
}

// ConsulNamespaceIsNotFound checks the gRPC error code and message to determine
// if a namespace does not exist. If the namespace exists this function returns false, true otherwise.
func ConsulNamespaceIsNotFound(err error) bool {
Expand Down
14 changes: 12 additions & 2 deletions control-plane/connect-inject/controllers/pod/pod_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu

r.Log.Info("retrieved", "name", pod.Name, "ns", pod.Namespace)

if inject.HasBeenMeshInjected(pod) {
if inject.HasBeenMeshInjected(pod) || inject.IsGateway(pod) {

// It is possible the pod was scheduled but doesn't have an allocated IP yet, so safely requeue
if pod.Status.PodIP == "" {
Expand Down Expand Up @@ -336,9 +336,11 @@ func (r *Controller) writeWorkload(ctx context.Context, pod corev1.Pod) error {
}
data := inject.ToProtoAny(workload)

resourceID := getWorkloadID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition())
r.Log.Info("registering workload with Consul", getLogFieldsForResource(resourceID)...)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This adds helpful logging analogous to what we do elsewhere

consul-k8s/control-plane/connect-inject/controllers/serviceaccount/serviceaccount_controller.go

Line 112 in 2b0ef6a

r.Log.Info("registering workload identity with Consul", getLogFieldsForResource(workloadIdentityResource.Id)...)

consul-k8s/control-plane/connect-inject/controllers/endpointsv2/endpoints_controller.go

Line 336 in 2b0ef6a

r.Log.Info("writing service to Consul", getLogFieldsForResource(consulSvcResource.Id)...)

req := &pbresource.WriteRequest{
Resource: &pbresource.Resource{
Id: getWorkloadID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()),
Id: resourceID,
Metadata: metaFromPod(pod),
Data: data,
},
Expand Down Expand Up @@ -762,3 +764,11 @@ func getDestinationsID(name, namespace, partition string) *pbresource.ID {
},
}
}

func getLogFieldsForResource(id *pbresource.ID) []any {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be helpful to move this out into a commonly-used package at some point but didn't do this now

return []any{
"name", id.Name,
"ns", id.Tenancy.Namespace,
"partition", id.Tenancy.Partition,
}
}
28 changes: 14 additions & 14 deletions control-plane/connect-inject/controllers/pod/pod_controller_ent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func TestReconcileCreatePodWithMirrorNamespaces(t *testing.T) {
expectedConsulNamespace: constants.DefaultConsulNS,
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
{
name: "kitchen sink new pod, non-default ns and partition",
Expand All @@ -127,7 +127,7 @@ func TestReconcileCreatePodWithMirrorNamespaces(t *testing.T) {
expectedConsulNamespace: "bar",
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
{
name: "new pod with namespace prefix",
Expand Down Expand Up @@ -194,7 +194,7 @@ func TestReconcileCreatePodWithMirrorNamespaces(t *testing.T) {
expectedConsulNamespace: constants.DefaultConsulNS,
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
expectedDestinations: createDestinations(),
},
{
Expand Down Expand Up @@ -273,12 +273,12 @@ func TestReconcileUpdatePodWithMirrorNamespaces(t *testing.T) {
existingConsulNamespace: "bar",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),

expectedConsulNamespace: "bar",
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
}

Expand Down Expand Up @@ -311,7 +311,7 @@ func TestReconcileDeletePodWithMirrorNamespaces(t *testing.T) {
existingConsulNamespace: "bar",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),

expectedConsulNamespace: "bar",
},
Expand All @@ -330,7 +330,7 @@ func TestReconcileDeletePodWithMirrorNamespaces(t *testing.T) {
existingConsulNamespace: "bar",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
existingDestinations: createDestinations(),

expectedConsulNamespace: "bar",
Expand Down Expand Up @@ -415,7 +415,7 @@ func TestReconcileCreatePodWithDestinationNamespace(t *testing.T) {
expectedConsulNamespace: constants.DefaultConsulNS,
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
{
name: "new pod with explicit destinations, ns and partition",
Expand All @@ -439,7 +439,7 @@ func TestReconcileCreatePodWithDestinationNamespace(t *testing.T) {
expectedConsulNamespace: constants.DefaultConsulNS,
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
expectedDestinations: createDestinations(),
},
{
Expand All @@ -466,7 +466,7 @@ func TestReconcileCreatePodWithDestinationNamespace(t *testing.T) {
expectedConsulNamespace: "a-penguin-walks-into-a-bar",
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
{
name: "namespace in Consul does not exist",
Expand Down Expand Up @@ -543,12 +543,12 @@ func TestReconcileUpdatePodWithDestinationNamespace(t *testing.T) {
existingConsulNamespace: "a-penguin-walks-into-a-bar",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),

expectedConsulNamespace: "a-penguin-walks-into-a-bar",
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
}

Expand Down Expand Up @@ -581,7 +581,7 @@ func TestReconcileDeletePodWithDestinationNamespace(t *testing.T) {
existingConsulNamespace: "a-penguin-walks-into-a-bar",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),

expectedConsulNamespace: "a-penguin-walks-into-a-bar",
},
Expand All @@ -600,7 +600,7 @@ func TestReconcileDeletePodWithDestinationNamespace(t *testing.T) {
existingConsulNamespace: "a-penguin-walks-into-a-bar",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
existingDestinations: createDestinations(),

expectedConsulNamespace: "a-penguin-walks-into-a-bar",
Expand Down
79 changes: 51 additions & 28 deletions control-plane/connect-inject/controllers/pod/pod_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -785,7 +785,7 @@ func TestProxyConfigurationDelete(t *testing.T) {
{
name: "proxy configuration delete",
pod: createPod("foo", "", true, true),
existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
}

Expand Down Expand Up @@ -1308,7 +1308,7 @@ func TestReconcileCreatePod(t *testing.T) {

testCases := []testCase{
{
name: "vanilla new pod",
name: "vanilla new mesh-injected pod",
podName: "foo",
k8sObjects: func() []runtime.Object {
pod := createPod("foo", "", true, true)
Expand All @@ -1322,7 +1322,26 @@ func TestReconcileCreatePod(t *testing.T) {
overwriteProbes: true,
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
expectedProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
{
name: "vanilla new gateway pod (not mesh-injected)",
podName: "foo",
k8sObjects: func() []runtime.Object {
pod := createPod("foo", "", false, true)
pod.Annotations[constants.AnnotationGatewayKind] = "mesh-gateway"
pod.Annotations[constants.AnnotationMeshInject] = "false"
pod.Annotations[constants.AnnotationTransparentProxyOverwriteProbes] = "false"

return []runtime.Object{pod}
},
tproxy: true,
telemetry: true,
metrics: true,
overwriteProbes: true,
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration("foo", false, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
{
name: "pod in ignored namespace",
Expand Down Expand Up @@ -1380,7 +1399,7 @@ func TestReconcileCreatePod(t *testing.T) {
overwriteProbes: true,
expectedWorkload: createWorkload(),
expectedHealthStatus: createPassingHealthStatus(),
expectedProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
expectedProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
expectedDestinations: createDestinations(),
},
{
Expand Down Expand Up @@ -1827,22 +1846,22 @@ func TestReconcileDeletePod(t *testing.T) {
podName: "foo",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
},
{
name: "annotated delete pod",
podName: "foo",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT),
existingDestinations: createDestinations(),
},
{
name: "delete pod w/ acls",
podName: "foo",
existingWorkload: createWorkload(),
existingHealthStatus: createPassingHealthStatus(),
existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT),
aclsEnabled: true,
},
}
Expand Down Expand Up @@ -1989,39 +2008,43 @@ func createCriticalHealthStatus(name string, namespace string) *pbcatalog.Health

// createProxyConfiguration creates a proxyConfiguration that matches the pod from createPod,
// assuming that metrics, telemetry, and overwrite probes are enabled separately.
func createProxyConfiguration(podName string, mode pbmesh.ProxyMode) *pbmesh.ProxyConfiguration {
func createProxyConfiguration(podName string, overwriteProbes bool, mode pbmesh.ProxyMode) *pbmesh.ProxyConfiguration {
mesh := &pbmesh.ProxyConfiguration{
Workloads: &pbcatalog.WorkloadSelector{
Names: []string{podName},
},
DynamicConfig: &pbmesh.DynamicConfig{
Mode: mode,
ExposeConfig: &pbmesh.ExposeConfig{
ExposePaths: []*pbmesh.ExposePath{
{
ListenerPort: 20400,
LocalPathPort: 2001,
Path: "/livez",
},
{
ListenerPort: 20300,
LocalPathPort: 2000,
Path: "/readyz",
},
{
ListenerPort: 20500,
LocalPathPort: 2002,
Path: "/startupz",
},
},
},
Mode: mode,
ExposeConfig: nil,
},
BootstrapConfig: &pbmesh.BootstrapConfig{
PrometheusBindAddr: "0.0.0.0:1234",
TelemetryCollectorBindSocketDir: DefaultTelemetryBindSocketDir,
},
}

if overwriteProbes {
mesh.DynamicConfig.ExposeConfig = &pbmesh.ExposeConfig{
ExposePaths: []*pbmesh.ExposePath{
{
ListenerPort: 20400,
LocalPathPort: 2001,
Path: "/livez",
},
{
ListenerPort: 20300,
LocalPathPort: 2000,
Path: "/readyz",
},
{
ListenerPort: 20500,
LocalPathPort: 2002,
Path: "/startupz",
},
},
}
}

if mode == pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT {
mesh.DynamicConfig.TransparentProxy = &pbmesh.TransparentProxy{
OutboundListenerPort: 15001,
Expand Down
7 changes: 6 additions & 1 deletion control-plane/gateways/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,13 @@ func (b *meshGatewayBuilder) deploymentSpec() (*appsv1.DeploymentSpec, error) {
ObjectMeta: metav1.ObjectMeta{
Labels: b.Labels(),
Annotations: map[string]string{
constants.AnnotationMeshInject: "false",
// Indicate that this pod is a mesh gateway pod so that the Pod controller,
// consul-k8s CLI, etc. can key off of it
constants.AnnotationGatewayKind: meshGatewayAnnotationKind,
// It's not logical to add a proxy sidecar since our workload is itself a proxy
constants.AnnotationMeshInject: "false",
// This functionality only applies when proxy sidecars are used
constants.AnnotationTransparentProxyOverwriteProbes: "false",
},
},
Spec: corev1.PodSpec{
Expand Down
10 changes: 6 additions & 4 deletions control-plane/gateways/deployment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,9 @@ func Test_meshGatewayBuilder_Deployment(t *testing.T) {
"mesh.consul.hashicorp.com/managed-by": "consul-k8s",
},
Annotations: map[string]string{
constants.AnnotationMeshInject: "false",
constants.AnnotationGatewayKind: meshGatewayAnnotationKind,
constants.AnnotationGatewayKind: meshGatewayAnnotationKind,
constants.AnnotationMeshInject: "false",
constants.AnnotationTransparentProxyOverwriteProbes: "false",
},
},
Spec: corev1.PodSpec{
Expand Down Expand Up @@ -323,8 +324,9 @@ func Test_meshGatewayBuilder_Deployment(t *testing.T) {
"mesh.consul.hashicorp.com/managed-by": "consul-k8s",
},
Annotations: map[string]string{
constants.AnnotationMeshInject: "false",
constants.AnnotationGatewayKind: meshGatewayAnnotationKind,
constants.AnnotationGatewayKind: meshGatewayAnnotationKind,
constants.AnnotationMeshInject: "false",
constants.AnnotationTransparentProxyOverwriteProbes: "false",
},
},
Spec: corev1.PodSpec{
Expand Down