diff --git a/control-plane/connect-inject/common/common.go b/control-plane/connect-inject/common/common.go index 99372f7aec..9bbaeaab58 100644 --- a/control-plane/connect-inject/common/common.go +++ b/control-plane/connect-inject/common/common.go @@ -192,6 +192,16 @@ func HasBeenMeshInjected(pod corev1.Pod) bool { return false } +func IsGateway(pod corev1.Pod) bool { + if pod.Annotations == nil { + return false + } + if anno, ok := pod.Annotations[constants.AnnotationGatewayKind]; ok && anno != "" { + return true + } + return false +} + // ConsulNamespaceIsNotFound checks the gRPC error code and message to determine // if a namespace does not exist. If the namespace exists this function returns false, true otherwise. func ConsulNamespaceIsNotFound(err error) bool { diff --git a/control-plane/connect-inject/controllers/pod/pod_controller.go b/control-plane/connect-inject/controllers/pod/pod_controller.go index 8e33a42af3..f0d319a475 100644 --- a/control-plane/connect-inject/controllers/pod/pod_controller.go +++ b/control-plane/connect-inject/controllers/pod/pod_controller.go @@ -150,7 +150,7 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu r.Log.Info("retrieved", "name", pod.Name, "ns", pod.Namespace) - if inject.HasBeenMeshInjected(pod) { + if inject.HasBeenMeshInjected(pod) || inject.IsGateway(pod) { // It is possible the pod was scheduled but doesn't have an allocated IP yet, so safely requeue if pod.Status.PodIP == "" { @@ -336,9 +336,11 @@ func (r *Controller) writeWorkload(ctx context.Context, pod corev1.Pod) error { } data := inject.ToProtoAny(workload) + resourceID := getWorkloadID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()) + r.Log.Info("registering workload with Consul", getLogFieldsForResource(resourceID)...) req := &pbresource.WriteRequest{ Resource: &pbresource.Resource{ - Id: getWorkloadID(pod.GetName(), r.getConsulNamespace(pod.Namespace), r.getPartition()), + Id: resourceID, Metadata: metaFromPod(pod), Data: data, }, @@ -762,3 +764,11 @@ func getDestinationsID(name, namespace, partition string) *pbresource.ID { }, } } + +func getLogFieldsForResource(id *pbresource.ID) []any { + return []any{ + "name", id.Name, + "ns", id.Tenancy.Namespace, + "partition", id.Tenancy.Partition, + } +} diff --git a/control-plane/connect-inject/controllers/pod/pod_controller_ent_test.go b/control-plane/connect-inject/controllers/pod/pod_controller_ent_test.go index 1c074dc940..a116122a03 100644 --- a/control-plane/connect-inject/controllers/pod/pod_controller_ent_test.go +++ b/control-plane/connect-inject/controllers/pod/pod_controller_ent_test.go @@ -101,7 +101,7 @@ func TestReconcileCreatePodWithMirrorNamespaces(t *testing.T) { expectedConsulNamespace: constants.DefaultConsulNS, expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, { name: "kitchen sink new pod, non-default ns and partition", @@ -127,7 +127,7 @@ func TestReconcileCreatePodWithMirrorNamespaces(t *testing.T) { expectedConsulNamespace: "bar", expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, { name: "new pod with namespace prefix", @@ -194,7 +194,7 @@ func TestReconcileCreatePodWithMirrorNamespaces(t *testing.T) { expectedConsulNamespace: constants.DefaultConsulNS, expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), expectedDestinations: createDestinations(), }, { @@ -273,12 +273,12 @@ func TestReconcileUpdatePodWithMirrorNamespaces(t *testing.T) { existingConsulNamespace: "bar", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), expectedConsulNamespace: "bar", expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, } @@ -311,7 +311,7 @@ func TestReconcileDeletePodWithMirrorNamespaces(t *testing.T) { existingConsulNamespace: "bar", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), expectedConsulNamespace: "bar", }, @@ -330,7 +330,7 @@ func TestReconcileDeletePodWithMirrorNamespaces(t *testing.T) { existingConsulNamespace: "bar", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), + existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), existingDestinations: createDestinations(), expectedConsulNamespace: "bar", @@ -415,7 +415,7 @@ func TestReconcileCreatePodWithDestinationNamespace(t *testing.T) { expectedConsulNamespace: constants.DefaultConsulNS, expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, { name: "new pod with explicit destinations, ns and partition", @@ -439,7 +439,7 @@ func TestReconcileCreatePodWithDestinationNamespace(t *testing.T) { expectedConsulNamespace: constants.DefaultConsulNS, expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), expectedDestinations: createDestinations(), }, { @@ -466,7 +466,7 @@ func TestReconcileCreatePodWithDestinationNamespace(t *testing.T) { expectedConsulNamespace: "a-penguin-walks-into-a-bar", expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, { name: "namespace in Consul does not exist", @@ -543,12 +543,12 @@ func TestReconcileUpdatePodWithDestinationNamespace(t *testing.T) { existingConsulNamespace: "a-penguin-walks-into-a-bar", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), expectedConsulNamespace: "a-penguin-walks-into-a-bar", expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + expectedProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, } @@ -581,7 +581,7 @@ func TestReconcileDeletePodWithDestinationNamespace(t *testing.T) { existingConsulNamespace: "a-penguin-walks-into-a-bar", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), expectedConsulNamespace: "a-penguin-walks-into-a-bar", }, @@ -600,7 +600,7 @@ func TestReconcileDeletePodWithDestinationNamespace(t *testing.T) { existingConsulNamespace: "a-penguin-walks-into-a-bar", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration(testPodName, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), + existingProxyConfiguration: createProxyConfiguration(testPodName, true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), existingDestinations: createDestinations(), expectedConsulNamespace: "a-penguin-walks-into-a-bar", diff --git a/control-plane/connect-inject/controllers/pod/pod_controller_test.go b/control-plane/connect-inject/controllers/pod/pod_controller_test.go index 413f9ac49a..55b5fd5ba1 100644 --- a/control-plane/connect-inject/controllers/pod/pod_controller_test.go +++ b/control-plane/connect-inject/controllers/pod/pod_controller_test.go @@ -785,7 +785,7 @@ func TestProxyConfigurationDelete(t *testing.T) { { name: "proxy configuration delete", pod: createPod("foo", "", true, true), - existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, } @@ -1308,7 +1308,7 @@ func TestReconcileCreatePod(t *testing.T) { testCases := []testCase{ { - name: "vanilla new pod", + name: "vanilla new mesh-injected pod", podName: "foo", k8sObjects: func() []runtime.Object { pod := createPod("foo", "", true, true) @@ -1322,7 +1322,26 @@ func TestReconcileCreatePod(t *testing.T) { overwriteProbes: true, expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + expectedProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + }, + { + name: "vanilla new gateway pod (not mesh-injected)", + podName: "foo", + k8sObjects: func() []runtime.Object { + pod := createPod("foo", "", false, true) + pod.Annotations[constants.AnnotationGatewayKind] = "mesh-gateway" + pod.Annotations[constants.AnnotationMeshInject] = "false" + pod.Annotations[constants.AnnotationTransparentProxyOverwriteProbes] = "false" + + return []runtime.Object{pod} + }, + tproxy: true, + telemetry: true, + metrics: true, + overwriteProbes: true, + expectedWorkload: createWorkload(), + expectedHealthStatus: createPassingHealthStatus(), + expectedProxyConfiguration: createProxyConfiguration("foo", false, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, { name: "pod in ignored namespace", @@ -1380,7 +1399,7 @@ func TestReconcileCreatePod(t *testing.T) { overwriteProbes: true, expectedWorkload: createWorkload(), expectedHealthStatus: createPassingHealthStatus(), - expectedProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_DEFAULT), + expectedProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), expectedDestinations: createDestinations(), }, { @@ -1827,14 +1846,14 @@ func TestReconcileDeletePod(t *testing.T) { podName: "foo", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), }, { name: "annotated delete pod", podName: "foo", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_DEFAULT), + existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_DEFAULT), existingDestinations: createDestinations(), }, { @@ -1842,7 +1861,7 @@ func TestReconcileDeletePod(t *testing.T) { podName: "foo", existingWorkload: createWorkload(), existingHealthStatus: createPassingHealthStatus(), - existingProxyConfiguration: createProxyConfiguration("foo", pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), + existingProxyConfiguration: createProxyConfiguration("foo", true, pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT), aclsEnabled: true, }, } @@ -1989,32 +2008,14 @@ func createCriticalHealthStatus(name string, namespace string) *pbcatalog.Health // createProxyConfiguration creates a proxyConfiguration that matches the pod from createPod, // assuming that metrics, telemetry, and overwrite probes are enabled separately. -func createProxyConfiguration(podName string, mode pbmesh.ProxyMode) *pbmesh.ProxyConfiguration { +func createProxyConfiguration(podName string, overwriteProbes bool, mode pbmesh.ProxyMode) *pbmesh.ProxyConfiguration { mesh := &pbmesh.ProxyConfiguration{ Workloads: &pbcatalog.WorkloadSelector{ Names: []string{podName}, }, DynamicConfig: &pbmesh.DynamicConfig{ - Mode: mode, - ExposeConfig: &pbmesh.ExposeConfig{ - ExposePaths: []*pbmesh.ExposePath{ - { - ListenerPort: 20400, - LocalPathPort: 2001, - Path: "/livez", - }, - { - ListenerPort: 20300, - LocalPathPort: 2000, - Path: "/readyz", - }, - { - ListenerPort: 20500, - LocalPathPort: 2002, - Path: "/startupz", - }, - }, - }, + Mode: mode, + ExposeConfig: nil, }, BootstrapConfig: &pbmesh.BootstrapConfig{ PrometheusBindAddr: "0.0.0.0:1234", @@ -2022,6 +2023,28 @@ func createProxyConfiguration(podName string, mode pbmesh.ProxyMode) *pbmesh.Pro }, } + if overwriteProbes { + mesh.DynamicConfig.ExposeConfig = &pbmesh.ExposeConfig{ + ExposePaths: []*pbmesh.ExposePath{ + { + ListenerPort: 20400, + LocalPathPort: 2001, + Path: "/livez", + }, + { + ListenerPort: 20300, + LocalPathPort: 2000, + Path: "/readyz", + }, + { + ListenerPort: 20500, + LocalPathPort: 2002, + Path: "/startupz", + }, + }, + } + } + if mode == pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT { mesh.DynamicConfig.TransparentProxy = &pbmesh.TransparentProxy{ OutboundListenerPort: 15001, diff --git a/control-plane/gateways/deployment.go b/control-plane/gateways/deployment.go index a2ed759643..15339ec3a0 100644 --- a/control-plane/gateways/deployment.go +++ b/control-plane/gateways/deployment.go @@ -63,8 +63,13 @@ func (b *meshGatewayBuilder) deploymentSpec() (*appsv1.DeploymentSpec, error) { ObjectMeta: metav1.ObjectMeta{ Labels: b.Labels(), Annotations: map[string]string{ - constants.AnnotationMeshInject: "false", + // Indicate that this pod is a mesh gateway pod so that the Pod controller, + // consul-k8s CLI, etc. can key off of it constants.AnnotationGatewayKind: meshGatewayAnnotationKind, + // It's not logical to add a proxy sidecar since our workload is itself a proxy + constants.AnnotationMeshInject: "false", + // This functionality only applies when proxy sidecars are used + constants.AnnotationTransparentProxyOverwriteProbes: "false", }, }, Spec: corev1.PodSpec{ diff --git a/control-plane/gateways/deployment_test.go b/control-plane/gateways/deployment_test.go index 33a7aca980..829de5f05c 100644 --- a/control-plane/gateways/deployment_test.go +++ b/control-plane/gateways/deployment_test.go @@ -77,8 +77,9 @@ func Test_meshGatewayBuilder_Deployment(t *testing.T) { "mesh.consul.hashicorp.com/managed-by": "consul-k8s", }, Annotations: map[string]string{ - constants.AnnotationMeshInject: "false", - constants.AnnotationGatewayKind: meshGatewayAnnotationKind, + constants.AnnotationGatewayKind: meshGatewayAnnotationKind, + constants.AnnotationMeshInject: "false", + constants.AnnotationTransparentProxyOverwriteProbes: "false", }, }, Spec: corev1.PodSpec{ @@ -323,8 +324,9 @@ func Test_meshGatewayBuilder_Deployment(t *testing.T) { "mesh.consul.hashicorp.com/managed-by": "consul-k8s", }, Annotations: map[string]string{ - constants.AnnotationMeshInject: "false", - constants.AnnotationGatewayKind: meshGatewayAnnotationKind, + constants.AnnotationGatewayKind: meshGatewayAnnotationKind, + constants.AnnotationMeshInject: "false", + constants.AnnotationTransparentProxyOverwriteProbes: "false", }, }, Spec: corev1.PodSpec{