From e7980442ef7dde8afddc66e8d576b4ac81a2ff78 Mon Sep 17 00:00:00 2001 From: dongjiang Date: Sun, 7 Jan 2024 00:50:32 +0800 Subject: [PATCH] `PodLifeTime`: update support pods with container status and pods reason (#1330) * update support podlifetime status Signed-off-by: dongjiang1989 * update verify gen Signed-off-by: dongjiang1989 --------- Signed-off-by: dongjiang1989 --- README.md | 5 +- .../plugins/podlifetime/pod_lifetime.go | 7 + .../plugins/podlifetime/pod_lifetime_test.go | 166 ++++++++++++++++++ .../plugins/podlifetime/validation.go | 16 +- .../plugins/podlifetime/validation_test.go | 8 + 5 files changed, 199 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 08c252343c..f172236e00 100644 --- a/README.md +++ b/README.md @@ -637,8 +637,9 @@ profiles: This strategy evicts pods that are older than `maxPodLifeTimeSeconds`. You can also specify `states` parameter to **only** evict pods matching the following conditions: - - [Pod Phase](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) status of: `Running`, `Pending` - - [Container State Waiting](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-state-waiting) condition of: `PodInitializing`, `ContainerCreating`, `ImagePullBackOff` + - [Pod Phase](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) status of: `Running`, `Pending`, `Unknown` + - [Pod Reason](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) reasons of: `NodeAffinity`, `NodeLost`, `Shutdown`, `UnexpectedAdmissionError` + - [Container State Waiting](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-state-waiting) condition of: `PodInitializing`, `ContainerCreating`, `ImagePullBackOff`, `CrashLoopBackOff`, `CreateContainerConfigError`, `ErrImagePull`, `ImagePullBackOff`, `CreateContainerError`, `InvalidImageName` If a value for `states` or `podStatusPhases` is not specified, Pods in any state (even `Running`) are considered for eviction. diff --git a/pkg/framework/plugins/podlifetime/pod_lifetime.go b/pkg/framework/plugins/podlifetime/pod_lifetime.go index d4d6103e70..388d131025 100644 --- a/pkg/framework/plugins/podlifetime/pod_lifetime.go +++ b/pkg/framework/plugins/podlifetime/pod_lifetime.go @@ -75,10 +75,17 @@ func New(args runtime.Object, handle frameworktypes.Handle) (frameworktypes.Plug if len(podLifeTimeArgs.States) > 0 { states := sets.New(podLifeTimeArgs.States...) podFilter = podutil.WrapFilterFuncs(podFilter, func(pod *v1.Pod) bool { + // Pod Status Phase if states.Has(string(pod.Status.Phase)) { return true } + // Pod Status Reason + if states.Has(pod.Status.Reason) { + return true + } + + // Container Status Reason for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.State.Waiting != nil && states.Has(containerStatus.State.Waiting.Reason) { return true diff --git a/pkg/framework/plugins/podlifetime/pod_lifetime_test.go b/pkg/framework/plugins/podlifetime/pod_lifetime_test.go index 44d2d4484c..16536cbc32 100644 --- a/pkg/framework/plugins/podlifetime/pod_lifetime_test.go +++ b/pkg/framework/plugins/podlifetime/pod_lifetime_test.go @@ -141,6 +141,12 @@ func TestPodLifeTime(t *testing.T) { p14.DeletionTimestamp = &metav1.Time{} p15.DeletionTimestamp = &metav1.Time{} + p16 := test.BuildTestPod("p16", 100, 0, node1.Name, nil) + p16.Namespace = "dev" + p16.ObjectMeta.CreationTimestamp = olderPodCreationTime + p16.Status.Phase = v1.PodUnknown + p16.ObjectMeta.OwnerReferences = ownerRef1 + var maxLifeTime uint = 600 testCases := []struct { description string @@ -338,6 +344,166 @@ func TestPodLifeTime(t *testing.T) { } }, }, + { + description: "1 pod with container status CrashLoopBackOff should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"CrashLoopBackOff"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.ContainerStatuses = []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}, + }, + }, + } + }, + }, + { + description: "1 pod with container status CreateContainerConfigError should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"CreateContainerConfigError"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.ContainerStatuses = []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{Reason: "CreateContainerConfigError"}, + }, + }, + } + }, + }, + { + description: "1 pod with container status ErrImagePull should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"ErrImagePull"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.ContainerStatuses = []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{Reason: "ErrImagePull"}, + }, + }, + } + }, + }, + { + description: "1 pod with container status CreateContainerError should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"CreateContainerError"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.ContainerStatuses = []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{Reason: "CreateContainerError"}, + }, + }, + } + }, + }, + { + description: "1 pod with container status InvalidImageName should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"InvalidImageName"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.ContainerStatuses = []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{Reason: "InvalidImageName"}, + }, + }, + } + }, + }, + { + description: "1 pod with pod status reason NodeLost should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"NodeLost"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.Reason = "NodeLost" + }, + }, + { + description: "1 pod with pod status reason NodeAffinity should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"NodeAffinity"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.Reason = "NodeAffinity" + }, + }, + { + description: "1 pod with pod status reason Shutdown should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"Shutdown"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.Reason = "Shutdown" + }, + }, + { + description: "1 pod with pod status reason UnexpectedAdmissionError should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{"UnexpectedAdmissionError"}, + }, + pods: []*v1.Pod{p9}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.Reason = "UnexpectedAdmissionError" + }, + }, + { + description: "1 pod with pod status phase v1.PodUnknown should be evicted", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + States: []string{string(v1.PodUnknown)}, + }, + pods: []*v1.Pod{p16}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + applyPodsFunc: func(pods []*v1.Pod) { + pods[0].Status.Phase = v1.PodUnknown + }, + }, { description: "1 pod without ImagePullBackOff States should be ignored", args: &PodLifeTimeArgs{ diff --git a/pkg/framework/plugins/podlifetime/validation.go b/pkg/framework/plugins/podlifetime/validation.go index b48368a357..cd97750183 100644 --- a/pkg/framework/plugins/podlifetime/validation.go +++ b/pkg/framework/plugins/podlifetime/validation.go @@ -44,15 +44,29 @@ func ValidatePodLifeTimeArgs(obj runtime.Object) error { } } podLifeTimeAllowedStates := sets.New( + // Pod Status Phase string(v1.PodRunning), string(v1.PodPending), + string(v1.PodUnknown), + // Pod Status Reasons + "NodeAffinity", + "NodeLost", + "Shutdown", + "UnexpectedAdmissionError", + + // Container Status Reasons // Container state reasons: https://github.com/kubernetes/kubernetes/blob/release-1.24/pkg/kubelet/kubelet_pods.go#L76-L79 "PodInitializing", "ContainerCreating", - // containerStatuses[*].state.waiting.reason: ImagePullBackOff + // containerStatuses[*].state.waiting.reason: ImagePullBackOff, etc. "ImagePullBackOff", + "CrashLoopBackOff", + "CreateContainerConfigError", + "ErrImagePull", + "CreateContainerError", + "InvalidImageName", ) if !podLifeTimeAllowedStates.HasAll(args.States...) { diff --git a/pkg/framework/plugins/podlifetime/validation_test.go b/pkg/framework/plugins/podlifetime/validation_test.go index b427974272..7caf85d1c2 100644 --- a/pkg/framework/plugins/podlifetime/validation_test.go +++ b/pkg/framework/plugins/podlifetime/validation_test.go @@ -36,6 +36,14 @@ func TestValidateRemovePodLifeTimeArgs(t *testing.T) { }, expectError: false, }, + { + description: "Pod Status Reasons CrashLoopBackOff ", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: func(i uint) *uint { return &i }(1), + States: []string{"CrashLoopBackOff"}, + }, + expectError: false, + }, { description: "nil MaxPodLifeTimeSeconds arg, expects errors", args: &PodLifeTimeArgs{