Skip to content

Commit

Permalink
feat : Add helper function to get all operator pods (#2130)
Browse files Browse the repository at this point in the history
* Add functions to get the csv managed pods in target namespaces

* Refined the logic to retrieve the pods based on owner references

* Refactoring

* Change the result format to map

* Address review comments
  • Loading branch information
bnshr authored Jun 18, 2024
1 parent 4044829 commit 50389f5
Show file tree
Hide file tree
Showing 11 changed files with 302 additions and 181 deletions.
3 changes: 2 additions & 1 deletion cnf-certification-test/accesscontrol/suite.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/test-network-function/cnf-certification-test/internal/crclient"
"github.com/test-network-function/cnf-certification-test/internal/log"
"github.com/test-network-function/cnf-certification-test/pkg/checksdb"
"github.com/test-network-function/cnf-certification-test/pkg/podhelper"
"github.com/test-network-function/cnf-certification-test/pkg/provider"
"github.com/test-network-function/cnf-certification-test/pkg/stringhelper"
"github.com/test-network-function/cnf-certification-test/pkg/testhelper"
Expand Down Expand Up @@ -685,7 +686,7 @@ func isInstallModeMultiNamespace(installModes []v1alpha1.InstallMode) bool {
// - bool: true if one of the passed topOwners is a CSV that is installed by a cluster-wide operator, otherwise return false
// - name string : the name of the matching object, if found.
// - aNamespace string : the namespace of the matching object, if found.
func ownedByClusterWideOperator(topOwners map[string]provider.TopOwner, env *provider.TestEnvironment) (aNamespace, name string, found bool) {
func ownedByClusterWideOperator(topOwners map[string]podhelper.TopOwner, env *provider.TestEnvironment) (aNamespace, name string, found bool) {
for _, owner := range topOwners {
if isCSVAndClusterWide(owner.Namespace, owner.Name, env) {
return owner.Namespace, owner.Name, true
Expand Down
30 changes: 15 additions & 15 deletions cnf-certification-test/lifecycle/podsets/podsets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,33 +23,33 @@ import (

"github.com/stretchr/testify/assert"
"github.com/test-network-function/cnf-certification-test/pkg/provider"
v1app "k8s.io/api/apps/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestIsDeploymentReady(t *testing.T) {
type dpStatus struct {
condition v1app.DeploymentConditionType
condition appsv1.DeploymentConditionType
replicas int32
ready int32
available int32
unavailable int32
updated int32
}
m := map[dpStatus]bool{
{v1app.DeploymentReplicaFailure, 10, 9, 10, 0, 0}: false,
{v1app.DeploymentAvailable, 10, 9, 9, 0, 10}: false,
{v1app.DeploymentAvailable, 10, 10, 10, 1, 10}: false,
{v1app.DeploymentAvailable, 10, 1, 10, 0, 10}: false,
{v1app.DeploymentAvailable, 10, 10, 10, 0, 9}: false,
{v1app.DeploymentAvailable, 10, 10, 10, 0, 10}: true,
{appsv1.DeploymentReplicaFailure, 10, 9, 10, 0, 0}: false,
{appsv1.DeploymentAvailable, 10, 9, 9, 0, 10}: false,
{appsv1.DeploymentAvailable, 10, 10, 10, 1, 10}: false,
{appsv1.DeploymentAvailable, 10, 1, 10, 0, 10}: false,
{appsv1.DeploymentAvailable, 10, 10, 10, 0, 9}: false,
{appsv1.DeploymentAvailable, 10, 10, 10, 0, 10}: true,
}
for key, v := range m {
dp := provider.Deployment{
Deployment: &v1app.Deployment{
Status: v1app.DeploymentStatus{
Conditions: []v1app.DeploymentCondition{
Deployment: &appsv1.Deployment{
Status: appsv1.DeploymentStatus{
Conditions: []appsv1.DeploymentCondition{
{
Type: key.condition,
},
Expand All @@ -59,7 +59,7 @@ func TestIsDeploymentReady(t *testing.T) {
UnavailableReplicas: key.unavailable,
UpdatedReplicas: key.updated,
},
Spec: v1app.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Replicas: &key.replicas,
},
},
Expand Down Expand Up @@ -87,11 +87,11 @@ func TestIsStatefulSetReady(t *testing.T) {
}
for k, v := range m {
ss := provider.StatefulSet{
StatefulSet: &v1app.StatefulSet{
Spec: v1app.StatefulSetSpec{
StatefulSet: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Replicas: &k.replicas,
},
Status: v1app.StatefulSetStatus{
Status: appsv1.StatefulSetStatus{
ReadyReplicas: k.ready,
AvailableReplicas: k.available,
UpdatedReplicas: k.updated,
Expand Down
22 changes: 11 additions & 11 deletions cnf-certification-test/lifecycle/scaling/deployment_scaling_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"github.com/test-network-function/cnf-certification-test/internal/clientsholder"
"github.com/test-network-function/cnf-certification-test/internal/log"
"github.com/test-network-function/cnf-certification-test/pkg/provider"
v1app "k8s.io/api/apps/v1"
appsv1 "k8s.io/api/apps/v1"
v1autoscaling "k8s.io/api/autoscaling/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -37,13 +37,13 @@ import (
)

func TestScaleDeploymentFunc(t *testing.T) {
generateDeployment := func(name string, replicas *int32) *v1app.Deployment {
return &v1app.Deployment{
generateDeployment := func(name string, replicas *int32) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: "namespace1",
},
Spec: v1app.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Replicas: replicas,
},
}
Expand Down Expand Up @@ -93,13 +93,13 @@ func TestScaleDeploymentFunc(t *testing.T) {
}

func TestScaleHpaDeploymentFunc(t *testing.T) {
generateDeployment := func(name string, replicas *int32) *v1app.Deployment {
return &v1app.Deployment{
generateDeployment := func(name string, replicas *int32) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: "namespace1",
},
Spec: v1app.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Replicas: replicas,
},
}
Expand Down Expand Up @@ -283,15 +283,15 @@ func TestScaleDeploymentHelper(t *testing.T) {
// Create a spoofed deployment to pass to the clientsholder.
// This is only needed because the podsets.WaitForDeploymentSetReady function
// utilizes the clientsholder straight from the function to retrieve the latest information.
dep := &v1app.Deployment{
dep := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "dp1",
Namespace: "ns1",
},
Status: v1app.DeploymentStatus{
Conditions: []v1app.DeploymentCondition{
Status: appsv1.DeploymentStatus{
Conditions: []appsv1.DeploymentCondition{
{
Type: v1app.DeploymentAvailable,
Type: appsv1.DeploymentAvailable,
},
},
Replicas: 1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/test-network-function/cnf-certification-test/internal/clientsholder"
"github.com/test-network-function/cnf-certification-test/internal/log"

v1app "k8s.io/api/apps/v1"
appsv1 "k8s.io/api/apps/v1"
v1autoscaling "k8s.io/api/autoscaling/v1"
v1 "k8s.io/client-go/kubernetes/typed/apps/v1"

Expand All @@ -36,7 +36,7 @@ import (
hps "k8s.io/client-go/kubernetes/typed/autoscaling/v1"
)

func TestScaleStatefulSet(statefulset *v1app.StatefulSet, timeout time.Duration, logger *log.Logger) bool {
func TestScaleStatefulSet(statefulset *appsv1.StatefulSet, timeout time.Duration, logger *log.Logger) bool {
clients := clientsholder.GetClientsHolder()
name, namespace := statefulset.Name, statefulset.Namespace
ssClients := clients.K8sClient.AppsV1().StatefulSets(namespace)
Expand Down Expand Up @@ -79,7 +79,7 @@ func TestScaleStatefulSet(statefulset *v1app.StatefulSet, timeout time.Duration,
return true
}

func scaleStateFulsetHelper(clients *clientsholder.ClientsHolder, ssClient v1.StatefulSetInterface, statefulset *v1app.StatefulSet, replicas int32, timeout time.Duration, logger *log.Logger) bool {
func scaleStateFulsetHelper(clients *clientsholder.ClientsHolder, ssClient v1.StatefulSetInterface, statefulset *appsv1.StatefulSet, replicas int32, timeout time.Duration, logger *log.Logger) bool {
name := statefulset.Name
namespace := statefulset.Namespace

Expand Down Expand Up @@ -110,7 +110,7 @@ func scaleStateFulsetHelper(clients *clientsholder.ClientsHolder, ssClient v1.St
return true
}

func TestScaleHpaStatefulSet(statefulset *v1app.StatefulSet, hpa *v1autoscaling.HorizontalPodAutoscaler, timeout time.Duration, logger *log.Logger) bool {
func TestScaleHpaStatefulSet(statefulset *appsv1.StatefulSet, hpa *v1autoscaling.HorizontalPodAutoscaler, timeout time.Duration, logger *log.Logger) bool {
clients := clientsholder.GetClientsHolder()
hpaName := hpa.Name
name, namespace := statefulset.Name, statefulset.Namespace
Expand Down
82 changes: 82 additions & 0 deletions pkg/autodiscover/autodiscover.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ package autodiscover
import (
"context"
"errors"
"fmt"
"regexp"
"strings"
"time"

configv1 "github.com/openshift/api/config/v1"
Expand All @@ -29,6 +31,7 @@ import (
"github.com/test-network-function/cnf-certification-test/internal/log"
"github.com/test-network-function/cnf-certification-test/pkg/compatibility"
"github.com/test-network-function/cnf-certification-test/pkg/configuration"
"github.com/test-network-function/cnf-certification-test/pkg/podhelper"
"helm.sh/helm/v3/pkg/release"
appsv1 "k8s.io/api/apps/v1"
scalingv1 "k8s.io/api/autoscaling/v1"
Expand Down Expand Up @@ -56,6 +59,7 @@ type DiscoveredTestData struct {
Pods []corev1.Pod
AllPods []corev1.Pod
DebugPods []corev1.Pod
CSVToPodListMap map[string][]*corev1.Pod
ResourceQuotaItems []corev1.ResourceQuota
PodDisruptionBudgets []policyv1.PodDisruptionBudget
NetworkPolicies []networkingv1.NetworkPolicy
Expand Down Expand Up @@ -182,6 +186,12 @@ func DoAutoDiscover(config *configuration.TestConfiguration) DiscoveredTestData
data.Subscriptions = findSubscriptions(oc.OlmClient, data.Namespaces)
data.HelmChartReleases = getHelmList(oc.RestConfig, data.Namespaces)

// Get all operator pods
data.CSVToPodListMap, err = getOperatorCsvPods(data.Csvs)
if err != nil {
log.Fatal("Failed to get the operator pods, err: %v", err)
}

openshiftVersion, err := getOpenshiftVersion(oc.OcpClient)
if err != nil {
log.Fatal("Failed to get the OpenShift version, err: %v", err)
Expand Down Expand Up @@ -280,3 +290,75 @@ func getOpenshiftVersion(oClient clientconfigv1.ConfigV1Interface) (ver string,

return "", errors.New("could not get openshift version from clusterOperator")
}

// Get a map of csvs with its managed pods from the target namespaces
func getOperatorCsvPods(csvList []*olmv1Alpha.ClusterServiceVersion) (map[string][]*corev1.Pod, error) {
client := clientsholder.GetClientsHolder()
csvToPodsMapping := make(map[string][]*corev1.Pod)

for _, csv := range csvList {
// Get target namespaces
operatorTargetNamespaces, err := getOperatorTargetNamespaces(csv, client)
if err != nil {
return csvToPodsMapping, err
}
var podList []*corev1.Pod

for _, targetNamespace := range operatorTargetNamespaces {
pods, err := getCsvPodsFromTargetNamespace(csv, strings.TrimSpace(targetNamespace), client)
if err != nil {
continue
}
podList = append(podList, pods...)
}

csvToPodsMapping[fmt.Sprintf(podNameWithNamespaceFormatStr, csv.Name, csv.Namespace)] = podList
}
return csvToPodsMapping, nil
}

// This function gets the pods of the specified csv in a target namespace
func getCsvPodsFromTargetNamespace(csv *olmv1Alpha.ClusterServiceVersion, targetNamespace string, client *clientsholder.ClientsHolder) (managedPods []*corev1.Pod, err error) {
// Get all pods from the target namespace
podsList, err := client.K8sClient.CoreV1().Pods(targetNamespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}

for index := range podsList.Items {
// Get the top owners of the pod
pod := podsList.Items[index]
topOwners, err := podhelper.GetPodTopOwner(pod.Namespace, pod.OwnerReferences)
if err != nil {
return nil, fmt.Errorf("could not get top owners of Pod %s (in namespace %s), err=%v", pod.Name, pod.Namespace, err)
}

// check if owner matches with the csv
for _, owner := range topOwners {
if owner.Kind == csv.Kind && owner.Namespace == csv.Namespace && owner.Name == csv.Name {
managedPods = append(managedPods, &podsList.Items[index])
break
}
}
}
return managedPods, nil
}

// This function lists the target namespaces of an operator
func getOperatorTargetNamespaces(csv *olmv1Alpha.ClusterServiceVersion, client *clientsholder.ClientsHolder) ([]string, error) {
annotations := csv.Annotations

targetNamespaces := annotations["olm.targetNamespaces"] // This is a comma-separated string, example : a,b,c where a, b and c are target namespaces
operatorTargetNamespaces := strings.Split(targetNamespaces, ",") // For cluster installed operator olm.targetNamespaces: ""

// When the operator is cluster installed operator
if len(operatorTargetNamespaces) == 0 {
// Get all namespaces
allNamespaces, err := getAllNamespaces(client.K8sClient.CoreV1())
if err != nil {
return nil, err
}
operatorTargetNamespaces = allNamespaces
}
return operatorTargetNamespaces, nil
}
5 changes: 3 additions & 2 deletions pkg/autodiscover/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package autodiscover

const (
debugHelperPodsLabelName = "test-network-function.com/app"
debugHelperPodsLabelValue = "tnf-debug"
debugHelperPodsLabelName = "test-network-function.com/app"
debugHelperPodsLabelValue = "tnf-debug"
podNameWithNamespaceFormatStr = "%s, ns=%s"
)
76 changes: 76 additions & 0 deletions pkg/podhelper/podhelper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package podhelper

import (
"context"
"fmt"
"strings"

"github.com/test-network-function/cnf-certification-test/internal/clientsholder"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
)

// Structure to describe a top owner of a pod
type TopOwner struct {
Kind string
Name string
Namespace string
}

// Get the list of top owners of pods
func GetPodTopOwner(podNamespace string, podOwnerReferences []metav1.OwnerReference) (topOwners map[string]TopOwner, err error) {
topOwners = make(map[string]TopOwner)
err = followOwnerReferences(clientsholder.GetClientsHolder().GroupResources, clientsholder.GetClientsHolder().DynamicClient, topOwners, podNamespace, podOwnerReferences)
if err != nil {
return topOwners, fmt.Errorf("could not get top owners, err=%s", err)
}
return topOwners, nil
}

// Recursively follow the ownership tree to find the top owners
func followOwnerReferences(resourceList []*metav1.APIResourceList, dynamicClient dynamic.Interface, topOwners map[string]TopOwner, namespace string, ownerRefs []metav1.OwnerReference) (err error) {
for _, ownerRef := range ownerRefs {
// Get group resource version
gvr := getResourceSchema(resourceList, ownerRef.APIVersion, ownerRef.Kind)
// Get the owner resources
resource, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("could not get object indicated by owner references")
}
// Get owner references of the unstructured object
ownerReferences := resource.GetOwnerReferences()
if err != nil {
return fmt.Errorf("error getting owner references. err= %s", err)
}
// if no owner references, we have reached the top record it
if len(ownerReferences) == 0 {
topOwners[ownerRef.Name] = TopOwner{Kind: ownerRef.Kind, Name: ownerRef.Name, Namespace: namespace}
}
// if not continue following other branches
err = followOwnerReferences(resourceList, dynamicClient, topOwners, namespace, ownerReferences)
if err != nil {
return fmt.Errorf("error following owners")
}
}
return nil
}

// Get the Group Version Resource based on APIVersion and kind
func getResourceSchema(resourceList []*metav1.APIResourceList, apiVersion, kind string) (gvr schema.GroupVersionResource) {
const groupVersionComponentsNumber = 2
for _, gr := range resourceList {
for i := 0; i < len(gr.APIResources); i++ {
if gr.APIResources[i].Kind == kind && gr.GroupVersion == apiVersion {
groupSplit := strings.Split(gr.GroupVersion, "/")
if len(groupSplit) == groupVersionComponentsNumber {
gvr.Group = groupSplit[0]
gvr.Version = groupSplit[1]
gvr.Resource = gr.APIResources[i].Name
}
return gvr
}
}
}
return gvr
}
Loading

0 comments on commit 50389f5

Please sign in to comment.