*: update to use the new msi-dataplane library

Signed-off-by: Steve Kuznetsov <[email protected]>

go.mod

@@ -1,6 +1,6 @@
 module github.com/Azure/ARO-RP
 
-go 1.23.2
+go 1.21
 
 require (
 	github.com/Azure/azure-sdk-for-go v63.1.0+incompatible

pkg/cluster/cluster.go

@@ -12,7 +12,6 @@ import (
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/msi-dataplane/pkg/dataplane"
-	"github.com/Azure/msi-dataplane/pkg/store"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	imageregistryclient "github.com/openshift/client-go/imageregistry/clientset/versioned"
 	machineclient "github.com/openshift/client-go/machine/clientset/versioned"
@@ -36,7 +35,6 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armauthorization"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armmsi"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armnetwork"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/azsecrets"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
@@ -48,6 +46,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/dns"
 	"github.com/Azure/ARO-RP/pkg/util/encryption"
 	utilgraph "github.com/Azure/ARO-RP/pkg/util/graph"
+	"github.com/Azure/ARO-RP/pkg/util/keyvault"
 	"github.com/Azure/ARO-RP/pkg/util/platformworkloadidentity"
 	"github.com/Azure/ARO-RP/pkg/util/refreshable"
 	"github.com/Azure/ARO-RP/pkg/util/storage"
@@ -128,8 +127,8 @@ type manager struct {
 
 	aroOperatorDeployer deploy.Operator
 
-	msiDataplane                           *dataplane.ManagedIdentityClient
-	clusterMsiKeyVaultStore                *store.MsiKeyVaultStore
+	msiDataplane                           dataplane.ClientFactory
+	clusterMsiKeyVaultStore                keyvault.Manager
 	clusterMsiFederatedIdentityCredentials armmsi.FederatedIdentityCredentialsClient
 
 	now func() time.Time
@@ -316,36 +315,36 @@ func New(ctx context.Context, log *logrus.Entry, _env env.Interface, db database
 			return nil, err
 		}
 
-		msiDataplaneClientOptions, err := _env.MsiDataplaneClientOptions(msiResourceId)
-		if err != nil {
-			return nil, err
-		}
+		var msiDataplane dataplane.ClientFactory
+		if _env.FeatureIsSet(env.FeatureUseMockMsiRp) {
+			msiDataplane = _env.MockMSIResponses(msiResourceId)
+		} else {
+			msiDataplaneClientOptions, err := _env.MsiDataplaneClientOptions()
+			if err != nil {
+				return nil, err
+			}
 
-		cloud, err := _env.Environment().CloudNameForMsiDataplane()
-		if err != nil {
-			return nil, err
-		}
+			// MSI dataplane client receives tenant from the bearer challenge, so we can't limit the allowed tenants in the credential
+			fpMSICred, err := _env.FPNewClientCertificateCredential(_env.TenantID(), []string{"*"})
+			if err != nil {
+				return nil, err
+			}
 
-		// MSI dataplane client receives tenant from the bearer challenge, so we can't limit the allowed tenants in the credential
-		fpMSICred, err := _env.FPNewClientCertificateCredential(_env.TenantID(), []string{"*"})
-		if err != nil {
-			return nil, err
-		}
-		authenticatorPolicy := dataplane.NewAuthenticatorPolicy(fpMSICred, _env.MsiRpEndpoint())
-		msiDataplane, err := dataplane.NewClient(cloud, authenticatorPolicy, msiDataplaneClientOptions)
-		if err != nil {
-			return nil, err
+			msiDataplane, err = dataplane.NewClientFactory(fpMSICred, _env.MsiRpEndpoint(), msiDataplaneClientOptions)
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		clusterMsiKeyVaultName := _env.ClusterMsiKeyVaultName()
 		clusterMsiKeyVaultURL := fmt.Sprintf("https://%s.%s", clusterMsiKeyVaultName, _env.Environment().KeyVaultDNSSuffix)
-		clusterMsiSecretsClient, err := azsecrets.NewClient(clusterMsiKeyVaultURL, msiCredential, clientOptions)
+		authorizer, err := _env.NewMSIAuthorizer(_env.Environment().KeyVaultScope)
 		if err != nil {
 			return nil, err
 		}
 
 		m.msiDataplane = msiDataplane
-		m.clusterMsiKeyVaultStore = store.NewMsiKeyVaultStore(clusterMsiSecretsClient)
+		m.clusterMsiKeyVaultStore = keyvault.NewManager(authorizer, clusterMsiKeyVaultURL)
 	}
 
 	return m, nil

pkg/cluster/clustermsi.go

@@ -5,16 +5,18 @@ package cluster
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault"
+	"github.com/Azure/go-autorest/autorest/date"
 	"github.com/Azure/msi-dataplane/pkg/dataplane"
-	"github.com/Azure/msi-dataplane/pkg/dataplane/swagger"
-	"github.com/Azure/msi-dataplane/pkg/store"
+	"k8s.io/utils/ptr"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/env"
@@ -39,7 +41,7 @@ func (m *manager) ensureClusterMsiCertificate(ctx context.Context) error {
 		return err
 	}
 
-	_, err = m.clusterMsiKeyVaultStore.GetCredentialsObject(ctx, secretName)
+	_, err = m.clusterMsiKeyVaultStore.GetSecret(ctx, secretName)
 	if err == nil {
 		return nil
 	} else if azcoreErr, ok := err.(*azcore.ResponseError); !ok || azcoreErr.StatusCode != http.StatusNotFound {
@@ -51,13 +53,16 @@ func (m *manager) ensureClusterMsiCertificate(ctx context.Context) error {
 		return err
 	}
 
-	uaMsiRequest := dataplane.UserAssignedMSIRequest{
-		IdentityURL: m.doc.OpenShiftCluster.Identity.IdentityURL,
-		ResourceIDs: []string{clusterMsiResourceId.String()},
-		TenantID:    m.doc.OpenShiftCluster.Identity.TenantID,
+	uaMsiRequest := dataplane.UserAssignedIdentitiesRequest{
+		DelegatedResources: &[]string{clusterMsiResourceId.String()},
 	}
 
-	msiCredObj, err := m.msiDataplane.GetUserAssignedIdentities(ctx, uaMsiRequest)
+	client, err := m.msiDataplane.NewClient(m.doc.OpenShiftCluster.Identity.IdentityURL)
+	if err != nil {
+		return err
+	}
+
+	msiCredObj, err := client.GetUserAssignedIdentitiesCredentials(ctx, uaMsiRequest)
 	if err != nil {
 		return err
 	}
@@ -76,21 +81,27 @@ func (m *manager) ensureClusterMsiCertificate(ctx context.Context) error {
 			return errors.New("unable to pull NotAfter from the MSI CredentialsObject")
 		}
 
-		// The swagger API spec for the MI RP specifies that NotAfter will be "in the format 2017-03-01T14:11:00Z".
+		// The OpenAPI spec for the MI RP specifies that NotAfter will be "in the format 2017-03-01T14:11:00Z".
 		expirationDate, err = time.Parse(time.RFC3339, *identity.NotAfter)
 		if err != nil {
 			return err
 		}
 	}
 
-	secretProperties := store.SecretProperties{
-		Enabled:   true,
-		Expires:   expirationDate,
-		Name:      secretName,
-		NotBefore: now,
+	raw, err := json.Marshal(msiCredObj)
+	if err != nil {
+		return err
 	}
 
-	return m.clusterMsiKeyVaultStore.SetCredentialsObject(ctx, secretProperties, msiCredObj.CredentialsObject)
+	return m.clusterMsiKeyVaultStore.SetSecret(ctx, secretName, keyvault.SecretSetParameters{
+		Value: ptr.To(string(raw)),
+		SecretAttributes: &keyvault.SecretAttributes{
+			Enabled:   ptr.To(true),
+			Expires:   ptr.To(date.UnixTime(expirationDate)),
+			NotBefore: ptr.To(date.UnixTime(expirationDate)),
+		},
+		Tags: nil,
+	})
 }
 
 // initializeClusterMsiClients intializes any Azure clients that use the cluster
@@ -101,17 +112,21 @@ func (m *manager) initializeClusterMsiClients(ctx context.Context) error {
 		return err
 	}
 
-	kvSecret, err := m.clusterMsiKeyVaultStore.GetCredentialsObject(ctx, secretName)
+	kvSecretResponse, err := m.clusterMsiKeyVaultStore.GetSecret(ctx, secretName)
 	if err != nil {
 		return err
 	}
 
-	cloud, err := m.env.Environment().CloudNameForMsiDataplane()
-	if err != nil {
+	if kvSecretResponse.Value == nil {
+		return fmt.Errorf("secret %q in keyvault missing value", secretName)
+	}
+
+	var kvSecret dataplane.ManagedIdentityCredentials
+	if err := json.Unmarshal([]byte(*kvSecretResponse.Value), &kvSecret); err != nil {
 		return err
 	}
 
-	uaIdentities, err := dataplane.NewUserAssignedIdentities(kvSecret.CredentialsObject, cloud)
+	cloud, err := m.env.Environment().CloudNameForMsiDataplane()
 	if err != nil {
 		return err
 	}
@@ -121,9 +136,20 @@ func (m *manager) initializeClusterMsiClients(ctx context.Context) error {
 		return err
 	}
 
-	azureCred, err := uaIdentities.GetCredential(msiResourceId.String())
-	if err != nil {
-		return err
+	var azureCred azcore.TokenCredential
+	if kvSecret.ExplicitIdentities != nil {
+		for _, identity := range *kvSecret.ExplicitIdentities {
+			if identity.ResourceId != nil && *identity.ResourceId == msiResourceId.String() {
+				var err error
+				azureCred, err = dataplane.GetCredential(cloud, identity)
+				if err != nil {
+					return fmt.Errorf("failed to get credential for msi identity %q: %v", msiResourceId, err)
+				}
+			}
+		}
+	}
+	if azureCred == nil {
+		return fmt.Errorf("managed identity credential missing user-assigned identity %q", msiResourceId)
 	}
 
 	// Note that we are assuming that all of the platform MIs are in the same subscription as the ARO resource.
@@ -165,13 +191,16 @@ func (m *manager) clusterIdentityIDs(ctx context.Context) error {
 		return err
 	}
 
-	uaMsiRequest := dataplane.UserAssignedMSIRequest{
-		IdentityURL: m.doc.OpenShiftCluster.Identity.IdentityURL,
-		ResourceIDs: []string{clusterMsiResourceId.String()},
-		TenantID:    m.doc.OpenShiftCluster.Identity.TenantID,
+	uaMsiRequest := dataplane.UserAssignedIdentitiesRequest{
+		DelegatedResources: &[]string{clusterMsiResourceId.String()},
+	}
+
+	client, err := m.msiDataplane.NewClient(m.doc.OpenShiftCluster.Identity.IdentityURL)
+	if err != nil {
+		return err
 	}
 
-	msiCredObj, err := m.msiDataplane.GetUserAssignedIdentities(ctx, uaMsiRequest)
+	msiCredObj, err := client.GetUserAssignedIdentitiesCredentials(ctx, uaMsiRequest)
 	if err != nil {
 		return err
 	}
@@ -180,7 +209,7 @@ func (m *manager) clusterIdentityIDs(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	if identity.ClientID == nil || identity.ObjectID == nil {
+	if identity.ClientId == nil || identity.ObjectId == nil {
 		return fmt.Errorf("unable to pull clientID and objectID from the MSI CredentialsObject")
 	}
 
@@ -190,8 +219,8 @@ func (m *manager) clusterIdentityIDs(ctx context.Context) error {
 		// passed-in casing on IDs even if it may be incorrect
 		for k, v := range doc.OpenShiftCluster.Identity.UserAssignedIdentities {
 			if strings.EqualFold(k, clusterMsiResourceId.String()) {
-				v.ClientID = *identity.ClientID
-				v.PrincipalID = *identity.ObjectID
+				v.ClientID = *identity.ClientId
+				v.PrincipalID = *identity.ObjectId
 
 				doc.OpenShiftCluster.Identity.UserAssignedIdentities[k] = v
 				return nil
@@ -207,14 +236,13 @@ func (m *manager) clusterIdentityIDs(ctx context.Context) error {
 // We expect the GetUserAssignedIdentities request to only ever be made for one identity
 // at a time (the cluster MSI) and thus we expect the response to only contain a single
 // identity's details.
-func getSingleExplicitIdentity(msiCredObj *dataplane.UserAssignedIdentities) (*swagger.NestedCredentialsObject, error) {
+func getSingleExplicitIdentity(msiCredObj *dataplane.ManagedIdentityCredentials) (dataplane.UserAssignedIdentityCredentials, error) {
 	if msiCredObj.ExplicitIdentities == nil ||
-		len(msiCredObj.ExplicitIdentities) == 0 ||
-		msiCredObj.ExplicitIdentities[0] == nil {
-		return nil, errClusterMsiNotPresentInResponse
+		len(*msiCredObj.ExplicitIdentities) == 0 {
+		return dataplane.UserAssignedIdentityCredentials{}, errClusterMsiNotPresentInResponse
 	}
 
-	return msiCredObj.ExplicitIdentities[0], nil
+	return (*msiCredObj.ExplicitIdentities)[0], nil
 }
 
 // fixupClusterMsiTenantID repopulates the cluster MSI's tenant ID in the cluster doc by