test: add integration test / CBT framework

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.21'
+  GO_VERSION: '1.22'
   GOLANGCI_VERSION: 'v1.54.0'
   DOCKER_BUILDX_VERSION: 'v0.8.2'
 
@@ -170,6 +170,49 @@ jobs:
           flags: unittests
           file: _output/tests/linux_amd64/coverage.txt
 
+  integration-tests:
+    runs-on: ubuntu-22.04
+    needs: detect-noop
+    if: needs.detect-noop.outputs.noop != 'true'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Fetch History
+        run: git fetch --prune --unshallow
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Find the Go Build Cache
+        id: go
+        run: echo "::set-output name=cache::$(make go.cachedir)"
+
+      - name: Cache the Go Build Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go.outputs.cache }}
+          key: ${{ runner.os }}-build-unit-tests-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-build-unit-tests-
+
+      - name: Cache Go Dependencies
+        uses: actions/cache@v2
+        with:
+          path: .work/pkg
+          key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-pkg-
+
+      - name: Vendor Dependencies
+        run: make vendor vendor.check
+
+      - name: Run Integration Tests
+        run: make -j2 test-integration
+
   local-deploy:
     runs-on: ubuntu-22.04
     needs: detect-noop

.vscode/launch.json

@@ -23,6 +23,20 @@
             },
             // "showLog": true,
             // "trace": "verbose",
-        }
+        },
+        {
+            "name": "Integration Tests",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/tests/integration/suite_test.go",
+            "env": {
+                "KUBECONFIG": "",
+                "KUBEBUILDER_ASSETS": "/Users/tylergillson/spectrocloud/repos/oss/spectrocloud/provider-palette/.cache/tools/darwin_arm64/k8s/1.30.0-darwin-arm64",
+                "TERRAFORM_VERSION": "1.3.3",
+                "TERRAFORM_PROVIDER_VERSION": "0.19.2",
+                // "TF_REATTACH_PROVIDERS": "{\"registry.terraform.io/spectrocloud/spectrocloud\":{\"Protocol\":\"grpc\",\"ProtocolVersion\":5,\"Pid\":5045,\"Test\":true,\"Addr\":{\"Network\":\"unix\",\"String\":\"/var/folders/w1/sm7f2fb959j2xkxln196ksj40000gn/T/plugin2100568909\"}}}",
+            },
+        },
     ]
 }

Makefile

@@ -158,6 +158,25 @@ run: go.build
 	@# To see other arguments that can be provided, run the command with --help instead
 	UPBOUND_CONTEXT="local" $(GO_OUT_DIR)/provider --debug
 
+# ===================
+# Integration Testing
+
+test-integration: init-envtest ## Run integration tests
+	@mkdir -p $(GO_TEST_OUTPUT) || $(FAIL)
+	KUBEBUILDER_ASSETS=${KUBEBUILDER_ASSETS} go test -v -timeout 10m \
+		-covermode=atomic -coverpkg=./... -coverprofile=$(GO_TEST_OUTPUT)/integration.out ./tests/...
+
+init-envtest: setup-envtest
+	$(TOOLS_HOST_DIR)/setup-envtest use --bin-dir $(TOOLS_HOST_DIR) $(ENVTEST_VERSION)
+KUBEBUILDER_ASSETS = $(shell $(TOOLS_HOST_DIR)/setup-envtest use -p path --bin-dir $(TOOLS_HOST_DIR) $(ENVTEST_VERSION))
+
+setup-envtest:
+ifeq ("$(wildcard $(TOOLS_HOST_DIR)/setup-envtest)", "")
+	go get sigs.k8s.io/controller-runtime/tools/setup-envtest
+	GOBIN=$(TOOLS_HOST_DIR) go install sigs.k8s.io/controller-runtime/tools/setup-envtest
+endif
+SETUP_ENVTEST=$(TOOLS_HOST_DIR)/setup-envtest
+
 # ====================================================================================
 # End to End Testing
 CROSSPLANE_NAMESPACE = upbound-system

cmd/provider/main.go

@@ -4,118 +4,8 @@ Copyright 2021 Upbound Inc.
 
 package main
 
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"time"
-
-	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
-	xpcontroller "github.com/crossplane/crossplane-runtime/pkg/controller"
-	"github.com/crossplane/crossplane-runtime/pkg/feature"
-	"github.com/crossplane/crossplane-runtime/pkg/logging"
-	"github.com/crossplane/crossplane-runtime/pkg/ratelimiter"
-	"github.com/crossplane/crossplane-runtime/pkg/resource"
-	tjcontroller "github.com/crossplane/upjet/pkg/controller"
-	"github.com/crossplane/upjet/pkg/terraform"
-	"github.com/go-logr/logr"
-	"gopkg.in/alecthomas/kingpin.v2"
-	kerrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
-	ctrl "sigs.k8s.io/controller-runtime"
-	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-
-	"github.com/crossplane-contrib/provider-palette/apis"
-	"github.com/crossplane-contrib/provider-palette/apis/v1alpha1"
-	"github.com/crossplane-contrib/provider-palette/config"
-	"github.com/crossplane-contrib/provider-palette/internal/clients"
-	"github.com/crossplane-contrib/provider-palette/internal/controller"
-	"github.com/crossplane-contrib/provider-palette/internal/features"
-)
+import "github.com/crossplane-contrib/provider-palette/cmd/provider/run"
 
 func main() {
-	var (
-		app              = kingpin.New(filepath.Base(os.Args[0]), "Terraform based Crossplane provider for Palette").DefaultEnvars()
-		debug            = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
-		leaderElection   = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
-		terraformVersion = app.Flag("terraform-version", "Terraform version.").Required().Envar("TERRAFORM_VERSION").String()
-		providerSource   = app.Flag("terraform-provider-source", "Terraform provider source.").Required().Envar("TERRAFORM_PROVIDER_SOURCE").String()
-		providerVersion  = app.Flag("terraform-provider-version", "Terraform provider version.").Required().Envar("TERRAFORM_PROVIDER_VERSION").String()
-		maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("10").Int()
-
-		namespace                  = app.Flag("namespace", "Namespace used to set as default scope in default secret store config.").Default("crossplane-system").Envar("POD_NAMESPACE").String()
-		enableExternalSecretStores = app.Flag("enable-external-secret-stores", "Enable support for ExternalSecretStores.").Default("false").Envar("ENABLE_EXTERNAL_SECRET_STORES").Bool()
-		enableManagementPolicies   = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("false").Envar("ENABLE_MANAGEMENT_POLICIES").Bool()
-	)
-
-	kingpin.MustParse(app.Parse(os.Args[1:]))
-
-	zl := zap.New(zap.UseDevMode(*debug))
-	log := logging.NewLogrLogger(zl.WithName("provider-palette"))
-	if *debug {
-		// The controller-runtime runs with a no-op logger by default. It is
-		// *very* verbose even at info level, so we only provide it a real
-		// logger when we're running in debug mode.
-		ctrl.SetLogger(zl)
-	} else {
-		ctrl.SetLogger(logr.New(ctrllog.NullLogSink{}))
-	}
-
-	log.Debug("Starting")
-
-	cfg, err := ctrl.GetConfig()
-	kingpin.FatalIfError(err, "Cannot get API server rest config")
-
-	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
-		LeaderElection:             *leaderElection,
-		LeaderElectionID:           "crossplane-leader-election-provider-palette",
-		LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
-		LeaseDuration:              func() *time.Duration { d := 60 * time.Second; return &d }(),
-		RenewDeadline:              func() *time.Duration { d := 50 * time.Second; return &d }(),
-	})
-	kingpin.FatalIfError(err, "Cannot create controller manager")
-	kingpin.FatalIfError(apis.AddToScheme(mgr.GetScheme()), "Cannot add Palette APIs to scheme")
-	o := tjcontroller.Options{
-		Options: xpcontroller.Options{
-			Logger:                  log,
-			GlobalRateLimiter:       ratelimiter.NewGlobal(*maxReconcileRate),
-			PollInterval:            1 * time.Minute,
-			MaxConcurrentReconciles: 1,
-			Features:                &feature.Flags{},
-		},
-		Provider: config.GetProvider(),
-		// use the following WorkspaceStoreOption to enable the shared gRPC mode
-		// terraform.WithProviderRunner(terraform.NewSharedProvider(log, os.Getenv("TERRAFORM_NATIVE_PROVIDER_PATH"), terraform.WithNativeProviderArgs("-debuggable")))
-		WorkspaceStore: terraform.NewWorkspaceStore(log),
-		SetupFn:        clients.TerraformSetupBuilder(*terraformVersion, *providerSource, *providerVersion),
-	}
-
-	if *enableManagementPolicies {
-		o.Features.Enable(features.EnableBetaManagementPolicies)
-		log.Info("Beta feature enabled", "flag", features.EnableBetaManagementPolicies)
-	}
-
-	if *enableExternalSecretStores {
-		o.SecretStoreConfigGVK = &v1alpha1.StoreConfigGroupVersionKind
-		log.Info("Alpha feature enabled", "flag", features.EnableAlphaExternalSecretStores)
-
-		// Ensure default store config exists.
-		kingpin.FatalIfError(resource.Ignore(kerrors.IsAlreadyExists, mgr.GetClient().Create(context.Background(), &v1alpha1.StoreConfig{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "default",
-			},
-			Spec: v1alpha1.StoreConfigSpec{
-				// NOTE(turkenh): We only set required spec and expect optional
-				// ones to properly be initialized with CRD level default values.
-				SecretStoreConfig: xpv1.SecretStoreConfig{
-					DefaultScope: *namespace,
-				},
-			},
-		})), "cannot create default store config")
-	}
-
-	kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup Palette controllers")
-	kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
+	run.Run()
 }

cmd/provider/run/run.go

@@ -0,0 +1,125 @@
+package run
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	xpcontroller "github.com/crossplane/crossplane-runtime/pkg/controller"
+	"github.com/crossplane/crossplane-runtime/pkg/feature"
+	"github.com/crossplane/crossplane-runtime/pkg/logging"
+	"github.com/crossplane/crossplane-runtime/pkg/ratelimiter"
+	"github.com/crossplane/crossplane-runtime/pkg/resource"
+	tjcontroller "github.com/crossplane/upjet/pkg/controller"
+	"github.com/crossplane/upjet/pkg/terraform"
+	"github.com/go-logr/logr"
+	"gopkg.in/alecthomas/kingpin.v2"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	"github.com/crossplane-contrib/provider-palette/apis"
+	"github.com/crossplane-contrib/provider-palette/apis/v1alpha1"
+	"github.com/crossplane-contrib/provider-palette/config"
+	"github.com/crossplane-contrib/provider-palette/internal/clients"
+	"github.com/crossplane-contrib/provider-palette/internal/controller"
+	"github.com/crossplane-contrib/provider-palette/internal/features"
+)
+
+var cancel context.CancelFunc
+
+func Run() {
+	var (
+		app              = kingpin.New(filepath.Base(os.Args[0]), "Terraform based Crossplane provider for Palette").DefaultEnvars()
+		debug            = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
+		leaderElection   = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
+		terraformVersion = app.Flag("terraform-version", "Terraform version.").Required().Envar("TERRAFORM_VERSION").String()
+		providerSource   = app.Flag("terraform-provider-source", "Terraform provider source.").Required().Envar("TERRAFORM_PROVIDER_SOURCE").String()
+		providerVersion  = app.Flag("terraform-provider-version", "Terraform provider version.").Required().Envar("TERRAFORM_PROVIDER_VERSION").String()
+		maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("10").Int()
+
+		namespace                  = app.Flag("namespace", "Namespace used to set as default scope in default secret store config.").Default("crossplane-system").Envar("POD_NAMESPACE").String()
+		enableExternalSecretStores = app.Flag("enable-external-secret-stores", "Enable support for ExternalSecretStores.").Default("false").Envar("ENABLE_EXTERNAL_SECRET_STORES").Bool()
+		enableManagementPolicies   = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("false").Envar("ENABLE_MANAGEMENT_POLICIES").Bool()
+	)
+
+	kingpin.MustParse(app.Parse(os.Args[1:]))
+
+	zl := zap.New(zap.UseDevMode(*debug))
+	log := logging.NewLogrLogger(zl.WithName("provider-palette"))
+	if *debug {
+		// The controller-runtime runs with a no-op logger by default. It is
+		// *very* verbose even at info level, so we only provide it a real
+		// logger when we're running in debug mode.
+		ctrl.SetLogger(zl)
+	} else {
+		ctrl.SetLogger(logr.New(ctrllog.NullLogSink{}))
+	}
+
+	log.Debug("Starting")
+
+	cfg, err := ctrl.GetConfig()
+	kingpin.FatalIfError(err, "Cannot get API server rest config")
+
+	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+		LeaderElection:             *leaderElection,
+		LeaderElectionID:           "crossplane-leader-election-provider-palette",
+		LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
+		LeaseDuration:              func() *time.Duration { d := 60 * time.Second; return &d }(),
+		RenewDeadline:              func() *time.Duration { d := 50 * time.Second; return &d }(),
+	})
+	kingpin.FatalIfError(err, "Cannot create controller manager")
+	kingpin.FatalIfError(apis.AddToScheme(mgr.GetScheme()), "Cannot add Palette APIs to scheme")
+	o := tjcontroller.Options{
+		Options: xpcontroller.Options{
+			Logger:                  log,
+			GlobalRateLimiter:       ratelimiter.NewGlobal(*maxReconcileRate),
+			PollInterval:            1 * time.Minute,
+			MaxConcurrentReconciles: 1,
+			Features:                &feature.Flags{},
+		},
+		Provider: config.GetProvider(),
+		// use the following WorkspaceStoreOption to enable the shared gRPC mode
+		// terraform.WithProviderRunner(terraform.NewSharedProvider(log, os.Getenv("TERRAFORM_NATIVE_PROVIDER_PATH"), terraform.WithNativeProviderArgs("-debuggable")))
+		WorkspaceStore: terraform.NewWorkspaceStore(log),
+		SetupFn:        clients.TerraformSetupBuilder(*terraformVersion, *providerSource, *providerVersion),
+	}
+
+	if *enableManagementPolicies {
+		o.Features.Enable(features.EnableBetaManagementPolicies)
+		log.Info("Beta feature enabled", "flag", features.EnableBetaManagementPolicies)
+	}
+
+	if *enableExternalSecretStores {
+		o.SecretStoreConfigGVK = &v1alpha1.StoreConfigGroupVersionKind
+		log.Info("Alpha feature enabled", "flag", features.EnableAlphaExternalSecretStores)
+
+		// Ensure default store config exists.
+		kingpin.FatalIfError(resource.Ignore(kerrors.IsAlreadyExists, mgr.GetClient().Create(context.Background(), &v1alpha1.StoreConfig{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "default",
+			},
+			Spec: v1alpha1.StoreConfigSpec{
+				// NOTE(turkenh): We only set required spec and expect optional
+				// ones to properly be initialized with CRD level default values.
+				SecretStoreConfig: xpv1.SecretStoreConfig{
+					DefaultScope: *namespace,
+				},
+			},
+		})), "cannot create default store config")
+	}
+
+	kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup Palette controllers")
+	ctx := ctrl.SetupSignalHandler()
+	ctx, cancel = context.WithCancel(ctx)
+	kingpin.FatalIfError(mgr.Start(ctx), "Cannot start controller manager")
+}
+
+func Cancel() {
+	cancel()
+}