Merge pull request #60 from FINTLabs/feature/handle-immutable-pod-sel…

…ector-update FLA-508 Handle update of immutable pod selector
FINTLabs · Oct 19, 2024 · 713a1c3 · 713a1c3
2 parents bdae615 + 38b006a
commit 713a1c3
Show file tree

Hide file tree

Showing 9 changed files with 144 additions and 20 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -49,6 +49,7 @@ dependencies {
     testImplementation(libs.operator.framework.junit5)
     testImplementation(libs.bundles.fabric8test)
     testImplementation(libs.bundles.koinTest)
+    testImplementation(libs.bundles.logunit)
 }
 
 testing {

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -9,6 +9,7 @@ hoplite = "2.8.2"
 logback = "1.5.8"
 logstashEncoder = "8.0"
 jackson = "2.17.0"
+logunit = "2.0.0"
 
 [libraries]
 fabric8-kubernetes-client = { module = "io.fabric8:kubernetes-client", version.ref = "fabric8" }
@@ -26,6 +27,8 @@ fabric8-kubernetes-server-mock = { module = "io.fabric8:kubernetes-server-mock",
 fabric8-kube-api-test = { module = "io.fabric8:kube-api-test", version.ref = "fabric8" }
 operator-framework-junit5 = { module = "io.javaoperatorsdk:operator-framework-junit-5", version.ref = "operatorSdk" }
 awaitility-kotlin = { module = "org.awaitility:awaitility-kotlin", version.ref = "awaitility" }
+logunit-core = { module = "io.github.netmikey.logunit:logunit-core", version.ref = "logunit"}
+logunit-logback = { module = "io.github.netmikey.logunit:logunit-logback", version.ref = "logunit"}
 
 koin-bom = { module = "io.insert-koin:koin-bom", version.ref = "koin" }
 koin-core = { module = "io.insert-koin:koin-core" }
@@ -42,4 +45,5 @@ koinTest = ["koin-test", "koin-test-junit5"]
 fabric8 = ["fabric8-kubernetes-client", "fabric8-crd-generator-apt"]
 fabric8test = ["fabric8-kubernetes-server-mock", "fabric8-kube-api-test"]
 logging = ["logback-classic", "logstash-logback-encoder"]
-hoplite = ["hoplite-core", "hoplite-yaml"]
+hoplite = ["hoplite-core", "hoplite-yaml"]
+logunit = ["logunit-core", "logunit-logback"]
diff --git a/src/main/kotlin/no/fintlabs/Application.kt b/src/main/kotlin/no/fintlabs/Application.kt
@@ -35,14 +35,21 @@ val baseModule = module {
             .withKubernetesSerialization(KubernetesSerialization(get(), true))
             .build()
     }
-    single<(Operator) -> Unit> {
-        { operator ->
+    single {
+        OperatorPostConfigHandler { operator ->
             getAll<Reconciler<*>>().forEach { operator.register(it) }
         }
     }
     single {
-        Operator(ConfigurationService.newOverriddenConfigurationService { it.withKubernetesClient(get()) }).apply {
-            get<(Operator) -> Unit>().invoke(this)
+        OperatorConfigHandler { config -> config.withKubernetesClient(get()) }
+    }
+    single {
+        val config = ConfigurationService.newOverriddenConfigurationService { config ->
+            getAll<OperatorConfigHandler>().reversed().forEach { it.accept(config) }
+        }
+
+        Operator(config).apply {
+            get<OperatorPostConfigHandler>().accept(this)
         }
     }
 }

diff --git a/src/main/kotlin/no/fintlabs/OperatorConfigHandlers.kt b/src/main/kotlin/no/fintlabs/OperatorConfigHandlers.kt
@@ -0,0 +1,8 @@
+package no.fintlabs
+
+import io.javaoperatorsdk.operator.Operator
+import io.javaoperatorsdk.operator.api.config.ConfigurationServiceOverrider
+import java.util.function.Consumer
+
+fun interface OperatorConfigHandler : Consumer<ConfigurationServiceOverrider>
+fun interface OperatorPostConfigHandler : Consumer<Operator>
diff --git a/src/main/kotlin/no/fintlabs/operator/DeploymentDR.kt b/src/main/kotlin/no/fintlabs/operator/DeploymentDR.kt
@@ -21,6 +21,7 @@ import org.koin.core.component.inject
 )
 class DeploymentDR : CRUDKubernetesDependentResource<Deployment, FlaisApplicationCrd>(Deployment::class.java), KoinComponent {
     private val config: Config by inject()
+    private val logger = getLogger()
 
 
     override fun desired(primary: FlaisApplicationCrd, context: Context<FlaisApplicationCrd>) = Deployment().apply {
@@ -41,6 +42,19 @@ class DeploymentDR : CRUDKubernetesDependentResource<Deployment, FlaisApplicatio
         return Matcher.Result.computed(CustomGenericKubernetesResourceMatcher.getInstance<Deployment>().matches(actual, desired, context), desired);
     }
 
+    override fun handleUpdate(actual: Deployment, desired: Deployment, primary: FlaisApplicationCrd, context: Context<FlaisApplicationCrd>): Deployment {
+        val kubernetesSerialization = context.client.kubernetesSerialization
+        val desiredSelector = kubernetesSerialization.convertValue(desired.spec.selector, Map::class.java)
+        val actualSelector = kubernetesSerialization.convertValue(actual.spec.selector, Map::class.java)
+        val podSelectorMatch = desiredSelector == actualSelector
+
+        if (podSelectorMatch) return handleUpdate(actual, desired, primary, context)
+
+        logger.info("Pod selector does not match, recreating deployment ${actual.metadata.name}")
+        handleDelete(primary, actual, context)
+        return handleCreate(desired, primary, context)
+    }
+
     private fun cretePodMetadata(primary: FlaisApplicationCrd) = createObjectMeta(primary).apply {
         annotations["kubectl.kubernetes.io/default-container"] = primary.metadata.name
         labels["observability.fintlabs.no/loki"] = primary.spec.observability?.logging?.loki?.toString() ?: "true"

diff --git a/src/test/integration/kotlin/no/fintlabs/extensions/KubernetesOperatorContext.kt b/src/test/integration/kotlin/no/fintlabs/extensions/KubernetesOperatorContext.kt
@@ -2,10 +2,12 @@ package no.fintlabs.extensions
 
 import io.fabric8.kubernetes.api.model.HasMetadata
 import io.fabric8.kubernetes.client.KubernetesClient
+import io.javaoperatorsdk.operator.Operator
 
 class KubernetesOperatorContext(
     val namespace: String,
-    private val kubernetesClient: KubernetesClient
+    val kubernetesClient: KubernetesClient,
+    val operator: Operator
 ) {
     inline fun <reified T : HasMetadata> get(name: String): T? {
         return get(T::class.java, name)

diff --git a/src/test/integration/kotlin/no/fintlabs/extensions/KubernetesOperatorExtension.kt b/src/test/integration/kotlin/no/fintlabs/extensions/KubernetesOperatorExtension.kt
@@ -15,12 +15,17 @@ import io.fabric8.kubernetes.client.utils.KubernetesSerialization
 import io.javaoperatorsdk.operator.Operator
 import io.javaoperatorsdk.operator.api.reconciler.Reconciler
 import io.javaoperatorsdk.operator.junit.DefaultNamespaceNameSupplier
+import no.fintlabs.OperatorConfigHandler
+import no.fintlabs.OperatorPostConfigHandler
 import org.awaitility.kotlin.atMost
 import org.awaitility.kotlin.await
 import org.awaitility.kotlin.until
 import org.junit.jupiter.api.extension.*
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.get
+import org.koin.core.context.loadKoinModules
+import org.koin.core.qualifier.named
+import org.koin.dsl.module
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.time.Duration
@@ -52,20 +57,23 @@ private constructor(private val crdClass: List<Class<out CustomResource<*, *>>>)
         prepareKoin(kubernetesClient)
         prepareKubernetes(kubernetesClient, namespace)
         applyAdditionalResources(kubernetesClient, namespace)
+
+        val operator = get<Operator>()
         context.store()
-            .put(KubernetesOperatorContext::class.simpleName, KubernetesOperatorContext(namespace, kubernetesClient))
+            .put(KubernetesOperatorContext::class.simpleName, KubernetesOperatorContext(namespace, kubernetesClient, operator))
 
-        get<Operator>().start()
+        operator.start()
     }
 
     override fun afterEach(context: ExtensionContext) {
-        val kubernetesClient = get<KubernetesClient>()
         val kubernetesOperatorContext =
             context.store().get(KubernetesOperatorContext::class.simpleName) as KubernetesOperatorContext
+        val kubernetesClient = kubernetesOperatorContext.kubernetesClient
 
         cleanupKubernetes(kubernetesClient, kubernetesOperatorContext.namespace)
 
-        get<Operator>().stop()
+        kubernetesOperatorContext.operator.stop()
+        kubernetesClient.close()
     }
 
     override fun supportsParameter(pContext: ParameterContext, eContext: ExtensionContext): Boolean =
@@ -98,15 +106,21 @@ private constructor(private val crdClass: List<Class<out CustomResource<*, *>>>)
 
     private fun prepareKoin(kubernetesClient: KubernetesClient) {
         getKoin().apply {
-            declare(kubernetesClient)
-            declare<(Operator) -> Unit>(
-                { operator ->
-                    getAll<Reconciler<*>>().forEach {
-                        operator.register(it) { config ->
-                            config.settingNamespace(kubernetesClient.namespace)
+            loadKoinModules(
+                module {
+                    single { kubernetesClient }
+                    single(named("test")) { OperatorConfigHandler { config -> config.withCloseClientOnStop(false) } }
+                    single {
+                        OperatorPostConfigHandler { operator ->
+                            getAll<Reconciler<*>>().forEach {
+                                operator.register(it) { config ->
+                                    config.settingNamespace(kubernetesClient.namespace)
+                                }
+                            }
                         }
                     }
-                })
+                }
+            )
         }
     }
 

diff --git a/src/test/integration/kotlin/no/fintlabs/operator/DeploymentDRTest.kt b/src/test/integration/kotlin/no/fintlabs/operator/DeploymentDRTest.kt
@@ -6,13 +6,16 @@ import io.fabric8.kubernetes.api.model.apps.Deployment
 import io.fabric8.kubernetes.api.model.apps.DeploymentStrategy
 import io.fabric8.kubernetes.api.model.apps.RollingUpdateDeployment
 import io.fabric8.kubernetes.client.KubernetesClientException
+import io.github.netmikey.logunit.api.LogCapturer
 import no.fintlabs.extensions.KubernetesOperatorContext
 import no.fintlabs.extensions.KubernetesResources
 import no.fintlabs.loadConfig
 import no.fintlabs.operator.Utils.createAndGetResource
 import no.fintlabs.operator.Utils.createKoinTestExtension
 import no.fintlabs.operator.Utils.createKubernetesOperatorExtension
 import no.fintlabs.operator.Utils.createTestFlaisApplication
+import no.fintlabs.operator.Utils.updateAndGetResource
+import no.fintlabs.operator.Utils.waitUntilIsDeployed
 import no.fintlabs.operator.api.LOKI_LOGGING_LABEL
 import no.fintlabs.operator.api.v1alpha1.*
 import no.fintlabs.v1alpha1.kafkauserandaclspec.Acls
@@ -465,10 +468,63 @@ class DeploymentDRTest {
     }
     //endregion
 
+    //region PodSelector
+    @Test
+    fun `should recreate deployment on pod selector change selector`(context: KubernetesOperatorContext) {
+        val flaisApplication = createTestFlaisApplication()
+
+        var deployment = context.createAndGetDeployment(flaisApplication)
+        assertNotNull(deployment)
+
+        context.operator.stop()
+        context.delete(deployment)
+
+        deployment.spec.apply {
+            selector.matchLabels["another"] = "another"
+            template.metadata.labels["another"] = "another"
+        }
+        deployment.metadata.resourceVersion = null
+
+        context.create(deployment)
+
+        context.operator.start()
+        context.waitUntilIsDeployed(flaisApplication)
+        deployment = context.get<Deployment>(deployment.metadata.name)
+
+        assertNotNull(deployment)
+        assertEquals(1, deployment.spec.selector.matchLabels.size)
+        assert(deployment.spec.selector.matchLabels.containsKey("app"))
+        assertEquals(deployment.metadata.name, deployment.spec.selector.matchLabels["app"])
+    }
+
+    @Test
+    fun `should not recreate deployment on pod selector match`(context: KubernetesOperatorContext) {
+        val flaisApplication = createTestFlaisApplication()
+
+        var deployment = context.createAndGetDeployment(flaisApplication)
+        assertNotNull(deployment)
+
+        flaisApplication.apply {
+            spec = spec.copy(
+                image = "test-image:234567890"
+            )
+        }
+
+
+        deployment = context.updateAndGetResource(flaisApplication)
+        assertNotNull(deployment)
+        logs.assertDoesNotContain("Pod selector does not match, recreating deployment")
+
+    }
+    //endregion
+
 
     private fun KubernetesOperatorContext.createAndGetDeployment(app: FlaisApplicationCrd) =
         createAndGetResource<Deployment>(app)
 
+    @RegisterExtension
+    val logs: LogCapturer = LogCapturer.create().captureForType(DeploymentDR::class.java)
+
     companion object {
         @RegisterExtension
         val koinTestExtension = createKoinTestExtension(module {

diff --git a/src/test/integration/kotlin/no/fintlabs/operator/Utils.kt b/src/test/integration/kotlin/no/fintlabs/operator/Utils.kt
@@ -24,12 +24,30 @@ import java.time.Duration
 object Utils {
     inline fun <reified T : HasMetadata> KubernetesOperatorContext.createAndGetResource(app: FlaisApplicationCrd, nameSelector: (FlaisApplicationCrd) -> String = { it.metadata.name }): T? {
         create(app)
-        await atMost Duration.ofSeconds(10) until {
-            get<FlaisApplicationCrd>(app.metadata.name)?.status?.state == FlaisApplicationState.DEPLOYED
-        }
+        waitUntilIsDeployed(app)
+        return get<T>(nameSelector(app))
+    }
+
+    inline fun <reified T : HasMetadata> KubernetesOperatorContext.updateAndGetResource(app: FlaisApplicationCrd, nameSelector: (FlaisApplicationCrd) -> String = { it.metadata.name }): T? {
+        update(app)
+        waitUntilIsDeployed(app)
         return get<T>(nameSelector(app))
     }
 
+    fun KubernetesOperatorContext.waitUntilIsDeployed(app: FlaisApplicationCrd) {
+        waitUntil<FlaisApplicationCrd>(
+            app.metadata.name,
+        ) { it.status?.state == FlaisApplicationState.DEPLOYED }
+    }
+
+    inline fun <reified T : HasMetadata> KubernetesOperatorContext.waitUntil(resourceName: String, timeout: Duration = Duration.ofSeconds(30), crossinline condition: (T) -> Boolean) {
+        await atMost timeout until {
+            get<T>(resourceName)?.let { condition(it) } ?: false
+        }
+    }
+
+
+
     fun createTestFlaisApplication(): FlaisApplicationCrd {
         return FlaisApplicationCrd().apply {
             metadata = ObjectMeta().apply {