Merge pull request #1439 from kruize/mvp_demo

Merge 0.3 changes to master
kruize · Dec 23, 2024 · 0d8bf0c · 0d8bf0c
2 parents fd2f201 + 8eac6a6
commit 0d8bf0c
Show file tree

Hide file tree

Showing 80 changed files with 3,556 additions and 986 deletions.
diff --git a/Dockerfile.autotune b/Dockerfile.autotune
@@ -16,7 +16,7 @@
 ##########################################################
 #            Build Docker Image
 ##########################################################
-FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5 as mvnbuild-jdk21
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5-1733767867 as mvnbuild-jdk21
 ARG USER=autotune
 ARG AUTOTUNE_HOME=/home/$USER
 
@@ -48,7 +48,7 @@ RUN jlink --strip-debug --compress 2 --no-header-files --no-man-pages --module-p
 #            Runtime Docker Image
 ##########################################################
 # Use ubi-minimal as the base image
-FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5-1733767867
 
 ARG AUTOTUNE_VERSION
 ARG USER=autotune

diff --git a/design/BulkAPI.md b/design/BulkAPI.md
@@ -28,18 +28,23 @@ progress of the job.
 {
   "filter": {
     "exclude": {
-      "namespace": [],
+      "namespace": ["openshift-.*"],
       "workload": [],
       "containers": [],
-      "labels": {}
+      "labels": {
+        "org_id": "ABCOrga",
+        "source_id": "ZZZ",
+        "cluster_id": "ABG"
+      }
     },
     "include": {
-      "namespace": [],
+      "namespace": ["openshift-tuning"],
       "workload": [],
       "containers": [],
       "labels": {
-        "key1": "value1",
-        "key2": "value2"
+        "org_id": "ABCOrga",
+        "source_id": "ZZZ",
+        "cluster_id": "ABG"
       }
     }
   },
@@ -105,10 +110,16 @@ The specified time range determines the period over which the data is analyzed t
 - The `start` timestamp precedes the `end` timestamp.
 
 #### 2. **Request Payload with `exclude` filter specified:**
-TBA
+
+- **`exclude`** As shown in the example above, it filters out all namespaces starting with the name `openshift-` . So, we'll create experiments and generate recommendations for every namespace except those.
 
 #### 3. **Request Payload with `include` filter specified:**
-TBA
+
+- **`include`** As shown in the example above, it filters out the namespace `openshift-`. So, we'll create experiments and generate recommendations for every namespace starting with the specified name.
+
+#### 3. **Request Payload with both `include` and `exclude` filter specified:**
+
+- **`include`** As shown in the example above, it filters out all namespaces starting with the name `openshift-` but includes the `openshift-tuning` one. So, we'll create experiments and generate recommendations for the `openshift-tuning` namespace.
 
 ### GET Request:
 

diff --git a/design/KruizeConfiguration.md b/design/KruizeConfiguration.md
@@ -139,3 +139,9 @@ The following environment variables are set using the `kubectl apply` command wi
     - Value: "true"
     - Details: This flag is added for getting the details of the inputs passed to the APIs and the corresponding response 
       generated by it. This helps us in debugging the API easily in case of failures.
+- **isROSEnabled**
+  - Description:  This flag enables the remote APIs such a updateResults and the corresponding DB Tables. If set to false, the corresponding APIs and the DB tables are not supported.
+  - value: "false"
+  - Details: 
+    - Default value: False.
+      Bulk API functionality is supported when the value is set to either True or False.
diff --git a/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml b/manifests/crc/BYODB-installation/minikube/kruize-crc-minikube.yaml
@@ -33,7 +33,6 @@ data:
       "savetodb": "true",
       "dbdriver": "jdbc:postgresql://",
       "plots": "true",
-      "local": "true",
       "logAllHttpReqAndResp": "true",
       "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s",    
       "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment",
@@ -92,7 +91,7 @@ spec:
               done
       containers:
         - name: kruize
-          image: kruize/autotune_operator:0.2
+          image: kruize/autotune_operator:0.3
           imagePullPolicy: Always
           volumeMounts:
             - name: config-volume

diff --git a/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/BYODB-installation/openshift/kruize-crc-openshift.yaml
@@ -46,7 +46,6 @@ data:
       "savetodb": "true",
       "dbdriver": "jdbc:postgresql://",
       "plots": "true",
-      "local": "true",
       "logAllHttpReqAndResp": "true",
       "recommendationsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/generateRecommendations?experiment_name=%s",
       "experimentsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/createExperiment",
@@ -120,7 +119,7 @@ spec:
               done
       containers:
         - name: kruize
-          image: kruize/autotune_operator:0.2
+          image: kruize/autotune_operator:0.3
           imagePullPolicy: Always
           volumeMounts:
             - name: config-volume

diff --git a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml
@@ -97,7 +97,6 @@ data:
       "savetodb": "true",
       "dbdriver": "jdbc:postgresql://",
       "plots": "true",
-      "local": "true",
       "logAllHttpReqAndResp": "true",
       "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s",    
       "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment",
@@ -155,7 +154,7 @@ spec:
     spec:
       containers:
         - name: kruize
-          image: quay.io/kruize/autotune_operator:0.2
+          image: quay.io/kruize/autotune_operator:0.3
           imagePullPolicy: Always
           volumeMounts:
             - name: config-volume
@@ -220,7 +219,7 @@ spec:
         spec:
           containers:
             - name: kruizecronjob
-              image: quay.io/kruize/autotune_operator:0.2
+              image: quay.io/kruize/autotune_operator:0.3
               imagePullPolicy: Always
               volumeMounts:
                 - name: config-volume
@@ -346,7 +345,7 @@ spec:
         spec:
           containers:
             - name: kruizedeletejob
-              image: quay.io/kruize/autotune_operator:0.2
+              image: quay.io/kruize/autotune_operator:0.3
               imagePullPolicy: Always
               volumeMounts:
                 - name: config-volume

diff --git a/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml b/manifests/crc/default-db-included-installation/minikube/kruize-crc-minikube.yaml
@@ -1,3 +1,55 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kruize-recommendation-updater
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - customresourcedefinitions
+    verbs:
+      - '*'
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - '*'
+  - apiGroups:
+      - autoscaling.k8s.io
+    resources:
+      - verticalpodautoscalers
+      - verticalpodautoscalers/status
+      - verticalpodautoscalercheckpoints
+    verbs:
+      - '*'
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterrolebindings
+    verbs:
+      - '*'
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    verbs:
+      - "*"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kruize-recommendation-updater-crb
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kruize-recommendation-updater
+---
 apiVersion: v1
 kind: PersistentVolume
 metadata:
@@ -111,7 +163,7 @@ data:
       "savetodb": "true",
       "dbdriver": "jdbc:postgresql://",
       "plots": "true",
-      "local": "true",
+      "isROSEnabled": "false",
       "logAllHttpReqAndResp": "true",
       "recommendationsURL" : "http://kruize.monitoring.svc.cluster.local:8080/generateRecommendations?experiment_name=%s",    
       "experimentsURL" : "http://kruize.monitoring.svc.cluster.local:8080/createExperiment",
@@ -181,7 +233,7 @@ spec:
               done
       containers:
         - name: kruize
-          image: quay.io/kruize/autotune_operator:0.2
+          image: quay.io/kruize/autotune_operator:0.3
           imagePullPolicy: Always
           volumeMounts:
             - name: config-volume
@@ -246,7 +298,7 @@ spec:
         spec:
           containers:
             - name: kruizecronjob
-              image: quay.io/kruize/autotune_operator:0.2
+              image: quay.io/kruize/autotune_operator:0.3
               imagePullPolicy: Always
               volumeMounts:
                 - name: config-volume
@@ -372,7 +424,7 @@ spec:
         spec:
           containers:
             - name: kruizedeletejob
-              image: quay.io/kruize/autotune_operator:0.2
+              image: quay.io/kruize/autotune_operator:0.3
               imagePullPolicy: Always
               volumeMounts:
                 - name: config-volume

diff --git a/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml b/manifests/crc/default-db-included-installation/openshift/kruize-crc-openshift.yaml
@@ -9,6 +9,58 @@ metadata:
   name: kruize-sa
   namespace: openshift-tuning
 ---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kruize-recommendation-updater
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - customresourcedefinitions
+    verbs:
+      - '*'
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - '*'
+  - apiGroups:
+      - autoscaling.k8s.io
+    resources:
+      - verticalpodautoscalers
+      - verticalpodautoscalers/status
+      - verticalpodautoscalercheckpoints
+    verbs:
+      - '*'
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterrolebindings
+    verbs:
+      - '*'
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    verbs:
+      - "*"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kruize-recommendation-updater-crb
+subjects:
+  - kind: ServiceAccount
+    name: kruize-sa
+    namespace: openshift-tuning
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kruize-recommendation-updater
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -105,7 +157,7 @@ data:
       "savetodb": "true",
       "dbdriver": "jdbc:postgresql://",
       "plots": "true",
-      "local": "true",
+      "isROSEnabled": "false",
       "logAllHttpReqAndResp": "true",
       "recommendationsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/generateRecommendations?experiment_name=%s",
       "experimentsURL" : "http://kruize.openshift-tuning.svc.cluster.local:8080/createExperiment",
@@ -246,7 +298,7 @@ spec:
               done
       containers:
         - name: kruize
-          image: quay.io/kruize/autotune_operator:0.2
+          image: quay.io/kruize/autotune_operator:0.3
           imagePullPolicy: Always
           volumeMounts:
             - name: config-volume
@@ -318,7 +370,7 @@ spec:
         spec:
           containers:
             - name: kruizecronjob
-              image: quay.io/kruize/autotune_operator:0.2
+              image: quay.io/kruize/autotune_operator:0.3
               imagePullPolicy: Always
               volumeMounts:
                 - name: config-volume
@@ -359,7 +411,7 @@ spec:
         spec:
           containers:
             - name: kruizedeletejob
-              image: quay.io/kruize/autotune_operator:0.2
+              image: quay.io/kruize/autotune_operator:0.3
               imagePullPolicy: Always
               volumeMounts:
                 - name: config-volume

diff --git a/migrations/kruize_local_ddl.sql b/migrations/kruize_local_ddl.sql
@@ -1,6 +1,8 @@
+create table IF NOT EXISTS kruize_lm_experiments (experiment_id varchar(255) not null, cluster_name varchar(255), experiment_name varchar(255), extended_data jsonb, meta_data jsonb, mode varchar(255), performance_profile varchar(255), status varchar(255), target_cluster varchar(255), version varchar(255),experiment_type varchar(255),datasource varchar(255),creation_date timestamp(6) ,updated_date timestamp(6) , primary key (experiment_id));
 create table IF NOT EXISTS kruize_authentication (id serial, authentication_type varchar(255), credentials jsonb, service_type varchar(255), primary key (id));
 create table IF NOT EXISTS kruize_datasources (version varchar(255), name varchar(255), provider varchar(255), serviceName varchar(255), namespace varchar(255), url varchar(255), authentication_id serial, FOREIGN KEY (authentication_id) REFERENCES kruize_authentication(id), primary key (name));
 create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), datasource_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id));
-alter table kruize_experiments add column experiment_type varchar(255), add column metadata_id bigint references kruize_dsmetadata(id), alter column datasource type varchar(255);
+alter table kruize_lm_experiments  add column metadata_id bigint references kruize_dsmetadata(id);
+alter table if exists kruize_lm_experiments add constraint UK_lm_experiment_name unique (experiment_name);
 create table IF NOT EXISTS kruize_metric_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, slo jsonb, primary key (name));
-alter table kruize_recommendations add column experiment_type varchar(255);
+create table IF NOT EXISTS kruize_lm_recommendations (interval_end_time timestamp(6) not null, experiment_name varchar(255) not null, cluster_name varchar(255), extended_data jsonb, version varchar(255),experiment_type varchar(255), primary key (experiment_name, interval_end_time)) PARTITION BY RANGE (interval_end_time);
diff --git a/pom.xml b/pom.xml
@@ -6,11 +6,11 @@
 
     <groupId>org.autotune</groupId>
     <artifactId>autotune</artifactId>
-    <version>0.2</version>
+    <version>0.3</version>
     <properties>
-        <fabric8-version>4.13.2</fabric8-version>
+        <fabric8-version>7.0.0</fabric8-version>
         <org-json-version>20240303</org-json-version>
-        <jetty-version>10.0.24</jetty-version>
+        <jetty-version>12.0.12</jetty-version>
         <slf4j-version>2.17.1</slf4j-version>
         <java-version>17</java-version>
         <prometheus-simpleclient>0.14.1</prometheus-simpleclient>
@@ -72,6 +72,12 @@
             <version>${fabric8-version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>io.fabric8</groupId>
+            <artifactId>verticalpodautoscaler-client</artifactId>
+            <version>${fabric8-version}</version>
+        </dependency>
+
         <!-- https://mvnrepository.com/artifact/org.json/json -->
         <dependency>
             <groupId>org.json</groupId>
@@ -88,15 +94,15 @@
 
         <!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-servlets -->
         <dependency>
-            <groupId>org.eclipse.jetty</groupId>
-            <artifactId>jetty-servlets</artifactId>
+            <groupId>org.eclipse.jetty.ee8</groupId>
+            <artifactId>jetty-ee8-servlets</artifactId>
             <version>${jetty-version}</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-servlet -->
         <dependency>
-            <groupId>org.eclipse.jetty</groupId>
-            <artifactId>jetty-servlet</artifactId>
+            <groupId>org.eclipse.jetty.ee8</groupId>
+            <artifactId>jetty-ee8-servlet</artifactId>
             <version>${jetty-version}</version>
         </dependency>