support pulsar v3

Signed-off-by: haorenfsa <[email protected]>
zilliztech · Nov 19, 2024 · 72d91da · 72d91da
1 parent 73f3f61
commit 72d91da
Show file tree

Hide file tree

Showing 9 changed files with 362 additions and 7 deletions.
diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
@@ -22,7 +22,7 @@ jobs:
       - name: Set up Helm
         uses: azure/setup-helm@v1
         with:
-          version: v3.4.0
+          version: v3.12.3
 
       - name: Add dependency chart repos
         run: |

diff --git a/charts/milvus/Chart.yaml b/charts/milvus/Chart.yaml
@@ -3,7 +3,7 @@ name: milvus
 appVersion: "2.4.15"
 kubeVersion: "^1.10.0-0"
 description: Milvus is an open-source vector database built to power AI applications and vector similarity search.
-version: 4.2.19
+version: 4.2.20
 keywords:
   - milvus
   - elastic
@@ -18,4 +18,3 @@ maintainers:
   - email: [email protected]
     name: contact
     url: milvus.io
-engine: gotpl
diff --git a/charts/milvus/README.md b/charts/milvus/README.md
@@ -9,8 +9,8 @@ This chart bootstraps Milvus deployment on a Kubernetes cluster using the Helm p
 
 ## Prerequisites
 
-- Kubernetes 1.14+ (Attu requires 1.18+)
-- Helm >= 3.2.0
+- Kubernetes 1.20+
+- Helm >= 3.12.3
 
 ## Compatibility Notice
 As of version 4.2.0, the Milvus Helm chart no longer supports Milvus v2.3.x. If you need to deploy Milvus v2.3.x using Helm, please use Milvus Helm chart version less than 4.2.0 (e.g 4.1.36).
@@ -567,6 +567,8 @@ The following table lists the configurable parameters of the Milvus Mixture Coor
 
 This version of the chart includes the dependent Pulsar chart in the charts/ directory.
 
+To use pulsar v3, set `pulsarv3.enabled` to `true` and `pulsar.enabled` to `false`. Set other values for pulsar v3 under `pulsarv3` field.
+
 You can find more information at:
 * [https://pulsar.apache.org/charts](https://pulsar.apache.org/charts)
 

diff --git a/charts/milvus/charts/pulsar-3.3.0.tgz b/charts/milvus/charts/pulsar-3.3.0.tgz
diff --git a/charts/milvus/ci/pulsarv3-values.yaml b/charts/milvus/ci/pulsarv3-values.yaml
@@ -0,0 +1,82 @@
+## Enable or disable Milvus Cluster mode
+cluster:
+  enabled: true
+etcd:
+  replicaCount: 1
+minio:
+  mode: standalone
+  resources:
+    requests:
+      memory: 1024Mi
+pulsar:
+  enabled: false
+pulsarv3:
+  enabled: true
+  volumes:
+    persistence: false
+  components:
+    autorecovery: false
+  autorecovery:
+    replicaCount: 0
+    podMonitor:
+      enabled: false
+  zookeeper:
+    replicaCount: 1
+    podMonitor:
+      enabled: false
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    volumes:
+      # use a persistent volume or emptyDir
+      persistence: false
+    configData:
+      PULSAR_MEM: >
+        -Xms64m -Xmx256m
+  bookkeeper:
+    component: bookie
+    replicaCount: 1
+    podMonitor:
+      enabled: false
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    volumes:
+      # use a persistent volume or emptyDir
+      persistence: false
+    configData:
+      PULSAR_MEM: >
+        -Xms64m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
+  broker:
+    replicaCount: 1
+    podMonitor:
+      enabled: false
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    configData:
+      PULSAR_MEM: >
+        -Xms64m -Xmx4096m -XX:MaxDirectMemorySize=8192m
+      managedLedgerDefaultEnsembleSize: '1'
+      managedLedgerDefaultWriteQuorum: '1'
+      managedLedgerDefaultAckQuorum: '1'
+  proxy:
+    replicaCount: 1
+    autoscaling:
+      enabled: false
+    podMonitor:
+      enabled: false
+      interval: 30s
+      scrapeTimeout: 10s
+    resources:
+      requests:
+        memory: 64Mi
+        cpu: 0.001
+    configData:
+      PULSAR_MEM: >
+        -Xms64m -Xmx512m -XX:MaxDirectMemorySize=2048m
diff --git a/charts/milvus/requirements.lock b/charts/milvus/requirements.lock
@@ -8,8 +8,11 @@ dependencies:
 - name: pulsar
   repository: https://pulsar.apache.org/charts
   version: 2.7.8
+- name: pulsar
+  repository: https://pulsar.apache.org/charts
+  version: 3.3.0
 - name: kafka
   repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
   version: 15.5.1
-digest: sha256:74a42f201ec912067249ae71c08e1ed6cf14f5ec21b350801340b1286816f9a8
-generated: "2024-05-06T15:58:35.219386+08:00"
+digest: sha256:6f855ae95c4f79e728ceb308eec8e60a5e609c3f8f5aceadd4f98778569c67ab
+generated: "2024-11-19T16:13:41.631526+08:00"
diff --git a/charts/milvus/requirements.yaml b/charts/milvus/requirements.yaml
@@ -17,6 +17,13 @@ dependencies:
   condition: pulsar.enabled
   tags:
     - pulsar
+- name: pulsar
+  alias: pulsarv3
+  version: 3.3.0
+  repository: https://pulsar.apache.org/charts
+  condition: pulsarv3.enabled
+  tags:
+    - pulsarv3
 - name: kafka
   version: 15.5.1
   repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami

diff --git a/charts/milvus/templates/config.tpl b/charts/milvus/templates/config.tpl
@@ -110,6 +110,22 @@ pulsar:
 {{- end }}
   port: {{ .Values.pulsar.proxy.ports.pulsar }}
   maxMessageSize: {{ .Values.pulsar.maxMessageSize }}
+
+{{- else if .Values.pulsarv3.enabled }}
+
+mq:
+  type: pulsar
+
+messageQueue: pulsar
+
+pulsar:
+{{- if contains .Values.pulsarv3.name .Release.Name }}
+  address: {{ .Release.Name }}-proxy
+{{- else }}
+  address: {{ .Release.Name }}-{{ .Values.pulsarv3.name }}-proxy
+{{- end }}
+  port: {{ .Values.pulsarv3.proxy.ports.pulsar }}
+  maxMessageSize: {{ .Values.pulsarv3.broker.configData.maxMessageSize }}
 {{- end }}
 
 {{- if .Values.externalKafka.enabled }}