Skip to content

Commit

Permalink
change pulsar chart to apache pulsar (#307)
Browse files Browse the repository at this point in the history 
Signed-off-by: Bennu <yunmei.li@zilliz.com>
  • Loading branch information
@Bennu-Li
Bennu-Li authored Mar 22, 2022
1 parent 35cebf5 commit 72d71b5
Show file tree
Hide file tree
Showing 9 changed files with 163 additions and 47 deletions.
2 changes: 1 addition & 1 deletion charts/milvus/Chart.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: milvus
appVersion: "2.0.1"
kubeVersion: "^1.10.0-0"
description: Milvus is an open-source vector database built to power AI applications and vector similarity search.
version: 3.0.10
version: 3.0.11
keywords:
- milvus
- elastic
Expand Down
2 changes: 1 addition & 1 deletion charts/milvus/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ The following table lists the configurable parameters of the Milvus Data Node co
This version of the chart includes the dependent Pulsar chart in the charts/ directory.

You can find more information at:
* [https://github.com/kafkaesque-io/pulsar-helm-chart](https://github.com/kafkaesque-io/pulsar-helm-chart)
* [https://pulsar.apache.org/charts](https://pulsar.apache.org/charts)

### Etcd Configuration

Expand Down
Binary file removed charts/milvus/charts/pulsar-1.0.31.tgz
View file
Binary file not shown.
Binary file added charts/milvus/charts/pulsar-2.7.8.tgz
View file
Binary file not shown.
62 changes: 50 additions & 12 deletions charts/milvus/ci/cluster-values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,9 @@ minio:
memory: 1024Mi
pulsar:
enabled: true
extra:
bastion: no
wsproxy: no
autoRecovery:
components:
pulsar_manager: false
autorecovery:
resources:
requests:
cpu: 0.1
Expand All @@ -24,33 +23,72 @@ pulsar:
requests:
cpu: 0.2
memory: 256Mi
wsResources:
requests:
memory: 256Mi
cpu: 0.1
configData:
PULSAR_MEM: "\"-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m\""
PULSAR_MEM: >
-Xms256m -Xmx256m
PULSAR_GC: >
-XX:MaxDirectMemorySize=256m
bookkeeper:
replicaCount: 2
resources:
requests:
cpu: 0.2
memory: 512Mi
configData:
BOOKIE_MEM: "\"-Xms512m -Xmx512m -XX:MaxDirectMemorySize=512m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -verbosegc\""
PULSAR_MEM: >
-Xms512m -Xmx512m
PULSAR_GC: >
-XX:MaxDirectMemorySize=512m
-Dio.netty.leakDetectionLevel=disabled
-Dio.netty.recycler.linkCapacity=1024
-XX:+UseG1GC -XX:MaxGCPauseMillis=10
-XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions
-XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32
-XX:ConcGCThreads=32 -XX:G1NewSizePercent=50
-XX:+DisableExplicitGC
-XX:-ResizePLAB
-XX:+ExitOnOutOfMemoryError
-XX:+PerfDisableSharedMem
-XX:+PrintGCDetails
zookeeper:
replicaCount: 1
resources:
requests:
cpu: 0.1
memory: 256Mi
configData:
PULSAR_MEM: "\"-Xms256m -Xmx256m -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760 -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC -XX:+PerfDisableSharedMem -Dzookeeper.forceSync=no\""
PULSAR_MEM: >
-Xms256m
-Xmx256m
PULSAR_GC: >
-Dcom.sun.management.jmxremote
-Djute.maxbuffer=10485760
-XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions
-XX:+DoEscapeAnalysis -XX:+DisableExplicitGC
-XX:+PerfDisableSharedMem
-Dzookeeper.forceSync=no
broker:
replicaCount: 1
resources:
requests:
cpu: 0.2
memory: 512Mi
configData:
PULSAR_MEM: "\"-Xms512m -Xmx512m -XX:MaxDirectMemorySize=512m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem\""
PULSAR_MEM: >
-Xms512m
-Xmx512m
PULSAR_GC: >
-XX:MaxDirectMemorySize=512m
-Dio.netty.leakDetectionLevel=disabled
-Dio.netty.recycler.linkCapacity=1024
-XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions
-XX:+DoEscapeAnalysis
-XX:ParallelGCThreads=32
-XX:ConcGCThreads=32
-XX:G1NewSizePercent=50
-XX:+DisableExplicitGC
-XX:-ResizePLAB
-XX:+ExitOnOutOfMemoryError
8 changes: 4 additions & 4 deletions charts/milvus/requirements.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ dependencies:
repository: https://milvus-io.github.io/milvus-helm
version: 8.0.11
- name: pulsar
repository: https://helm.kafkaesque.io
version: 1.0.31
digest: sha256:beb39a6b3f6a84b2915f53d6417c7c0d090567872fb89fbe183c92d59761ffa8
generated: "2022-03-10T14:04:57.494005+08:00"
repository: https://pulsar.apache.org/charts
version: 2.7.8
digest: sha256:5a785d4805e5e75f58240a5f683bb0a47f06d2dd24682e503ce01ab475cc73a1
generated: "2022-03-10T15:19:41.299459+08:00"
4 changes: 2 additions & 2 deletions charts/milvus/requirements.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ dependencies:
tags:
- minio
- name: pulsar
version: 1.0.31
repository: https://helm.kafkaesque.io
version: 2.7.8
repository: https://pulsar.apache.org/charts
condition: pulsar.enabled
tags:
- pulsar
130 changes: 104 additions & 26 deletions charts/milvus/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,46 +430,87 @@ etcd:
value: "2500"

## Configuration values for the pulsar dependency
## ref: https://github.com/kafkaesque-io/pulsar-helm-chart
## ref: https://github.com/apache/pulsar-helm-chart
##

pulsar:
enabled: true
name: pulsar

fullnameOverride: ""
persistence: yes
enableAntiAffinity: no
persistence: true

image:
rbac:
enabled: false
psp: false
limit_to_namespace: true

affinity:
anti_affinity: false

## enableAntiAffinity: no

components:
zookeeper: true
bookkeeper: true
# bookkeeper - autorecovery
autorecovery: true
broker: true
functions: false
proxy: true
toolset: false
pulsar_manager: false

monitoring:
prometheus: false
grafana: false
node_exporter: false
alert_manager: false

images:
broker:
repository: apachepulsar/pulsar
pullPolicy: IfNotPresent
tag: 2.8.2
autorecovery:
repository: apachepulsar/pulsar
tag: 2.8.2
pullPolicy: IfNotPresent
zookeeper:
repository: apachepulsar/pulsar
pullPolicy: IfNotPresent
tag: 2.8.2
bookkeeper:
bookie:
repository: apachepulsar/pulsar
pullPolicy: IfNotPresent
tag: 2.8.2
proxy:
repository: apachepulsar/pulsar
pullPolicy: IfNotPresent
tag: 2.8.2
bastion:
repository: apachepulsar/pulsar
pulsar_manager:
repository: apachepulsar/pulsar-manager
pullPolicy: IfNotPresent
tag: 2.8.2
tag: v0.1.0

zookeeper:
resources:
requests:
memory: 1024Mi
cpu: 0.3
configData:
PULSAR_MEM: "\"-Xms1024m -Xmx1024m -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760 -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC -XX:+PerfDisableSharedMem -Dzookeeper.forceSync=no\""
PULSAR_MEM: >
-Xms1024m
-Xmx1024m
PULSAR_GC: >
-Dcom.sun.management.jmxremote
-Djute.maxbuffer=10485760
-XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions
-XX:+DoEscapeAnalysis
-XX:+DisableExplicitGC
-XX:+PerfDisableSharedMem
-Dzookeeper.forceSync=no
bookkeeper:
replicaCount: 3
Expand All @@ -485,7 +526,25 @@ pulsar:
memory: 2048Mi
cpu: 1
configData:
BOOKIE_MEM: "\"-Xms4096m -Xmx4096m -XX:MaxDirectMemorySize=8192m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -verbosegc\""
PULSAR_MEM: >
-Xms4096m
-Xmx4096m
-XX:MaxDirectMemorySize=8192m
PULSAR_GC: >
-Dio.netty.leakDetectionLevel=disabled
-Dio.netty.recycler.linkCapacity=1024
-XX:+UseG1GC -XX:MaxGCPauseMillis=10
-XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions
-XX:+DoEscapeAnalysis
-XX:ParallelGCThreads=32
-XX:ConcGCThreads=32
-XX:G1NewSizePercent=50
-XX:+DisableExplicitGC
-XX:-ResizePLAB
-XX:+ExitOnOutOfMemoryError
-XX:+PerfDisableSharedMem
-XX:+PrintGCDetails
nettyMaxFrameSizeBytes: "104867840"

broker:
Expand All @@ -498,28 +557,34 @@ pulsar:
memory: 4096Mi
cpu: 1.5
configData:
PULSAR_MEM: "\"-Xms4096m -Xmx4096m -XX:MaxDirectMemorySize=8192m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem\""
PULSAR_MEM: >
-Xms4096m
-Xmx4096m
-XX:MaxDirectMemorySize=8192m
PULSAR_GC: >
-Dio.netty.leakDetectionLevel=disabled
-Dio.netty.recycler.linkCapacity=1024
-XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions
-XX:+DoEscapeAnalysis
-XX:ParallelGCThreads=32
-XX:ConcGCThreads=32
-XX:G1NewSizePercent=50
-XX:+DisableExplicitGC
-XX:-ResizePLAB
-XX:+ExitOnOutOfMemoryError
maxMessageSize: "104857600"
defaultRetentionTimeInMinutes: "10080"
defaultRetentionSizeInMB: "8192"
backlogQuotaDefaultLimitGB: "8"
backlogQuotaDefaultRetentionPolicy: producer_exception

autoRecovery:
autorecovery:
resources:
requests:
memory: 512Mi
cpu: 1

function:
replicaCount: 1
resources:
requests:
memory: 512Mi
cpu: 0.3
configData:
BOOKIE_MEM: "\"-Xms512m -Xmx512m -XX:MaxDirectMemorySize=512m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem\""

proxy:
replicaCount: 1
podMonitor:
Expand All @@ -528,12 +593,25 @@ pulsar:
requests:
memory: 2048Mi
cpu: 1
wsResources:
requests:
memory: 512Mi
cpu: 0.3
service:
type: ClusterIP
configData:
PULSAR_MEM: "\"-Xms2048m -Xmx2048m -XX:MaxDirectMemorySize=2048m\""
PULSAR_MEM: >
-Xms2048m -Xmx2048m
PULSAR_GC: >
-XX:MaxDirectMemorySize=2048m
httpNumThreads: "50"

pulsar_manager:
service:
type: ClusterIP

pulsar_metadata:
component: pulsar-init
image:
# the image used for running `pulsar-cluster-initialize` job
repository: apachepulsar/pulsar
tag: 2.8.2


###################################
Expand Down
2 changes: 1 addition & 1 deletion ct.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ remote: origin
chart-dirs:
- charts
chart-repos:
- kafkaesque=https://helm.kafkaesque.io
- apache=https://pulsar.apache.org/charts
- bitnami=https://charts.bitnami.com/bitnami
- incubator=https://charts.helm.sh/incubator
helm-extra-args: --timeout 900s

0 comments on commit 72d71b5

Please sign in to comment.