Skip to content

Commit 64354b9

Browse files
freeznetfreeznet
authored
feat(config): add package management support (#1280)
1 parent 414105d commit 64354b9

8 files changed

Lines changed: 178 additions & 0 deletions

File tree

‎charts/sn-platform-slim/templates/_helpers.tpl‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,33 @@ Pulsar Cluster Name.
189189
{{- end }}
190190
{{- end }}
191191

192+
{{/*
193+
Package management values validation.
194+
*/}}
195+
{{- define "pulsar.functions.packageManagement.validate" -}}
196+
{{- $pm := .Values.functions.packageManagement | default dict -}}
197+
{{- if $pm.enabled -}}
198+
{{- $functionmesh := .Values.broker.functionmesh | default dict -}}
199+
{{- $mesh := $functionmesh.mesh | default dict -}}
200+
{{- if and $functionmesh.enabled (hasKey $mesh "uploadEnabled") (not $mesh.uploadEnabled) -}}
201+
{{- fail "functions.packageManagement.enabled=true conflicts with broker.functionmesh.mesh.uploadEnabled=false" -}}
202+
{{- end -}}
203+
{{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}}
204+
{{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}}
205+
{{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}}
206+
{{- end -}}
207+
{{- if and (include "pulsar.metadata.isOxia" .) (ne $provider "cloudStorage") -}}
208+
{{- fail "functions.packageManagement.provider must be cloudStorage when Oxia metadata is enabled" -}}
209+
{{- end -}}
210+
{{- if eq $provider "cloudStorage" -}}
211+
{{- $cloud := $pm.cloudStorage | default dict -}}
212+
{{- $_ := required "functions.packageManagement.cloudStorage.type is required when provider=cloudStorage" $cloud.type -}}
213+
{{- $_ := required "functions.packageManagement.cloudStorage.bucket is required when provider=cloudStorage" $cloud.bucket -}}
214+
{{- $_ := required "functions.packageManagement.cloudStorage.bucketPath is required when provider=cloudStorage" $cloud.bucketPath -}}
215+
{{- end -}}
216+
{{- end -}}
217+
{{- end -}}
218+
192219
{{/*
193220
Metadata provider selection.
194221
*/}}

‎charts/sn-platform-slim/templates/broker/broker-cluster.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#
44
# deploy PulsarBroker only when `components.broker and operator.enabled` is true
55
{{- if .Values.components.broker }}
6+
{{- include "pulsar.functions.packageManagement.validate" . }}
67
apiVersion: pulsar.streamnative.io/v1alpha1
78
kind: PulsarBroker
89
metadata:
@@ -336,10 +337,13 @@ spec:
336337
{{- end }}
337338
custom:
338339
PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet"
340+
{{- $pm := .Values.functions.packageManagement | default dict }}
339341
{{- if include "pulsar.metadata.isOxia" . }}
340342
PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
341343
PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
344+
{{- if not $pm.enabled }}
342345
PULSAR_PREFIX_enablePackagesManagement: "false"
346+
{{- end }}
343347
PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true"
344348
PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true"
345349
PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true"
@@ -351,6 +355,21 @@ spec:
351355
PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }}
352356
PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService"
353357
{{- end }}
358+
{{- if $pm.enabled }}
359+
PULSAR_PREFIX_enablePackagesManagement: "true"
360+
{{- if eq $pm.provider "cloudStorage" }}
361+
{{- $cloud := $pm.cloudStorage | default dict }}
362+
PULSAR_PREFIX_packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }}
363+
PULSAR_PREFIX_packagesCloudStorageType: {{ $cloud.type | quote }}
364+
PULSAR_PREFIX_packagesCloudStorageBucket: {{ $cloud.bucket | quote }}
365+
PULSAR_PREFIX_packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }}
366+
{{- else }}
367+
{{- $bookkeeper := $pm.bookkeeper | default dict }}
368+
PULSAR_PREFIX_packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }}
369+
PULSAR_PREFIX_packagesReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize $bookkeeper.replicas | quote }}
370+
PULSAR_PREFIX_packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }}
371+
{{- end }}
372+
{{- end }}
354373
{{- with .Values.broker.configData }}
355374
{{ toYaml . | indent 6 }}
356375
{{- end }}

‎charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#
44

55
{{- if .Values.components.functions }}
6+
{{- include "pulsar.functions.packageManagement.validate" . }}
67
## function config map
78
apiVersion: v1
89
kind: ConfigMap
@@ -27,7 +28,26 @@ data:
2728
tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
2829
{{- end }}
2930
{{- end }}
31+
{{- $pm := .Values.functions.packageManagement | default dict }}
32+
{{- if and $pm.enabled (eq $pm.provider "bookkeeper") }}
33+
numFunctionPackageReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize .Values.functions.packageManagement.bookkeeper.replicas }}
34+
{{- else }}
3035
numFunctionPackageReplicas: {{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }}
36+
{{- end }}
37+
{{- if $pm.enabled }}
38+
enablePackagesManagement: "true"
39+
{{- if eq $pm.provider "cloudStorage" }}
40+
{{- $cloud := $pm.cloudStorage | default dict }}
41+
packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }}
42+
packagesCloudStorageType: {{ $cloud.type | quote }}
43+
packagesCloudStorageBucket: {{ $cloud.bucket | quote }}
44+
packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }}
45+
{{- else }}
46+
{{- $bookkeeper := $pm.bookkeeper | default dict }}
47+
packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }}
48+
packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }}
49+
{{- end }}
50+
{{- end }}
3151
pulsarFunctionsCluster: {{ template "pulsar.fullname" . }}
3252
functionRuntimeFactoryConfigs:
3353
jobNamespace: {{ template "pulsar.functions.namespace" . }}

‎charts/sn-platform-slim/values.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1562,6 +1562,29 @@ functions:
15621562
clusterRole: true
15631563
autoRollDeployment: true
15641564
annotations: {}
1565+
packageManagement:
1566+
# Whether to render explicit package management service config.
1567+
# Defaults to false to preserve existing behavior, especially for Oxia deployments.
1568+
enabled: false
1569+
# Storage provider selector. Valid values: bookkeeper, cloudStorage.
1570+
# When Oxia metadata is enabled, cloudStorage is required if package management is enabled.
1571+
provider: bookkeeper
1572+
bookkeeper:
1573+
storageProvider: "org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider"
1574+
# Defaults to broker.configData.managedLedgerDefaultEnsembleSize when empty.
1575+
replicas: ""
1576+
ledgerRootPath: "/ledgers"
1577+
cloudStorage:
1578+
storageProvider: "io.streamnative.packages.storage.cloud.CloudStorageProvider"
1579+
# Valid values: S3, GCS, AZUREBLOB.
1580+
type: ""
1581+
# Examples: s3a://bucket, gs://bucket, abfs://container@account.
1582+
bucket: ""
1583+
# Examples: /packages, /sub/path.
1584+
bucketPath: ""
1585+
# Optional documentation-only compatibility field matching cloud-api-server.
1586+
# Helm does not parse this; set type, bucket, and bucketPath explicitly.
1587+
storagePath: ""
15651588
ports:
15661589
http: 8080
15671590
https: 8443

‎charts/sn-platform/templates/_helpers.tpl‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,33 @@ Pulsar Cluster Name.
187187
{{- end }}
188188
{{- end }}
189189

190+
{{/*
191+
Package management values validation.
192+
*/}}
193+
{{- define "pulsar.functions.packageManagement.validate" -}}
194+
{{- $pm := .Values.functions.packageManagement | default dict -}}
195+
{{- if $pm.enabled -}}
196+
{{- $functionmesh := .Values.broker.functionmesh | default dict -}}
197+
{{- $mesh := $functionmesh.mesh | default dict -}}
198+
{{- if and $functionmesh.enabled (hasKey $mesh "uploadEnabled") (not $mesh.uploadEnabled) -}}
199+
{{- fail "functions.packageManagement.enabled=true conflicts with broker.functionmesh.mesh.uploadEnabled=false" -}}
200+
{{- end -}}
201+
{{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}}
202+
{{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}}
203+
{{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}}
204+
{{- end -}}
205+
{{- if and (include "pulsar.metadata.isOxia" .) (ne $provider "cloudStorage") -}}
206+
{{- fail "functions.packageManagement.provider must be cloudStorage when Oxia metadata is enabled" -}}
207+
{{- end -}}
208+
{{- if eq $provider "cloudStorage" -}}
209+
{{- $cloud := $pm.cloudStorage | default dict -}}
210+
{{- $_ := required "functions.packageManagement.cloudStorage.type is required when provider=cloudStorage" $cloud.type -}}
211+
{{- $_ := required "functions.packageManagement.cloudStorage.bucket is required when provider=cloudStorage" $cloud.bucket -}}
212+
{{- $_ := required "functions.packageManagement.cloudStorage.bucketPath is required when provider=cloudStorage" $cloud.bucketPath -}}
213+
{{- end -}}
214+
{{- end -}}
215+
{{- end -}}
216+
190217
{{/*
191218
Metadata provider selection.
192219
*/}}

‎charts/sn-platform/templates/broker/broker-cluster.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#
44
# deploy PulsarBroker only when `components.broker and operator.enabled` is true
55
{{- if .Values.components.broker }}
6+
{{- include "pulsar.functions.packageManagement.validate" . }}
67
apiVersion: pulsar.streamnative.io/v1alpha1
78
kind: PulsarBroker
89
metadata:
@@ -337,10 +338,13 @@ spec:
337338
{{- end }}
338339
custom:
339340
PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet"
341+
{{- $pm := .Values.functions.packageManagement | default dict }}
340342
{{- if include "pulsar.metadata.isOxia" . }}
341343
PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
342344
PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
345+
{{- if not $pm.enabled }}
343346
PULSAR_PREFIX_enablePackagesManagement: "false"
347+
{{- end }}
344348
PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true"
345349
PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true"
346350
PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true"
@@ -352,6 +356,21 @@ spec:
352356
PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }}
353357
PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService"
354358
{{- end }}
359+
{{- if $pm.enabled }}
360+
PULSAR_PREFIX_enablePackagesManagement: "true"
361+
{{- if eq $pm.provider "cloudStorage" }}
362+
{{- $cloud := $pm.cloudStorage | default dict }}
363+
PULSAR_PREFIX_packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }}
364+
PULSAR_PREFIX_packagesCloudStorageType: {{ $cloud.type | quote }}
365+
PULSAR_PREFIX_packagesCloudStorageBucket: {{ $cloud.bucket | quote }}
366+
PULSAR_PREFIX_packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }}
367+
{{- else }}
368+
{{- $bookkeeper := $pm.bookkeeper | default dict }}
369+
PULSAR_PREFIX_packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }}
370+
PULSAR_PREFIX_packagesReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize $bookkeeper.replicas | quote }}
371+
PULSAR_PREFIX_packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }}
372+
{{- end }}
373+
{{- end }}
355374
{{- with .Values.broker.configData }}
356375
{{ toYaml . | indent 6 }}
357376
{{- end }}

‎charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#
44

55
{{- if .Values.components.functions }}
6+
{{- include "pulsar.functions.packageManagement.validate" . }}
67
## function config map
78
apiVersion: v1
89
kind: ConfigMap
@@ -27,7 +28,26 @@ data:
2728
tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt"
2829
{{- end }}
2930
{{- end }}
31+
{{- $pm := .Values.functions.packageManagement | default dict }}
32+
{{- if and $pm.enabled (eq $pm.provider "bookkeeper") }}
33+
numFunctionPackageReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize .Values.functions.packageManagement.bookkeeper.replicas }}
34+
{{- else }}
3035
numFunctionPackageReplicas: {{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }}
36+
{{- end }}
37+
{{- if $pm.enabled }}
38+
enablePackagesManagement: "true"
39+
{{- if eq $pm.provider "cloudStorage" }}
40+
{{- $cloud := $pm.cloudStorage | default dict }}
41+
packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }}
42+
packagesCloudStorageType: {{ $cloud.type | quote }}
43+
packagesCloudStorageBucket: {{ $cloud.bucket | quote }}
44+
packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }}
45+
{{- else }}
46+
{{- $bookkeeper := $pm.bookkeeper | default dict }}
47+
packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }}
48+
packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }}
49+
{{- end }}
50+
{{- end }}
3151
pulsarFunctionsCluster: {{ template "pulsar.fullname" . }}
3252
functionRuntimeFactoryConfigs:
3353
jobNamespace: {{ template "pulsar.functions.namespace" . }}

‎charts/sn-platform/values.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1638,6 +1638,29 @@ functions:
16381638
clusterRole: true
16391639
autoRollDeployment: true
16401640
annotations: {}
1641+
packageManagement:
1642+
# Whether to render explicit package management service config.
1643+
# Defaults to false to preserve existing behavior, especially for Oxia deployments.
1644+
enabled: false
1645+
# Storage provider selector. Valid values: bookkeeper, cloudStorage.
1646+
# When Oxia metadata is enabled, cloudStorage is required if package management is enabled.
1647+
provider: bookkeeper
1648+
bookkeeper:
1649+
storageProvider: "org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider"
1650+
# Defaults to broker.configData.managedLedgerDefaultEnsembleSize when empty.
1651+
replicas: ""
1652+
ledgerRootPath: "/ledgers"
1653+
cloudStorage:
1654+
storageProvider: "io.streamnative.packages.storage.cloud.CloudStorageProvider"
1655+
# Valid values: S3, GCS, AZUREBLOB.
1656+
type: ""
1657+
# Examples: s3a://bucket, gs://bucket, abfs://container@account.
1658+
bucket: ""
1659+
# Examples: /packages, /sub/path.
1660+
bucketPath: ""
1661+
# Optional documentation-only compatibility field matching cloud-api-server.
1662+
# Helm does not parse this; set type, bucket, and bucketPath explicitly.
1663+
storagePath: ""
16411664
ports:
16421665
http: 8080
16431666
https: 8443

0 commit comments

Comments
 (0)