Skip to content

Commit 8aa95d7

Browse files
mattisonchaomattisonchao
authored
feat: split KoP schema registry namespace (#1278)
## Motivation - Oxia metadata deployments currently let KoP schema registry fall back to the broker metadata namespace when `oxiaSchemaRegistryUrl` is not set. - KoP reads `oxiaSchemaRegistryUrl`, while Pulsar schema storage is controlled separately by `schemaRegistryStorageClassName` and `oxiaSchemaStorageUrl`. ## Modifications - Add a `kafka-schema` Oxia namespace helper and render a matching `OxiaNamespace`. - Configure KoP schema registry to use `kafka-schema` via `PULSAR_PREFIX_oxiaSchemaRegistryUrl`. - Replace the `oxiaBasedSystemTopic` chart block with explicit broker custom config so Kafka schema registry, group offsets, producer state, and topic-policy settings are visible in chart output. - Do not force `schemaRegistryStorageClassName`; the current default `3.3.3.3` image does not include `io.streamnative.pulsar.schema.OxiaSchemaStorageFactory`. Pulsar schema metadata therefore remains on the default schema storage path unless users provide an image/plugin and set that storage class explicitly. ## Testing - `git diff --check` - `helm lint charts/sn-platform` - `helm lint charts/sn-platform-slim` - `helm template split-schema charts/sn-platform --namespace pulsar --set pulsar_metadata.provider=oxia` - `helm template split-schema-slim charts/sn-platform-slim --namespace pulsar --set pulsar_metadata.provider=oxia` - Fresh local kind deployment on context `kind-matt` with `charts/sn-platform-slim`, Oxia metadata, 2 brokers, and existing `sn-license` secret. Verified all pods became ready and rendered config uses `/kafka-schema` for `oxiaSchemaRegistryUrl`.
1 parent 3511e25 commit 8aa95d7

10 files changed

Lines changed: 124 additions & 16 deletions

File tree

‎charts/sn-platform-slim/templates/_helpers.tpl‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,10 @@ bookkeeper
228228
pulsar-schema
229229
{{- end -}}
230230

231+
{{- define "pulsar.oxia.kafkaSchema.namespace" -}}
232+
kafka-schema
233+
{{- end -}}
234+
231235
{{- define "pulsar.oxia.function.namespace" -}}
232236
function
233237
{{- end -}}
@@ -244,6 +248,10 @@ oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.b
244248
oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.schema.namespace" . }}
245249
{{- end -}}
246250

251+
{{- define "pulsar.oxia.kafkaSchema.url" -}}
252+
oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.kafkaSchema.namespace" . }}
253+
{{- end -}}
254+
247255
{{- define "pulsar.oxia.metadata.url" -}}
248256
{{- template "pulsar.oxia.broker.url" . -}}
249257
{{- end -}}

‎charts/sn-platform-slim/templates/broker/broker-cluster.yaml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -232,12 +232,6 @@ spec:
232232
{{- if .Values.pulsar_metadata.clusterName }}
233233
clusterName: {{ .Values.pulsar_metadata.clusterName }}
234234
{{- end }}
235-
{{- if include "pulsar.metadata.isOxia" . }}
236-
oxiaBasedSystemTopic:
237-
enabled: true
238-
schemaStorageUrl: "{{ template "pulsar.oxia.schema.url" . }}"
239-
topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService"
240-
{{- end }}
241235
{{- if .Values.broker.functionmesh.enabled }}
242236
function:
243237
enabled: true
@@ -333,13 +327,26 @@ spec:
333327
{{- end }}
334328
{{- end }}
335329

330+
{{- end }}
331+
{{- if include "pulsar.metadata.isOxia" . }}
332+
useStorageCatalog: true
336333
{{- end }}
337334
custom:
338335
PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet"
339336
{{- if include "pulsar.metadata.isOxia" . }}
340337
PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
341338
PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
342339
PULSAR_PREFIX_enablePackagesManagement: "false"
340+
PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true"
341+
PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true"
342+
PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true"
343+
{{- $oxia := default dict .Values.oxia }}
344+
{{- $storageCatalog := default dict $oxia.storageCatalog }}
345+
PULSAR_PREFIX_oxiaSchemaStorageUrl: {{ default (include "pulsar.oxia.schema.url" .) $storageCatalog.schemaStorageUrl | quote }}
346+
PULSAR_PREFIX_oxiaSchemaRegistryUrl: "{{ template "pulsar.oxia.kafkaSchema.url" . }}"
347+
PULSAR_PREFIX_schemaRegistryUrl: {{ default (include "pulsar.oxia.kafkaSchema.url" .) $storageCatalog.schemaRegistryUrl | quote }}
348+
PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }}
349+
PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService"
343350
{{- end }}
344351
{{- with .Values.broker.configData }}
345352
{{ toYaml . | indent 6 }}

‎charts/sn-platform-slim/templates/oxia/oxia-namespace.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
{{- $brokerNamespace := dict "name" (include "pulsar.oxia.broker.namespace" .) "type" "broker" }}
77
{{- $bookkeeperNamespace := dict "name" (include "pulsar.oxia.bookkeeper.namespace" .) "type" "bookkeeper" }}
88
{{- $schemaNamespace := dict "name" (include "pulsar.oxia.schema.namespace" .) "type" "pulsar-schema" }}
9+
{{- $kafkaSchemaNamespace := dict "name" (include "pulsar.oxia.kafkaSchema.namespace" .) "type" "kafka-schema" }}
910
{{- $functionNamespace := dict "name" (include "pulsar.oxia.function.namespace" .) "type" "function" }}
10-
{{- $namespaces := list $brokerNamespace $bookkeeperNamespace $schemaNamespace $functionNamespace }}
11+
{{- $namespaces := list $brokerNamespace $bookkeeperNamespace $schemaNamespace $kafkaSchemaNamespace $functionNamespace }}
1112
{{- range $index, $oxiaNamespace := $namespaces }}
1213
{{- if $index }}
1314
---

‎charts/sn-platform-slim/templates/oxia/storage-catalog.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#
2+
# Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
3+
#
4+
# deploy StorageCatalog only when `pulsar_metadata.provider` is oxia
5+
{{- if and .Values.components.broker (include "pulsar.metadata.isOxia" .) }}
6+
{{- $oxia := default dict .Values.oxia }}
7+
{{- $storageCatalog := default dict $oxia.storageCatalog }}
8+
apiVersion: k8s.streamnative.io/v1alpha1
9+
kind: StorageCatalog
10+
metadata:
11+
name: "{{ template "pulsar.fullname" . }}"
12+
namespace: {{ template "pulsar.namespace" . }}
13+
labels:
14+
{{- include "pulsar.standardLabels" . | nindent 4 }}
15+
component: broker
16+
spec:
17+
oxiaMetadataServiceUrl: {{ default (include "pulsar.oxia.metadata.url" .) $storageCatalog.oxiaMetadataServiceUrl | quote }}
18+
schemaStorageUrl: {{ default (include "pulsar.oxia.schema.url" .) $storageCatalog.schemaStorageUrl | quote }}
19+
schemaRegistryUrl: {{ default (include "pulsar.oxia.kafkaSchema.url" .) $storageCatalog.schemaRegistryUrl | quote }}
20+
schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }}
21+
{{- with $storageCatalog.storageUrl }}
22+
storageUrl: {{ . | quote }}
23+
{{- end }}
24+
{{- with $storageCatalog.backendStorageType }}
25+
backendStorageType: {{ . | quote }}
26+
{{- end }}
27+
{{- with $storageCatalog.bucket }}
28+
bucket: {{ . | quote }}
29+
{{- end }}
30+
{{- with $storageCatalog.prefix }}
31+
prefix: {{ . | quote }}
32+
{{- end }}
33+
{{- with $storageCatalog.region }}
34+
region: {{ . | quote }}
35+
{{- end }}
36+
{{- if hasKey $storageCatalog "useOwnStorage" }}
37+
useOwnStorage: {{ $storageCatalog.useOwnStorage }}
38+
{{- end }}
39+
{{- end }}

‎charts/sn-platform-slim/values.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1159,7 +1159,6 @@ pulsar_metadata:
11591159
# limits:
11601160
# cpu: "1"
11611161
# memory: "1Gi"
1162-
11631162
## deprecated: move to broker.kop
11641163
# kop:
11651164
# ports:

‎charts/sn-platform/templates/_helpers.tpl‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,10 @@ bookkeeper
226226
pulsar-schema
227227
{{- end -}}
228228

229+
{{- define "pulsar.oxia.kafkaSchema.namespace" -}}
230+
kafka-schema
231+
{{- end -}}
232+
229233
{{- define "pulsar.oxia.function.namespace" -}}
230234
function
231235
{{- end -}}
@@ -242,6 +246,10 @@ oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.b
242246
oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.schema.namespace" . }}
243247
{{- end -}}
244248

249+
{{- define "pulsar.oxia.kafkaSchema.url" -}}
250+
oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.kafkaSchema.namespace" . }}
251+
{{- end -}}
252+
245253
{{- define "pulsar.oxia.metadata.url" -}}
246254
{{- template "pulsar.oxia.broker.url" . -}}
247255
{{- end -}}

‎charts/sn-platform/templates/broker/broker-cluster.yaml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -233,12 +233,6 @@ spec:
233233
{{- if .Values.pulsar_metadata.clusterName }}
234234
clusterName: {{ .Values.pulsar_metadata.clusterName }}
235235
{{- end }}
236-
{{- if include "pulsar.metadata.isOxia" . }}
237-
oxiaBasedSystemTopic:
238-
enabled: true
239-
schemaStorageUrl: "{{ template "pulsar.oxia.schema.url" . }}"
240-
topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService"
241-
{{- end }}
242236
{{- if .Values.broker.functionmesh.enabled }}
243237
function:
244238
enabled: true
@@ -334,13 +328,26 @@ spec:
334328
{{- end }}
335329
{{- end }}
336330

331+
{{- end }}
332+
{{- if include "pulsar.metadata.isOxia" . }}
333+
useStorageCatalog: true
337334
{{- end }}
338335
custom:
339336
PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet"
340337
{{- if include "pulsar.metadata.isOxia" . }}
341338
PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
342339
PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
343340
PULSAR_PREFIX_enablePackagesManagement: "false"
341+
PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true"
342+
PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true"
343+
PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true"
344+
{{- $oxia := default dict .Values.oxia }}
345+
{{- $storageCatalog := default dict $oxia.storageCatalog }}
346+
PULSAR_PREFIX_oxiaSchemaStorageUrl: {{ default (include "pulsar.oxia.schema.url" .) $storageCatalog.schemaStorageUrl | quote }}
347+
PULSAR_PREFIX_oxiaSchemaRegistryUrl: "{{ template "pulsar.oxia.kafkaSchema.url" . }}"
348+
PULSAR_PREFIX_schemaRegistryUrl: {{ default (include "pulsar.oxia.kafkaSchema.url" .) $storageCatalog.schemaRegistryUrl | quote }}
349+
PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }}
350+
PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService"
344351
{{- end }}
345352
{{- with .Values.broker.configData }}
346353
{{ toYaml . | indent 6 }}

‎charts/sn-platform/templates/oxia/oxia-namespace.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
{{- $brokerNamespace := dict "name" (include "pulsar.oxia.broker.namespace" .) "type" "broker" }}
77
{{- $bookkeeperNamespace := dict "name" (include "pulsar.oxia.bookkeeper.namespace" .) "type" "bookkeeper" }}
88
{{- $schemaNamespace := dict "name" (include "pulsar.oxia.schema.namespace" .) "type" "pulsar-schema" }}
9+
{{- $kafkaSchemaNamespace := dict "name" (include "pulsar.oxia.kafkaSchema.namespace" .) "type" "kafka-schema" }}
910
{{- $functionNamespace := dict "name" (include "pulsar.oxia.function.namespace" .) "type" "function" }}
10-
{{- $namespaces := list $brokerNamespace $bookkeeperNamespace $schemaNamespace $functionNamespace }}
11+
{{- $namespaces := list $brokerNamespace $bookkeeperNamespace $schemaNamespace $kafkaSchemaNamespace $functionNamespace }}
1112
{{- range $index, $oxiaNamespace := $namespaces }}
1213
{{- if $index }}
1314
---

‎charts/sn-platform/templates/oxia/storage-catalog.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#
2+
# Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
3+
#
4+
# deploy StorageCatalog only when `pulsar_metadata.provider` is oxia
5+
{{- if and .Values.components.broker (include "pulsar.metadata.isOxia" .) }}
6+
{{- $oxia := default dict .Values.oxia }}
7+
{{- $storageCatalog := default dict $oxia.storageCatalog }}
8+
apiVersion: k8s.streamnative.io/v1alpha1
9+
kind: StorageCatalog
10+
metadata:
11+
name: "{{ template "pulsar.fullname" . }}"
12+
namespace: {{ template "pulsar.namespace" . }}
13+
labels:
14+
{{- include "pulsar.standardLabels" . | nindent 4 }}
15+
component: broker
16+
spec:
17+
oxiaMetadataServiceUrl: {{ default (include "pulsar.oxia.metadata.url" .) $storageCatalog.oxiaMetadataServiceUrl | quote }}
18+
schemaStorageUrl: {{ default (include "pulsar.oxia.schema.url" .) $storageCatalog.schemaStorageUrl | quote }}
19+
schemaRegistryUrl: {{ default (include "pulsar.oxia.kafkaSchema.url" .) $storageCatalog.schemaRegistryUrl | quote }}
20+
schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }}
21+
{{- with $storageCatalog.storageUrl }}
22+
storageUrl: {{ . | quote }}
23+
{{- end }}
24+
{{- with $storageCatalog.backendStorageType }}
25+
backendStorageType: {{ . | quote }}
26+
{{- end }}
27+
{{- with $storageCatalog.bucket }}
28+
bucket: {{ . | quote }}
29+
{{- end }}
30+
{{- with $storageCatalog.prefix }}
31+
prefix: {{ . | quote }}
32+
{{- end }}
33+
{{- with $storageCatalog.region }}
34+
region: {{ . | quote }}
35+
{{- end }}
36+
{{- if hasKey $storageCatalog "useOwnStorage" }}
37+
useOwnStorage: {{ $storageCatalog.useOwnStorage }}
38+
{{- end }}
39+
{{- end }}

‎charts/sn-platform/values.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1234,7 +1234,6 @@ pulsar_metadata:
12341234
# limits:
12351235
# cpu: "1"
12361236
# memory: "1Gi"
1237-
12381237
## deprecated: move to broker.kop
12391238
# kop:
12401239
# ports:

0 commit comments

Comments
 (0)